From 84efdb7c607648324e4c8d8e905ff272c2682895 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Mon, 4 Aug 2025 00:08:40 +0400 Subject: [PATCH 01/98] feat: set proposal vp value on proposal creation chore: fix types --- src/helpers/vpValue.ts | 58 ++++++++++++++++++++++++++++++++++++++++++ src/writer/proposal.ts | 3 +++ 2 files changed, 61 insertions(+) create mode 100644 src/helpers/vpValue.ts diff --git a/src/helpers/vpValue.ts b/src/helpers/vpValue.ts new file mode 100644 index 00000000..c0e09e3d --- /dev/null +++ b/src/helpers/vpValue.ts @@ -0,0 +1,58 @@ +import hubDB from './mysql'; +import { fetchWithKeepAlive } from './utils'; + +type Proposal = { + id: string; + network: string; + strategies: any[]; + start: number; +}; + +const OVERLORD_URL = 'https://overlord.snapshot.org'; +const CB_LAST = 5; +const CB_ERROR = 0; + +export async function setProposalVpValue(proposal: Proposal) { + if (proposal.start > Date.now()) { + return; + } + + try { + const vpValue = await getVpValue(proposal); + + await hubDB.queryAsync('UPDATE proposals SET vp_value = ?, cb = ? WHERE id = ? LIMIT 1', [ + vpValue, + CB_LAST, + proposal.id + ]); + } catch { + await hubDB.queryAsync('UPDATE proposals SET cb = ? WHERE id = ? LIMIT 1', [ + CB_ERROR, + proposal.id + ]); + return; + } +} + +async function getVpValue(proposal: Proposal) { + const init = { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + jsonrpc: '2.0', + method: 'get_vp_value_by_strategy', + params: { + network: proposal.network, + strategies: proposal.strategies, + snapshot: proposal.start + }, + id: Math.random().toString(36).substring(7) + }) + }; + const res = await fetchWithKeepAlive(OVERLORD_URL, init); + const { result } = await res.json(); + return result; +} diff --git a/src/writer/proposal.ts b/src/writer/proposal.ts index 9cec1e74..eeded875 100644 --- a/src/writer/proposal.ts +++ b/src/writer/proposal.ts @@ -10,6 +10,7 @@ import { isMalicious } from '../helpers/monitoring'; import db from '../helpers/mysql'; import { getLimits, getSpaceType } from '../helpers/options'; import { captureError, getQuorum, jsonParse, validateChoices } from '../helpers/utils'; +import { setProposalVpValue } from '../helpers/vpValue'; const scoreAPIUrl = process.env.SCORE_API_URL || 'https://score.snapshot.org'; const broviderUrl = process.env.BROVIDER_URL || 'https://rpc.snapshot.org'; @@ -316,4 +317,6 @@ export async function action(body, ipfs, receipt, id): Promise { `; await db.queryAsync(query, [proposal, space, author, space]); + + setProposalVpValue({ ...proposal, strategies: spaceSettings.strategies }); } From ed31f785d1e56a32ec025cdb4214432019484362 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Mon, 4 Aug 2025 00:08:40 +0400 Subject: [PATCH 02/98] feat: set proposal strategies value on proposal creation fix: fix botched merge conflict chore: remove typo fix: fix typescript error fix: fix lint error fix: delete unused file --- .../{vpValue.ts => strategiesValue.ts} | 28 +--------------- src/writer/proposal.ts | 33 ++++++++++++++++--- test/integration/writer/proposal.test.ts | 4 +-- 3 files changed, 31 insertions(+), 34 deletions(-) rename src/helpers/{vpValue.ts => strategiesValue.ts} (54%) diff --git a/src/helpers/vpValue.ts b/src/helpers/strategiesValue.ts similarity index 54% rename from src/helpers/vpValue.ts rename to src/helpers/strategiesValue.ts index c0e09e3d..38805fd6 100644 --- a/src/helpers/vpValue.ts +++ b/src/helpers/strategiesValue.ts @@ -1,40 +1,14 @@ -import hubDB from './mysql'; import { fetchWithKeepAlive } from './utils'; type Proposal = { - id: string; network: string; strategies: any[]; start: number; }; const OVERLORD_URL = 'https://overlord.snapshot.org'; -const CB_LAST = 5; -const CB_ERROR = 0; -export async function setProposalVpValue(proposal: Proposal) { - if (proposal.start > Date.now()) { - return; - } - - try { - const vpValue = await getVpValue(proposal); - - await hubDB.queryAsync('UPDATE proposals SET vp_value = ?, cb = ? WHERE id = ? LIMIT 1', [ - vpValue, - CB_LAST, - proposal.id - ]); - } catch { - await hubDB.queryAsync('UPDATE proposals SET cb = ? WHERE id = ? LIMIT 1', [ - CB_ERROR, - proposal.id - ]); - return; - } -} - -async function getVpValue(proposal: Proposal) { +export default async function getStrategiesValue(proposal: Proposal): Promise { const init = { method: 'POST', headers: { diff --git a/src/writer/proposal.ts b/src/writer/proposal.ts index eeded875..d8020712 100644 --- a/src/writer/proposal.ts +++ b/src/writer/proposal.ts @@ -9,8 +9,8 @@ import { containsFlaggedLinks, flaggedAddresses } from '../helpers/moderation'; import { isMalicious } from '../helpers/monitoring'; import db from '../helpers/mysql'; import { getLimits, getSpaceType } from '../helpers/options'; +import getStrategiesValue from '../helpers/strategiesValue'; import { captureError, getQuorum, jsonParse, validateChoices } from '../helpers/utils'; -import { setProposalVpValue } from '../helpers/vpValue'; const scoreAPIUrl = process.env.SCORE_API_URL || 'https://score.snapshot.org'; const broviderUrl = process.env.BROVIDER_URL || 'https://rpc.snapshot.org'; @@ -242,9 +242,34 @@ export async function verify(body): Promise { if (msg.payload.choices.length > choicesLimit) { return Promise.reject(`number of choices can not exceed ${choicesLimit}`); } + + let strategiesValue: number[] = []; + + try { + strategiesValue = await getStrategiesValue({ + network: space.network, + start: msg.payload.start, + strategies: space.strategies + }); + + // Handle unlikely case where strategies value array length does not match strategies length + if (strategiesValue.length !== space.strategies.length) { + capture(new Error('Strategies value length mismatch'), { + space: space.id, + strategiesLength: space.strategies.length, + strategiesValue: JSON.stringify(strategiesValue) + }); + return Promise.reject('failed to get strategies value'); + } + } catch (e: any) { + console.log('unable to get strategies value', e.message); + return Promise.reject('failed to get strategies value'); + } + + return { strategiesValue }; } -export async function action(body, ipfs, receipt, id): Promise { +export async function action(body, ipfs, receipt, id, context): Promise { const msg = jsonParse(body.msg); const space = msg.space; @@ -302,7 +327,7 @@ export async function action(body, ipfs, receipt, id): Promise { scores_state: 'pending', scores_total: 0, scores_updated: 0, - vp_value_by_strategy: JSON.stringify([]), + vp_value_by_strategy: JSON.stringify(context.strategiesValue), votes: 0, validation, flagged: +containsFlaggedLinks(msg.payload.body) @@ -317,6 +342,4 @@ export async function action(body, ipfs, receipt, id): Promise { `; await db.queryAsync(query, [proposal, space, author, space]); - - setProposalVpValue({ ...proposal, strategies: spaceSettings.strategies }); } diff --git a/test/integration/writer/proposal.test.ts b/test/integration/writer/proposal.test.ts index 28565d67..dd9507f5 100644 --- a/test/integration/writer/proposal.test.ts +++ b/test/integration/writer/proposal.test.ts @@ -45,7 +45,7 @@ describe('writer/proposal', () => { expect.hasAssertions(); mockContainsFlaggedLinks.mockReturnValueOnce(true); const id = '0x01-flagged'; - expect(await action(input, 'ipfs', 'receipt', id)).toBeUndefined(); + expect(await action(input, 'ipfs', 'receipt', id, {})).toBeUndefined(); expect(mockContainsFlaggedLinks).toBeCalledTimes(1); const [proposal] = await db.queryAsync('SELECT * FROM proposals WHERE id = ?', [id]); @@ -57,7 +57,7 @@ describe('writer/proposal', () => { it('creates and does not flag proposal', async () => { expect.hasAssertions(); const id = '0x02-non-flagged'; - expect(await action(input, 'ipfs', 'receipt', id)).toBeUndefined(); + expect(await action(input, 'ipfs', 'receipt', id, {})).toBeUndefined(); expect(mockContainsFlaggedLinks).toBeCalledTimes(1); const [proposal] = await db.queryAsync('SELECT * FROM proposals WHERE id = ?', [id]); From 599f9e46215ba61ee85678d31d8f6cd3394c2775 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 6 Aug 2025 00:34:03 +0400 Subject: [PATCH 03/98] fix: set `cb` column --- .env.example | 7 ++++--- src/writer/proposal.ts | 5 ++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index 3ff9dd04..294931c7 100644 --- a/.env.example +++ b/.env.example @@ -11,11 +11,12 @@ PINEAPPLE_URL=https://.../upload # optional SCORE_API_URL=https://score.snapshot.org SCHNAPS_API_URL=https://schnaps.snapshot.box # If you need unlimted access to score-api, use `https://score.snapshot.org?apiKey=...` -RATE_LIMIT_DATABASE_URL= # optional +RATE_LIMIT_DATABASE_URL= # optional RATE_LIMIT_KEYS_PREFIX=snapshot-sequencer: # optional -BROVIDER_URL=https://rpc.snapshot.org # optional -STARKNET_RPC_URL= # optional +BROVIDER_URL=https://rpc.snapshot.org # optional +STARKNET_RPC_URL= # optional # Secret for laser AUTH_SECRET=1dfd84a695705665668c260222ded178d1f1d62d251d7bee8148428dac6d0487 # optional WALLETCONNECT_PROJECT_ID=e6454bd61aba40b786e866a69bd4c5c6 REOWN_SECRET= # optional +LAST_CB=0 diff --git a/src/writer/proposal.ts b/src/writer/proposal.ts index d8020712..c530d9f2 100644 --- a/src/writer/proposal.ts +++ b/src/writer/proposal.ts @@ -14,6 +14,7 @@ import { captureError, getQuorum, jsonParse, validateChoices } from '../helpers/ const scoreAPIUrl = process.env.SCORE_API_URL || 'https://score.snapshot.org'; const broviderUrl = process.env.BROVIDER_URL || 'https://rpc.snapshot.org'; +const LAST_CB = process.env.LAST_CB ?? 0; export const getProposalsCount = async (space, author) => { const query = ` @@ -327,10 +328,12 @@ export async function action(body, ipfs, receipt, id, context): Promise { scores_state: 'pending', scores_total: 0, scores_updated: 0, + scores_total_value: 0, vp_value_by_strategy: JSON.stringify(context.strategiesValue), votes: 0, validation, - flagged: +containsFlaggedLinks(msg.payload.body) + flagged: +containsFlaggedLinks(msg.payload.body), + cb: LAST_CB }; const query = ` From cb9859ba322ab230aa855e43bdf0dd466df3ae14 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 6 Aug 2025 00:40:49 +0400 Subject: [PATCH 04/98] fix: default last_cb to 1 --- .env.example | 2 +- src/writer/proposal.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 294931c7..fb0da2e8 100644 --- a/.env.example +++ b/.env.example @@ -19,4 +19,4 @@ STARKNET_RPC_URL= # optional AUTH_SECRET=1dfd84a695705665668c260222ded178d1f1d62d251d7bee8148428dac6d0487 # optional WALLETCONNECT_PROJECT_ID=e6454bd61aba40b786e866a69bd4c5c6 REOWN_SECRET= # optional -LAST_CB=0 +LAST_CB=1 diff --git a/src/writer/proposal.ts b/src/writer/proposal.ts index c530d9f2..1057c52b 100644 --- a/src/writer/proposal.ts +++ b/src/writer/proposal.ts @@ -14,7 +14,7 @@ import { captureError, getQuorum, jsonParse, validateChoices } from '../helpers/ const scoreAPIUrl = process.env.SCORE_API_URL || 'https://score.snapshot.org'; const broviderUrl = process.env.BROVIDER_URL || 'https://rpc.snapshot.org'; -const LAST_CB = process.env.LAST_CB ?? 0; +const LAST_CB = parseInt(process.env.LAST_CB ?? '1'); export const getProposalsCount = async (space, author) => { const query = ` From e589d82203cbca67c30ac5f8a685312dff69ab79 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 6 Aug 2025 00:45:11 +0400 Subject: [PATCH 05/98] fix: round strategies value up to 9 decimals --- src/writer/proposal.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/writer/proposal.ts b/src/writer/proposal.ts index 1057c52b..17b8ba9e 100644 --- a/src/writer/proposal.ts +++ b/src/writer/proposal.ts @@ -262,6 +262,8 @@ export async function verify(body): Promise { }); return Promise.reject('failed to get strategies value'); } + + strategiesValue = strategiesValue.map(value => parseFloat(value.toFixed(9))); } catch (e: any) { console.log('unable to get strategies value', e.message); return Promise.reject('failed to get strategies value'); From 4fb62beee338842e4ef3a0ac97050f0f6eab5b04 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 6 Aug 2025 00:47:44 +0400 Subject: [PATCH 06/98] refactor: move precision to a const --- src/writer/proposal.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/writer/proposal.ts b/src/writer/proposal.ts index 17b8ba9e..b0c601a4 100644 --- a/src/writer/proposal.ts +++ b/src/writer/proposal.ts @@ -15,6 +15,7 @@ import { captureError, getQuorum, jsonParse, validateChoices } from '../helpers/ const scoreAPIUrl = process.env.SCORE_API_URL || 'https://score.snapshot.org'; const broviderUrl = process.env.BROVIDER_URL || 'https://rpc.snapshot.org'; const LAST_CB = parseInt(process.env.LAST_CB ?? '1'); +const STRATEGIES_VALUE_PRECISION = 9; // Precision for strategies value export const getProposalsCount = async (space, author) => { const query = ` @@ -263,7 +264,9 @@ export async function verify(body): Promise { return Promise.reject('failed to get strategies value'); } - strategiesValue = strategiesValue.map(value => parseFloat(value.toFixed(9))); + strategiesValue = strategiesValue.map(value => + parseFloat(value.toFixed(STRATEGIES_VALUE_PRECISION)) + ); } catch (e: any) { console.log('unable to get strategies value', e.message); return Promise.reject('failed to get strategies value'); From 43def63f8bea137405183141053e70bbf8fc8a89 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 6 Aug 2025 00:51:22 +0400 Subject: [PATCH 07/98] chore: fix typo --- .env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.example b/.env.example index fb0da2e8..22ff6a63 100644 --- a/.env.example +++ b/.env.example @@ -10,7 +10,7 @@ SIDEKICK_URL=https://sh5.co PINEAPPLE_URL=https://.../upload # optional SCORE_API_URL=https://score.snapshot.org SCHNAPS_API_URL=https://schnaps.snapshot.box -# If you need unlimted access to score-api, use `https://score.snapshot.org?apiKey=...` +# If you need unlimited access to score-api, use `https://score.snapshot.org?apiKey=...` RATE_LIMIT_DATABASE_URL= # optional RATE_LIMIT_KEYS_PREFIX=snapshot-sequencer: # optional BROVIDER_URL=https://rpc.snapshot.org # optional From 4510db7f0c85b88a95d44f1a7a954b698904d423 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 6 Aug 2025 00:56:04 +0400 Subject: [PATCH 08/98] test: fix test test: fix tests --- test/integration/writer/proposal.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/writer/proposal.test.ts b/test/integration/writer/proposal.test.ts index dd9507f5..68141e64 100644 --- a/test/integration/writer/proposal.test.ts +++ b/test/integration/writer/proposal.test.ts @@ -45,7 +45,7 @@ describe('writer/proposal', () => { expect.hasAssertions(); mockContainsFlaggedLinks.mockReturnValueOnce(true); const id = '0x01-flagged'; - expect(await action(input, 'ipfs', 'receipt', id, {})).toBeUndefined(); + expect(await action(input, 'ipfs', 'receipt', id, { strategiesValue: [] })).toBeUndefined(); expect(mockContainsFlaggedLinks).toBeCalledTimes(1); const [proposal] = await db.queryAsync('SELECT * FROM proposals WHERE id = ?', [id]); @@ -57,7 +57,7 @@ describe('writer/proposal', () => { it('creates and does not flag proposal', async () => { expect.hasAssertions(); const id = '0x02-non-flagged'; - expect(await action(input, 'ipfs', 'receipt', id, {})).toBeUndefined(); + expect(await action(input, 'ipfs', 'receipt', id, { strategiesValue: [] })).toBeUndefined(); expect(mockContainsFlaggedLinks).toBeCalledTimes(1); const [proposal] = await db.queryAsync('SELECT * FROM proposals WHERE id = ?', [id]); From 8a28e02b787889620d476e4f0378fc7483e4f875 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 6 Aug 2025 01:41:10 +0400 Subject: [PATCH 09/98] test: fix tests --- src/helpers/strategiesValue.ts | 11 +++++++ src/writer/proposal.ts | 10 ------ test/integration/ingestor.test.ts | 5 +++ test/integration/writer/proposal.test.ts | 5 +++ test/unit/writer/proposal.test.ts | 41 +++++++++++++----------- 5 files changed, 44 insertions(+), 28 deletions(-) diff --git a/src/helpers/strategiesValue.ts b/src/helpers/strategiesValue.ts index 38805fd6..0a8123b0 100644 --- a/src/helpers/strategiesValue.ts +++ b/src/helpers/strategiesValue.ts @@ -1,3 +1,4 @@ +import { capture } from '@snapshot-labs/snapshot-sentry'; import { fetchWithKeepAlive } from './utils'; type Proposal = { @@ -28,5 +29,15 @@ export default async function getStrategiesValue(proposal: Proposal): Promise { strategies: space.strategies }); - // Handle unlikely case where strategies value array length does not match strategies length - if (strategiesValue.length !== space.strategies.length) { - capture(new Error('Strategies value length mismatch'), { - space: space.id, - strategiesLength: space.strategies.length, - strategiesValue: JSON.stringify(strategiesValue) - }); - return Promise.reject('failed to get strategies value'); - } - strategiesValue = strategiesValue.map(value => parseFloat(value.toFixed(STRATEGIES_VALUE_PRECISION)) ); diff --git a/test/integration/ingestor.test.ts b/test/integration/ingestor.test.ts index bc2483b0..ff4d5a1b 100644 --- a/test/integration/ingestor.test.ts +++ b/test/integration/ingestor.test.ts @@ -111,6 +111,11 @@ jest.mock('@snapshot-labs/pineapple', () => { }; }); +jest.mock('../../src/helpers/strategiesValue', () => ({ + __esModule: true, + default: jest.fn(() => Promise.resolve([])) +})); + const proposalRequest = { headers: { 'x-real-ip': '1.1.1.1' }, body: proposalInput diff --git a/test/integration/writer/proposal.test.ts b/test/integration/writer/proposal.test.ts index 68141e64..5a91187e 100644 --- a/test/integration/writer/proposal.test.ts +++ b/test/integration/writer/proposal.test.ts @@ -19,6 +19,11 @@ jest.mock('../../../src/helpers/moderation', () => { }; }); +jest.mock('../../../src/helpers/strategiesValue', () => ({ + __esModule: true, + default: jest.fn(() => Promise.resolve([])) +})); + const getSpaceMock = jest.spyOn(actionHelper, 'getSpace'); getSpaceMock.mockResolvedValue(spacesGetSpaceFixtures); diff --git a/test/unit/writer/proposal.test.ts b/test/unit/writer/proposal.test.ts index 90a87af5..8228e569 100644 --- a/test/unit/writer/proposal.test.ts +++ b/test/unit/writer/proposal.test.ts @@ -93,6 +93,11 @@ jest.mock('../../../src/helpers/moderation', () => { }; }); +jest.mock('../../../src/helpers/strategiesValue', () => ({ + __esModule: true, + default: jest.fn(() => Promise.resolve([])) +})); + const mockGetProposalsCount = jest.spyOn(writer, 'getProposalsCount'); mockGetProposalsCount.mockResolvedValue([ { @@ -155,7 +160,7 @@ describe('writer/proposal', () => { msg.payload.type = 'basic'; msg.payload.choices = ['For', 'Against', 'Abstain']; - await expect(writer.verify({ ...input, msg: JSON.stringify(msg) })).resolves.toBeUndefined(); + await expect(writer.verify({ ...input, msg: JSON.stringify(msg) })).resolves.toBeDefined(); expect(mockGetSpace).toHaveBeenCalledTimes(1); expect(mockGetProposalsCount).toHaveBeenCalledTimes(1); }); @@ -193,7 +198,7 @@ describe('writer/proposal', () => { voteValidation: { name: 'gitcoin' } }); - await expect(writer.verify(input)).resolves.toBeUndefined(); + await expect(writer.verify(input)).resolves.toBeDefined(); expect(mockGetSpace).toHaveBeenCalledTimes(1); expect(mockGetProposalsCount).toHaveBeenCalledTimes(1); }); @@ -223,7 +228,7 @@ describe('writer/proposal', () => { voting: { ...spacesGetSpaceFixtures.voting, period: VOTING_PERIOD } }); - await expect(writer.verify(inputWithVotingPeriod)).resolves.toBeUndefined(); + await expect(writer.verify(inputWithVotingPeriod)).resolves.toBeDefined(); expect(mockGetSpace).toHaveBeenCalledTimes(1); expect(mockGetProposalsCount).toHaveBeenCalledTimes(1); }); @@ -255,7 +260,7 @@ describe('writer/proposal', () => { await expect( writer.verify({ ...input, msg: JSON.stringify(msg) }) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); expect(mockGetSpace).toHaveBeenCalledTimes(1); expect(mockGetProposalsCount).toHaveBeenCalledTimes(1); }); @@ -320,7 +325,7 @@ describe('writer/proposal', () => { it('does not validate the space validation', async () => { expect.assertions(2); - await expect(writer.verify(input)).resolves.toBeUndefined(); + await expect(writer.verify(input)).resolves.toBeDefined(); expect(mockSnapshotUtilsValidate).toHaveBeenCalledTimes(0); }); }); @@ -329,7 +334,7 @@ describe('writer/proposal', () => { it('does not validate the space validation', async () => { expect.assertions(2); - await expect(writer.verify(input)).resolves.toBeUndefined(); + await expect(writer.verify(input)).resolves.toBeDefined(); expect(mockSnapshotUtilsValidate).toHaveBeenCalledTimes(0); }); }); @@ -345,19 +350,19 @@ describe('writer/proposal', () => { it('accepts a proposal with shutter privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: 'shutter' })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); it('accepts a proposal with undefined privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: undefined })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); it('accepts a proposal with no privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: '' })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); }); @@ -373,19 +378,19 @@ describe('writer/proposal', () => { it('accepts a proposal with shutter privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: 'shutter' })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); it('accepts a proposal with undefined privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: undefined })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); it('accepts a proposal with no privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: '' })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); }); @@ -406,13 +411,13 @@ describe('writer/proposal', () => { it('accepts a proposal with undefined privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: undefined })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); it('accepts a proposal with no privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: '' })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); }); @@ -427,13 +432,13 @@ describe('writer/proposal', () => { it('accepts a proposal with shutter privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: 'shutter' })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); it('accepts a proposal with undefined privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: undefined })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); it('rejects a proposal with privacy empty string', async () => { @@ -605,7 +610,7 @@ describe('writer/proposal', () => { turbo: '1' }); - await expect(writer.verify(input)).resolves.toBeUndefined(); + await expect(writer.verify(input)).resolves.toBeDefined(); expect(mockGetSpace).toHaveBeenCalledTimes(1); expect(mockGetProposalsCount).toHaveBeenCalledTimes(1); }); @@ -678,7 +683,7 @@ describe('writer/proposal', () => { it('verifies a valid input', async () => { expect.assertions(1); - await expect(writer.verify(input)).resolves.toBeUndefined(); + await expect(writer.verify(input)).resolves.toBeDefined(); }); }); }); From 8b8875e6028eae785c814fe917afe223b574bd80 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 6 Aug 2025 22:02:41 +0400 Subject: [PATCH 10/98] refactor: use entityValue as value logic handler --- src/helpers/{strategiesValue.ts => entityValue.ts} | 5 +++-- src/writer/proposal.ts | 7 +------ test/integration/ingestor.test.ts | 4 ++-- test/integration/writer/proposal.test.ts | 4 ++-- test/unit/writer/proposal.test.ts | 4 ++-- 5 files changed, 10 insertions(+), 14 deletions(-) rename src/helpers/{strategiesValue.ts => entityValue.ts} (84%) diff --git a/src/helpers/strategiesValue.ts b/src/helpers/entityValue.ts similarity index 84% rename from src/helpers/strategiesValue.ts rename to src/helpers/entityValue.ts index 0a8123b0..dda03547 100644 --- a/src/helpers/strategiesValue.ts +++ b/src/helpers/entityValue.ts @@ -8,8 +8,9 @@ type Proposal = { }; const OVERLORD_URL = 'https://overlord.snapshot.org'; +const STRATEGIES_VALUE_PRECISION = 9; -export default async function getStrategiesValue(proposal: Proposal): Promise { +export async function getStrategiesValue(proposal: Proposal): Promise { const init = { method: 'POST', headers: { @@ -39,5 +40,5 @@ export default async function getStrategiesValue(proposal: Proposal): Promise parseFloat(value.toFixed(STRATEGIES_VALUE_PRECISION))); } diff --git a/src/writer/proposal.ts b/src/writer/proposal.ts index e7cac8a0..6622c368 100644 --- a/src/writer/proposal.ts +++ b/src/writer/proposal.ts @@ -4,18 +4,17 @@ import networks from '@snapshot-labs/snapshot.js/src/networks.json'; import { uniq } from 'lodash'; import { validateSpaceSettings } from './settings'; import { getPremiumNetworkIds, getSpace } from '../helpers/actions'; +import { getStrategiesValue } from '../helpers/entityValue'; import log from '../helpers/log'; import { containsFlaggedLinks, flaggedAddresses } from '../helpers/moderation'; import { isMalicious } from '../helpers/monitoring'; import db from '../helpers/mysql'; import { getLimits, getSpaceType } from '../helpers/options'; -import getStrategiesValue from '../helpers/strategiesValue'; import { captureError, getQuorum, jsonParse, validateChoices } from '../helpers/utils'; const scoreAPIUrl = process.env.SCORE_API_URL || 'https://score.snapshot.org'; const broviderUrl = process.env.BROVIDER_URL || 'https://rpc.snapshot.org'; const LAST_CB = parseInt(process.env.LAST_CB ?? '1'); -const STRATEGIES_VALUE_PRECISION = 9; // Precision for strategies value export const getProposalsCount = async (space, author) => { const query = ` @@ -253,10 +252,6 @@ export async function verify(body): Promise { start: msg.payload.start, strategies: space.strategies }); - - strategiesValue = strategiesValue.map(value => - parseFloat(value.toFixed(STRATEGIES_VALUE_PRECISION)) - ); } catch (e: any) { console.log('unable to get strategies value', e.message); return Promise.reject('failed to get strategies value'); diff --git a/test/integration/ingestor.test.ts b/test/integration/ingestor.test.ts index ff4d5a1b..f9edb56a 100644 --- a/test/integration/ingestor.test.ts +++ b/test/integration/ingestor.test.ts @@ -111,9 +111,9 @@ jest.mock('@snapshot-labs/pineapple', () => { }; }); -jest.mock('../../src/helpers/strategiesValue', () => ({ +jest.mock('../../src/helpers/entityValue', () => ({ __esModule: true, - default: jest.fn(() => Promise.resolve([])) + getStrategiesValue: jest.fn(() => Promise.resolve([])) })); const proposalRequest = { diff --git a/test/integration/writer/proposal.test.ts b/test/integration/writer/proposal.test.ts index 5a91187e..171d322f 100644 --- a/test/integration/writer/proposal.test.ts +++ b/test/integration/writer/proposal.test.ts @@ -19,9 +19,9 @@ jest.mock('../../../src/helpers/moderation', () => { }; }); -jest.mock('../../../src/helpers/strategiesValue', () => ({ +jest.mock('../../../src/helpers/entityValue', () => ({ __esModule: true, - default: jest.fn(() => Promise.resolve([])) + getStrategiesValue: jest.fn(() => Promise.resolve([])) })); const getSpaceMock = jest.spyOn(actionHelper, 'getSpace'); diff --git a/test/unit/writer/proposal.test.ts b/test/unit/writer/proposal.test.ts index 8228e569..3e99e46d 100644 --- a/test/unit/writer/proposal.test.ts +++ b/test/unit/writer/proposal.test.ts @@ -93,9 +93,9 @@ jest.mock('../../../src/helpers/moderation', () => { }; }); -jest.mock('../../../src/helpers/strategiesValue', () => ({ +jest.mock('../../../src/helpers/entityValue', () => ({ __esModule: true, - default: jest.fn(() => Promise.resolve([])) + getStrategiesValue: jest.fn(() => Promise.resolve([])) })); const mockGetProposalsCount = jest.spyOn(writer, 'getProposalsCount'); From afed3e33eb1db05b37f9b0a98b570bf2f9d9ce2b Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 6 Aug 2025 22:15:57 +0400 Subject: [PATCH 11/98] fix: use randomUUID to generate random number --- package.json | 2 +- src/helpers/entityValue.ts | 3 ++- yarn.lock | 8 ++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index ecf6cd20..451eb213 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "@snapshot-labs/eslint-config": "^0.1.0-beta.18", "@snapshot-labs/prettier-config": "^0.1.0-beta.7", "@types/jest": "^29.5.2", - "@types/node": "^14.0.13", + "@types/node": "^16.0.0", "dotenv-cli": "^7.2.1", "eslint": "^8.28.0", "jest": "^29.6.1", diff --git a/src/helpers/entityValue.ts b/src/helpers/entityValue.ts index dda03547..9524cc50 100644 --- a/src/helpers/entityValue.ts +++ b/src/helpers/entityValue.ts @@ -1,3 +1,4 @@ +import { randomUUID } from 'crypto'; import { capture } from '@snapshot-labs/snapshot-sentry'; import { fetchWithKeepAlive } from './utils'; @@ -25,7 +26,7 @@ export async function getStrategiesValue(proposal: Proposal): Promise strategies: proposal.strategies, snapshot: proposal.start }, - id: Math.random().toString(36).substring(7) + id: randomUUID() }) }; const res = await fetchWithKeepAlive(OVERLORD_URL, init); diff --git a/yarn.lock b/yarn.lock index 40804aa9..25e06b6b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1450,10 +1450,10 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-20.4.0.tgz#01d637d1891e419bc85763b46f42809cd2d5addb" integrity sha512-jfT7iTf/4kOQ9S7CHV9BIyRaQqHu67mOjsIQBC3BKZvzvUB6zLxEwJ6sBE3ozcvP8kF6Uk5PXN0Q+c0dfhGX0g== -"@types/node@^14.0.13": - version "14.14.44" - resolved "https://registry.yarnpkg.com/@types/node/-/node-14.14.44.tgz#df7503e6002847b834371c004b372529f3f85215" - integrity sha512-+gaugz6Oce6ZInfI/tK4Pq5wIIkJMEJUu92RB3Eu93mtj4wjjjz9EB5mLp5s1pSsLXdC/CPut/xF20ZzAQJbTA== +"@types/node@^16.0.0": + version "16.18.126" + resolved "https://registry.yarnpkg.com/@types/node/-/node-16.18.126.tgz#27875faa2926c0f475b39a8bb1e546c0176f8d4b" + integrity sha512-OTcgaiwfGFBKacvfwuHzzn1KLxH/er8mluiy8/uM3sGXHaRe73RrSIj01jow9t4kJEW633Ov+cOexXeiApTyAw== "@types/prettier@^2.1.5": version "2.7.3" From a2e025c12b28e90dc0ebb30fc0995c3b20968e1b Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 6 Aug 2025 22:17:50 +0400 Subject: [PATCH 12/98] chore: use consistent logging --- src/writer/proposal.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/writer/proposal.ts b/src/writer/proposal.ts index 6622c368..2a730a09 100644 --- a/src/writer/proposal.ts +++ b/src/writer/proposal.ts @@ -253,7 +253,7 @@ export async function verify(body): Promise { strategies: space.strategies }); } catch (e: any) { - console.log('unable to get strategies value', e.message); + log.warn('unable to get strategies value', e.message); return Promise.reject('failed to get strategies value'); } @@ -285,7 +285,7 @@ export async function action(body, ipfs, receipt, id, context): Promise { try { quorum = await getQuorum(spaceSettings.plugins.quorum, spaceNetwork, proposalSnapshot); } catch (e: any) { - console.log('unable to get quorum', e.message); + log.warn('unable to get quorum', e.message); return Promise.reject('unable to get quorum'); } } From ec1028e2a2c869663ff410889ffb05f368f78db2 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 6 Aug 2025 23:22:50 +0400 Subject: [PATCH 13/98] fix: handle overlord error --- src/helpers/entityValue.ts | 68 ++++++++++++++++++++++++++++++++------ 1 file changed, 57 insertions(+), 11 deletions(-) diff --git a/src/helpers/entityValue.ts b/src/helpers/entityValue.ts index 9524cc50..41e3a158 100644 --- a/src/helpers/entityValue.ts +++ b/src/helpers/entityValue.ts @@ -29,17 +29,63 @@ export async function getStrategiesValue(proposal: Proposal): Promise id: randomUUID() }) }; - const res = await fetchWithKeepAlive(OVERLORD_URL, init); - const { result } = await res.json(); - - // Handle unlikely case where strategies value array length does not match strategies length - if (result.length !== proposal.strategies.length) { - capture(new Error('Strategies value length mismatch'), { - strategiesLength: proposal.strategies.length, - result: JSON.stringify(result) + + try { + const res = await fetchWithKeepAlive(OVERLORD_URL, init); + + // Check HTTP status + if (!res.ok) { + capture(new Error('HTTP error response'), { + status: res.status, + statusText: res.statusText, + url: OVERLORD_URL + }); + return Promise.reject(`HTTP error: ${res.status} ${res.statusText}`); + } + + const response = await res.json(); + + // Handle JSON-RPC error response + if (response.error) { + capture(new Error('JSON-RPC error response'), { + error: response.error, + request: { + network: proposal.network, + strategiesLength: proposal.strategies.length, + snapshot: proposal.start + } + }); + return Promise.reject( + `JSON-RPC error: ${response.error.message || response.error.code || 'Unknown error'}` + ); + } + + const { result } = response; + + // Handle unlikely case where strategies value array length does not match strategies length + if (result.length !== proposal.strategies.length) { + capture(new Error('Strategies value length mismatch'), { + strategiesLength: proposal.strategies.length, + result: JSON.stringify(result) + }); + return Promise.reject('failed to get strategies value'); + } + + return result.map(value => parseFloat(value.toFixed(STRATEGIES_VALUE_PRECISION))); + } catch (error) { + capture(new Error('Network or parsing error'), { + error: error instanceof Error ? error.message : String(error), + url: OVERLORD_URL, + request: { + network: proposal.network, + strategiesLength: proposal.strategies.length, + snapshot: proposal.start + } }); - return Promise.reject('failed to get strategies value'); + return Promise.reject( + `Failed to fetch strategies value: ${ + error instanceof Error ? error.message : 'Unknown error' + }` + ); } - - return result.map(value => parseFloat(value.toFixed(STRATEGIES_VALUE_PRECISION))); } From a64a0e6d239c08bc3714a25cf1e0c14993fae0e1 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 6 Aug 2025 23:27:26 +0400 Subject: [PATCH 14/98] fix: more precise rounding --- src/helpers/entityValue.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/helpers/entityValue.ts b/src/helpers/entityValue.ts index 41e3a158..283765b9 100644 --- a/src/helpers/entityValue.ts +++ b/src/helpers/entityValue.ts @@ -10,6 +10,7 @@ type Proposal = { const OVERLORD_URL = 'https://overlord.snapshot.org'; const STRATEGIES_VALUE_PRECISION = 9; +const PRECISION_MULTIPLIER = Math.pow(10, STRATEGIES_VALUE_PRECISION); export async function getStrategiesValue(proposal: Proposal): Promise { const init = { @@ -71,7 +72,7 @@ export async function getStrategiesValue(proposal: Proposal): Promise return Promise.reject('failed to get strategies value'); } - return result.map(value => parseFloat(value.toFixed(STRATEGIES_VALUE_PRECISION))); + return result.map(value => Math.round(value * PRECISION_MULTIPLIER) / PRECISION_MULTIPLIER); } catch (error) { capture(new Error('Network or parsing error'), { error: error instanceof Error ? error.message : String(error), From adc6257aba58c6cd29d52a2b11264d58af03a63c Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Thu, 7 Aug 2025 19:43:30 +0400 Subject: [PATCH 15/98] refactor: DRY json rpc request fetcher --- src/helpers/entityValue.ts | 91 ++++++-------------------------------- src/helpers/shutter.ts | 24 +--------- src/helpers/utils.ts | 42 +++++++++++++++++- 3 files changed, 56 insertions(+), 101 deletions(-) diff --git a/src/helpers/entityValue.ts b/src/helpers/entityValue.ts index 283765b9..6c07125b 100644 --- a/src/helpers/entityValue.ts +++ b/src/helpers/entityValue.ts @@ -1,6 +1,4 @@ -import { randomUUID } from 'crypto'; -import { capture } from '@snapshot-labs/snapshot-sentry'; -import { fetchWithKeepAlive } from './utils'; +import { jsonRpcRequest } from './utils'; type Proposal = { network: string; @@ -9,84 +7,21 @@ type Proposal = { }; const OVERLORD_URL = 'https://overlord.snapshot.org'; +// Round strategy values to 9 decimal places const STRATEGIES_VALUE_PRECISION = 9; const PRECISION_MULTIPLIER = Math.pow(10, STRATEGIES_VALUE_PRECISION); export async function getStrategiesValue(proposal: Proposal): Promise { - const init = { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json' - }, - body: JSON.stringify({ - jsonrpc: '2.0', - method: 'get_vp_value_by_strategy', - params: { - network: proposal.network, - strategies: proposal.strategies, - snapshot: proposal.start - }, - id: randomUUID() - }) - }; - - try { - const res = await fetchWithKeepAlive(OVERLORD_URL, init); - - // Check HTTP status - if (!res.ok) { - capture(new Error('HTTP error response'), { - status: res.status, - statusText: res.statusText, - url: OVERLORD_URL - }); - return Promise.reject(`HTTP error: ${res.status} ${res.statusText}`); - } - - const response = await res.json(); - - // Handle JSON-RPC error response - if (response.error) { - capture(new Error('JSON-RPC error response'), { - error: response.error, - request: { - network: proposal.network, - strategiesLength: proposal.strategies.length, - snapshot: proposal.start - } - }); - return Promise.reject( - `JSON-RPC error: ${response.error.message || response.error.code || 'Unknown error'}` - ); - } - - const { result } = response; - - // Handle unlikely case where strategies value array length does not match strategies length - if (result.length !== proposal.strategies.length) { - capture(new Error('Strategies value length mismatch'), { - strategiesLength: proposal.strategies.length, - result: JSON.stringify(result) - }); - return Promise.reject('failed to get strategies value'); - } - - return result.map(value => Math.round(value * PRECISION_MULTIPLIER) / PRECISION_MULTIPLIER); - } catch (error) { - capture(new Error('Network or parsing error'), { - error: error instanceof Error ? error.message : String(error), - url: OVERLORD_URL, - request: { - network: proposal.network, - strategiesLength: proposal.strategies.length, - snapshot: proposal.start - } - }); - return Promise.reject( - `Failed to fetch strategies value: ${ - error instanceof Error ? error.message : 'Unknown error' - }` - ); + const result: number[] = await jsonRpcRequest(OVERLORD_URL, 'get_vp_value_by_strategy', { + network: proposal.network, + strategies: proposal.strategies, + snapshot: proposal.start + }); + + // Handle unlikely case where strategies value array length does not match strategies length + if (result.length !== proposal.strategies.length) { + throw new Error('Strategies value length mismatch'); } + + return result.map(value => Math.round(value * PRECISION_MULTIPLIER) / PRECISION_MULTIPLIER); } diff --git a/src/helpers/shutter.ts b/src/helpers/shutter.ts index 4b062402..85c7ea98 100644 --- a/src/helpers/shutter.ts +++ b/src/helpers/shutter.ts @@ -1,4 +1,3 @@ -import { randomBytes } from 'crypto'; import { arrayify } from '@ethersproject/bytes'; import { toUtf8String } from '@ethersproject/strings'; import { decrypt, init } from '@shutter-network/shutter-crypto'; @@ -6,7 +5,7 @@ import { capture } from '@snapshot-labs/snapshot-sentry'; import express from 'express'; import log from './log'; import db from './mysql'; -import { fetchWithKeepAlive, getIp, jsonParse, rpcError, rpcSuccess } from './utils'; +import { getIp, jsonParse, jsonRpcRequest, rpcError, rpcSuccess } from './utils'; import { updateProposalAndVotes } from '../scores'; init().then(() => log.info('[shutter] init')); @@ -36,28 +35,9 @@ function idToProposal(id: string): string { return `0x${id}`; } -export async function rpcRequest(method, params, url: string = SHUTTER_URL) { - const init = { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json' - }, - body: JSON.stringify({ - jsonrpc: '2.0', - method, - params, - id: randomBytes(6).toString('hex') - }) - }; - const res = await fetchWithKeepAlive(url, init); - const { result } = await res.json(); - return result; -} - export async function getDecryptionKey(proposal: string, url: string = SHUTTER_URL) { const id = proposalToId(proposal); - const result = await rpcRequest('get_decryption_key', ['1', id], url); + const result = await jsonRpcRequest(url, 'get_decryption_key', ['1', id]); log.info(`[shutter] get_decryption_key ${proposal} ${JSON.stringify(result)}`); return result; diff --git a/src/helpers/utils.ts b/src/helpers/utils.ts index 3ce0cfea..b0cb80dd 100644 --- a/src/helpers/utils.ts +++ b/src/helpers/utils.ts @@ -1,4 +1,4 @@ -import { createHash } from 'crypto'; +import { createHash, randomUUID } from 'crypto'; import http from 'http'; import https from 'https'; import { URL } from 'url'; @@ -72,6 +72,46 @@ export function rpcError(res, code, e, id) { }); } +export async function jsonRpcRequest(url: string, method: string, params: any): Promise { + const init = { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + jsonrpc: '2.0', + method, + params, + id: randomUUID() + }) + }; + + try { + const res = await fetchWithKeepAlive(url, init); + + if (!res.ok) { + throw new Error(`HTTP error: ${res.status} ${res.statusText}`); + } + + const response = await res.json(); + + if (response.error) { + throw new Error( + `JSON-RPC error: ${response.error.message || response.error.code || 'Unknown error'}` + ); + } + + return response.result; + } catch (error) { + capture(error, { + url, + request: { method, params } + }); + throw error; + } +} + export function hasStrategyOverride(strategies: any[]) { const keywords = [ '"aura-vlaura-vebal-with-overrides"', From 602555e67f7130741d4782129a4abd8216f102d5 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Thu, 7 Aug 2025 20:06:29 +0400 Subject: [PATCH 16/98] refactor: simplify rounding --- src/helpers/entityValue.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/helpers/entityValue.ts b/src/helpers/entityValue.ts index 6c07125b..5c6fd80d 100644 --- a/src/helpers/entityValue.ts +++ b/src/helpers/entityValue.ts @@ -9,7 +9,6 @@ type Proposal = { const OVERLORD_URL = 'https://overlord.snapshot.org'; // Round strategy values to 9 decimal places const STRATEGIES_VALUE_PRECISION = 9; -const PRECISION_MULTIPLIER = Math.pow(10, STRATEGIES_VALUE_PRECISION); export async function getStrategiesValue(proposal: Proposal): Promise { const result: number[] = await jsonRpcRequest(OVERLORD_URL, 'get_vp_value_by_strategy', { @@ -23,5 +22,5 @@ export async function getStrategiesValue(proposal: Proposal): Promise throw new Error('Strategies value length mismatch'); } - return result.map(value => Math.round(value * PRECISION_MULTIPLIER) / PRECISION_MULTIPLIER); + return result.map(value => parseFloat(value.toFixed(STRATEGIES_VALUE_PRECISION))); } From 3e52e3acec73499eb7747d119fcd19ebb2e58955 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Thu, 7 Aug 2025 20:28:03 +0400 Subject: [PATCH 17/98] fix: allow customize overlord url via env var --- .env.example | 1 + src/helpers/entityValue.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 22ff6a63..1bd3b482 100644 --- a/.env.example +++ b/.env.example @@ -19,4 +19,5 @@ STARKNET_RPC_URL= # optional AUTH_SECRET=1dfd84a695705665668c260222ded178d1f1d62d251d7bee8148428dac6d0487 # optional WALLETCONNECT_PROJECT_ID=e6454bd61aba40b786e866a69bd4c5c6 REOWN_SECRET= # optional +OVERLORD_URL=https://overlord.snapshot.org LAST_CB=1 diff --git a/src/helpers/entityValue.ts b/src/helpers/entityValue.ts index 5c6fd80d..66478445 100644 --- a/src/helpers/entityValue.ts +++ b/src/helpers/entityValue.ts @@ -6,7 +6,7 @@ type Proposal = { start: number; }; -const OVERLORD_URL = 'https://overlord.snapshot.org'; +const OVERLORD_URL = process.env.OVERLORD_URL ?? 'https://overlord.snapshot.org'; // Round strategy values to 9 decimal places const STRATEGIES_VALUE_PRECISION = 9; From 6589f97d52fe1cb7b93daefc961be781670834e0 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Thu, 7 Aug 2025 20:32:06 +0400 Subject: [PATCH 18/98] chore: remove white spaces --- .env.example | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 1bd3b482..7a9ab964 100644 --- a/.env.example +++ b/.env.example @@ -11,10 +11,10 @@ PINEAPPLE_URL=https://.../upload # optional SCORE_API_URL=https://score.snapshot.org SCHNAPS_API_URL=https://schnaps.snapshot.box # If you need unlimited access to score-api, use `https://score.snapshot.org?apiKey=...` -RATE_LIMIT_DATABASE_URL= # optional +RATE_LIMIT_DATABASE_URL= # optional RATE_LIMIT_KEYS_PREFIX=snapshot-sequencer: # optional -BROVIDER_URL=https://rpc.snapshot.org # optional -STARKNET_RPC_URL= # optional +BROVIDER_URL=https://rpc.snapshot.org # optional +STARKNET_RPC_URL= # optional # Secret for laser AUTH_SECRET=1dfd84a695705665668c260222ded178d1f1d62d251d7bee8148428dac6d0487 # optional WALLETCONNECT_PROJECT_ID=e6454bd61aba40b786e866a69bd4c5c6 From 585622a5bee18db236c56f24779167524951d27c Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 16 Sep 2025 14:48:52 +0400 Subject: [PATCH 19/98] fix: fix types --- src/helpers/entityValue.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/helpers/entityValue.ts b/src/helpers/entityValue.ts index 32e59519..8ec2459c 100644 --- a/src/helpers/entityValue.ts +++ b/src/helpers/entityValue.ts @@ -8,7 +8,6 @@ type Proposal = { network: string; strategies: any[]; start: number; - vp_value_by_strategy: number[]; }; const OVERLORD_URL = process.env.OVERLORD_URL ?? 'https://overlord.snapshot.org'; @@ -34,7 +33,7 @@ export async function getStrategiesValue(proposal: Proposal): Promise * Calculates the total vote value based on the voting power and the proposal's value per strategy. * @returns The total vote value, in the currency unit specified by the proposal's vp_value_by_strategy values **/ -export function getVoteValue(proposal: Proposal, vote: Vote): number { +export function getVoteValue(proposal: { vp_value_by_strategy: number[] }, vote: Vote): number { if (!proposal.vp_value_by_strategy.length) return 0; if (proposal.vp_value_by_strategy.length !== vote.vp_by_strategy.length) { From 69d4ff834c99c79b9ef1117864db79b5b636c836 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 16 Sep 2025 14:54:31 +0400 Subject: [PATCH 20/98] chore: lint fix --- .env.example | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 1bd3b482..7a9ab964 100644 --- a/.env.example +++ b/.env.example @@ -11,10 +11,10 @@ PINEAPPLE_URL=https://.../upload # optional SCORE_API_URL=https://score.snapshot.org SCHNAPS_API_URL=https://schnaps.snapshot.box # If you need unlimited access to score-api, use `https://score.snapshot.org?apiKey=...` -RATE_LIMIT_DATABASE_URL= # optional +RATE_LIMIT_DATABASE_URL= # optional RATE_LIMIT_KEYS_PREFIX=snapshot-sequencer: # optional -BROVIDER_URL=https://rpc.snapshot.org # optional -STARKNET_RPC_URL= # optional +BROVIDER_URL=https://rpc.snapshot.org # optional +STARKNET_RPC_URL= # optional # Secret for laser AUTH_SECRET=1dfd84a695705665668c260222ded178d1f1d62d251d7bee8148428dac6d0487 # optional WALLETCONNECT_PROJECT_ID=e6454bd61aba40b786e866a69bd4c5c6 From e80d0c47e3d09f5ea5518c3fa9f81bfaca7ffac2 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 16 Sep 2025 15:16:21 +0400 Subject: [PATCH 21/98] fix: update overlord url --- .env.example | 2 +- src/helpers/entityValue.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 7a9ab964..04fbdf4c 100644 --- a/.env.example +++ b/.env.example @@ -19,5 +19,5 @@ STARKNET_RPC_URL= # optional AUTH_SECRET=1dfd84a695705665668c260222ded178d1f1d62d251d7bee8148428dac6d0487 # optional WALLETCONNECT_PROJECT_ID=e6454bd61aba40b786e866a69bd4c5c6 REOWN_SECRET= # optional -OVERLORD_URL=https://overlord.snapshot.org +OVERLORD_URL=https://overlord.snapshot.box LAST_CB=1 diff --git a/src/helpers/entityValue.ts b/src/helpers/entityValue.ts index 8ec2459c..4a3e67dc 100644 --- a/src/helpers/entityValue.ts +++ b/src/helpers/entityValue.ts @@ -10,7 +10,7 @@ type Proposal = { start: number; }; -const OVERLORD_URL = process.env.OVERLORD_URL ?? 'https://overlord.snapshot.org'; +const OVERLORD_URL = process.env.OVERLORD_URL ?? 'https://overlord.snapshot.box'; // Round strategy values to 9 decimal places const STRATEGIES_VALUE_PRECISION = 9; From 83b9d1baedaec78d5484df5dfbea49a9537e6e5a Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 16 Sep 2025 15:27:54 +0400 Subject: [PATCH 22/98] refactor: more consistent function name --- src/helpers/entityValue.ts | 2 +- src/writer/proposal.ts | 4 ++-- test/integration/ingestor.test.ts | 2 +- test/integration/writer/proposal.test.ts | 2 +- test/unit/writer/proposal.test.ts | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/helpers/entityValue.ts b/src/helpers/entityValue.ts index 4a3e67dc..e052e484 100644 --- a/src/helpers/entityValue.ts +++ b/src/helpers/entityValue.ts @@ -14,7 +14,7 @@ const OVERLORD_URL = process.env.OVERLORD_URL ?? 'https://overlord.snapshot.box' // Round strategy values to 9 decimal places const STRATEGIES_VALUE_PRECISION = 9; -export async function getStrategiesValue(proposal: Proposal): Promise { +export async function getVpValueByStrategy(proposal: Proposal): Promise { const result: number[] = await jsonRpcRequest(OVERLORD_URL, 'get_vp_value_by_strategy', { network: proposal.network, strategies: proposal.strategies, diff --git a/src/writer/proposal.ts b/src/writer/proposal.ts index f086fd32..5a999cbe 100644 --- a/src/writer/proposal.ts +++ b/src/writer/proposal.ts @@ -3,7 +3,7 @@ import snapshot from '@snapshot-labs/snapshot.js'; import networks from '@snapshot-labs/snapshot.js/src/networks.json'; import { uniq } from 'lodash'; import { getPremiumNetworkIds, getSpace } from '../helpers/actions'; -import { getStrategiesValue } from '../helpers/entityValue'; +import { getVpValueByStrategy } from '../helpers/entityValue'; import log from '../helpers/log'; import { containsFlaggedLinks, flaggedAddresses } from '../helpers/moderation'; import { isMalicious } from '../helpers/monitoring'; @@ -243,7 +243,7 @@ export async function verify(body): Promise { let strategiesValue: number[] = []; try { - strategiesValue = await getStrategiesValue({ + strategiesValue = await getVpValueByStrategy({ network: space.network, start: msg.payload.start, strategies: space.strategies diff --git a/test/integration/ingestor.test.ts b/test/integration/ingestor.test.ts index f0df86ce..9900aa69 100644 --- a/test/integration/ingestor.test.ts +++ b/test/integration/ingestor.test.ts @@ -114,7 +114,7 @@ jest.mock('@snapshot-labs/pineapple', () => { jest.mock('../../src/helpers/entityValue', () => ({ __esModule: true, - getStrategiesValue: jest.fn(() => Promise.resolve([])) + getVpValueByStrategy: jest.fn(() => Promise.resolve([])) })); jest.mock('../../src/helpers/strategies', () => ({ getStrategies: jest.fn(() => ({ diff --git a/test/integration/writer/proposal.test.ts b/test/integration/writer/proposal.test.ts index 171d322f..5a7074bd 100644 --- a/test/integration/writer/proposal.test.ts +++ b/test/integration/writer/proposal.test.ts @@ -21,7 +21,7 @@ jest.mock('../../../src/helpers/moderation', () => { jest.mock('../../../src/helpers/entityValue', () => ({ __esModule: true, - getStrategiesValue: jest.fn(() => Promise.resolve([])) + getVpValueByStrategy: jest.fn(() => Promise.resolve([])) })); const getSpaceMock = jest.spyOn(actionHelper, 'getSpace'); diff --git a/test/unit/writer/proposal.test.ts b/test/unit/writer/proposal.test.ts index d57a52fd..9e6c6700 100644 --- a/test/unit/writer/proposal.test.ts +++ b/test/unit/writer/proposal.test.ts @@ -100,7 +100,7 @@ jest.mock('../../../src/helpers/moderation', () => { jest.mock('../../../src/helpers/entityValue', () => ({ __esModule: true, - getStrategiesValue: jest.fn(() => Promise.resolve([])) + getVpValueByStrategy: jest.fn(() => Promise.resolve([])) })); // Get the mocked function after the mock is created const { validateSpaceSettings: mockValidateSpaceSettings } = jest.requireMock( From c0761e1f8afc39a13b439350f5978ba3e0b0680c Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 16 Sep 2025 15:39:14 +0400 Subject: [PATCH 23/98] fix: fix invalid method name --- src/helpers/entityValue.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/helpers/entityValue.ts b/src/helpers/entityValue.ts index e052e484..aba1d996 100644 --- a/src/helpers/entityValue.ts +++ b/src/helpers/entityValue.ts @@ -15,7 +15,7 @@ const OVERLORD_URL = process.env.OVERLORD_URL ?? 'https://overlord.snapshot.box' const STRATEGIES_VALUE_PRECISION = 9; export async function getVpValueByStrategy(proposal: Proposal): Promise { - const result: number[] = await jsonRpcRequest(OVERLORD_URL, 'get_vp_value_by_strategy', { + const result: number[] = await jsonRpcRequest(OVERLORD_URL, 'get_value_by_strategy', { network: proposal.network, strategies: proposal.strategies, snapshot: proposal.start From 5ff71c4c4a40b292b9790f5a5a52f4c7cdbeff69 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 16 Sep 2025 15:44:04 +0400 Subject: [PATCH 24/98] fix: send JSON-RPC id as number --- src/helpers/utils.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/helpers/utils.ts b/src/helpers/utils.ts index bf246eeb..5720d097 100644 --- a/src/helpers/utils.ts +++ b/src/helpers/utils.ts @@ -1,4 +1,4 @@ -import { createHash, randomUUID } from 'crypto'; +import { createHash, randomInt } from 'crypto'; import http from 'http'; import https from 'https'; import { URL } from 'url'; @@ -73,7 +73,7 @@ export async function jsonRpcRequest(url: string, method: string, params: any): jsonrpc: '2.0', method, params, - id: randomUUID() + id: randomInt(10000) }) }; From 5b4ffd08eddc904aed26d5e6f9d72c42dbf33c80 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 16 Sep 2025 16:42:53 +0400 Subject: [PATCH 25/98] fix: skip vp value fetching for future proposals --- src/helpers/entityValue.ts | 2 +- src/writer/proposal.ts | 23 +++++++++++++---------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/helpers/entityValue.ts b/src/helpers/entityValue.ts index aba1d996..765ce1eb 100644 --- a/src/helpers/entityValue.ts +++ b/src/helpers/entityValue.ts @@ -18,7 +18,7 @@ export async function getVpValueByStrategy(proposal: Proposal): Promise { return Promise.reject('wrong proposal format'); } - const tsInt = (Date.now() / 1e3).toFixed(); + const tsInt = +(Date.now() / 1e3).toFixed(); if (msg.payload.end <= tsInt) { return Promise.reject('proposal end date must be in the future'); } @@ -242,15 +242,18 @@ export async function verify(body): Promise { let strategiesValue: number[] = []; - try { - strategiesValue = await getVpValueByStrategy({ - network: space.network, - start: msg.payload.start, - strategies: space.strategies - }); - } catch (e: any) { - log.warn('unable to get strategies value', e.message); - return Promise.reject('failed to get strategies value'); + // Token value are not available yet for future proposals + if (msg.payload.start <= tsInt) { + try { + strategiesValue = await getVpValueByStrategy({ + network: space.network, + start: msg.payload.start, + strategies: space.strategies + }); + } catch (e: any) { + log.warn('unable to get strategies value', e.message); + return Promise.reject('failed to get strategies value'); + } } return { strategiesValue }; From f124efbd62c7a4707ca87652b8318d3d7f815e15 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 17 Sep 2025 18:16:00 +0400 Subject: [PATCH 26/98] fix: remove the getValue setting on proposal creation it should be done async, via a separate infinite loop script --- src/constants.ts | 4 +++ src/writer/proposal.ts | 29 ++++-------------- test/integration/ingestor.test.ts | 4 --- test/integration/writer/proposal.test.ts | 17 ++++------- test/unit/writer/proposal.test.ts | 38 +++++++++++------------- 5 files changed, 33 insertions(+), 59 deletions(-) create mode 100644 src/constants.ts diff --git a/src/constants.ts b/src/constants.ts new file mode 100644 index 00000000..53d57fd5 --- /dev/null +++ b/src/constants.ts @@ -0,0 +1,4 @@ +export const CB = { + UNELIGIBLE: -1, + PENDING_SYNC: -10 +}; diff --git a/src/writer/proposal.ts b/src/writer/proposal.ts index 042719cd..db3a7b2b 100644 --- a/src/writer/proposal.ts +++ b/src/writer/proposal.ts @@ -2,8 +2,8 @@ import { capture } from '@snapshot-labs/snapshot-sentry'; import snapshot from '@snapshot-labs/snapshot.js'; import networks from '@snapshot-labs/snapshot.js/src/networks.json'; import { uniq } from 'lodash'; +import { CB } from '../constants'; import { getPremiumNetworkIds, getSpace } from '../helpers/actions'; -import { getVpValueByStrategy } from '../helpers/entityValue'; import log from '../helpers/log'; import { containsFlaggedLinks, flaggedAddresses } from '../helpers/moderation'; import { isMalicious } from '../helpers/monitoring'; @@ -14,7 +14,6 @@ import { captureError, getQuorum, jsonParse, validateChoices } from '../helpers/ const scoreAPIUrl = process.env.SCORE_API_URL || 'https://score.snapshot.org'; const broviderUrl = process.env.BROVIDER_URL || 'https://rpc.snapshot.org'; -const LAST_CB = parseInt(process.env.LAST_CB ?? '1'); export const getProposalsCount = async (space, author) => { const query = ` @@ -105,7 +104,7 @@ export async function verify(body): Promise { return Promise.reject('wrong proposal format'); } - const tsInt = +(Date.now() / 1e3).toFixed(); + const tsInt = (Date.now() / 1e3).toFixed(); if (msg.payload.end <= tsInt) { return Promise.reject('proposal end date must be in the future'); } @@ -239,27 +238,9 @@ export async function verify(body): Promise { if (msg.payload.choices.length > choicesLimit) { return Promise.reject(`number of choices can not exceed ${choicesLimit}`); } - - let strategiesValue: number[] = []; - - // Token value are not available yet for future proposals - if (msg.payload.start <= tsInt) { - try { - strategiesValue = await getVpValueByStrategy({ - network: space.network, - start: msg.payload.start, - strategies: space.strategies - }); - } catch (e: any) { - log.warn('unable to get strategies value', e.message); - return Promise.reject('failed to get strategies value'); - } - } - - return { strategiesValue }; } -export async function action(body, ipfs, receipt, id, context): Promise { +export async function action(body, ipfs, receipt, id): Promise { const msg = jsonParse(body.msg); const space = msg.space; @@ -318,11 +299,11 @@ export async function action(body, ipfs, receipt, id, context): Promise { scores_total: 0, scores_updated: 0, scores_total_value: 0, - vp_value_by_strategy: JSON.stringify(context.strategiesValue), + vp_value_by_strategy: JSON.stringify([]), votes: 0, validation, flagged: +containsFlaggedLinks(msg.payload.body), - cb: LAST_CB + cb: CB.PENDING_SYNC }; const query = ` diff --git a/test/integration/ingestor.test.ts b/test/integration/ingestor.test.ts index 9900aa69..82fff3c9 100644 --- a/test/integration/ingestor.test.ts +++ b/test/integration/ingestor.test.ts @@ -112,10 +112,6 @@ jest.mock('@snapshot-labs/pineapple', () => { }; }); -jest.mock('../../src/helpers/entityValue', () => ({ - __esModule: true, - getVpValueByStrategy: jest.fn(() => Promise.resolve([])) -})); jest.mock('../../src/helpers/strategies', () => ({ getStrategies: jest.fn(() => ({ 'erc20-balance-of': { id: 'erc20-balance-of', override: false, disabled: false }, diff --git a/test/integration/writer/proposal.test.ts b/test/integration/writer/proposal.test.ts index 5a7074bd..80df065c 100644 --- a/test/integration/writer/proposal.test.ts +++ b/test/integration/writer/proposal.test.ts @@ -1,9 +1,9 @@ -import { action } from '../../../src/writer/proposal'; +import * as actionHelper from '../../../src/helpers/actions'; +import { setData } from '../../../src/helpers/moderation'; import db, { sequencerDB } from '../../../src/helpers/mysql'; -import input from '../../fixtures/writer-payload/proposal.json'; +import { action } from '../../../src/writer/proposal'; import { spacesGetSpaceFixtures } from '../../fixtures/space'; -import { setData } from '../../../src/helpers/moderation'; -import * as actionHelper from '../../../src/helpers/actions'; +import input from '../../fixtures/writer-payload/proposal.json'; const mockContainsFlaggedLinks = jest.fn((): any => { return false; @@ -19,11 +19,6 @@ jest.mock('../../../src/helpers/moderation', () => { }; }); -jest.mock('../../../src/helpers/entityValue', () => ({ - __esModule: true, - getVpValueByStrategy: jest.fn(() => Promise.resolve([])) -})); - const getSpaceMock = jest.spyOn(actionHelper, 'getSpace'); getSpaceMock.mockResolvedValue(spacesGetSpaceFixtures); @@ -50,7 +45,7 @@ describe('writer/proposal', () => { expect.hasAssertions(); mockContainsFlaggedLinks.mockReturnValueOnce(true); const id = '0x01-flagged'; - expect(await action(input, 'ipfs', 'receipt', id, { strategiesValue: [] })).toBeUndefined(); + expect(await action(input, 'ipfs', 'receipt', id)).toBeUndefined(); expect(mockContainsFlaggedLinks).toBeCalledTimes(1); const [proposal] = await db.queryAsync('SELECT * FROM proposals WHERE id = ?', [id]); @@ -62,7 +57,7 @@ describe('writer/proposal', () => { it('creates and does not flag proposal', async () => { expect.hasAssertions(); const id = '0x02-non-flagged'; - expect(await action(input, 'ipfs', 'receipt', id, { strategiesValue: [] })).toBeUndefined(); + expect(await action(input, 'ipfs', 'receipt', id)).toBeUndefined(); expect(mockContainsFlaggedLinks).toBeCalledTimes(1); const [proposal] = await db.queryAsync('SELECT * FROM proposals WHERE id = ?', [id]); diff --git a/test/unit/writer/proposal.test.ts b/test/unit/writer/proposal.test.ts index 9e6c6700..9c98d93e 100644 --- a/test/unit/writer/proposal.test.ts +++ b/test/unit/writer/proposal.test.ts @@ -98,10 +98,6 @@ jest.mock('../../../src/helpers/moderation', () => { }; }); -jest.mock('../../../src/helpers/entityValue', () => ({ - __esModule: true, - getVpValueByStrategy: jest.fn(() => Promise.resolve([])) -})); // Get the mocked function after the mock is created const { validateSpaceSettings: mockValidateSpaceSettings } = jest.requireMock( '../../../src/helpers/spaceValidation' @@ -174,7 +170,7 @@ describe('writer/proposal', () => { msg.payload.type = 'basic'; msg.payload.choices = ['For', 'Against', 'Abstain']; - await expect(writer.verify({ ...input, msg: JSON.stringify(msg) })).resolves.toBeDefined(); + await expect(writer.verify({ ...input, msg: JSON.stringify(msg) })).resolves.toBeUndefined(); expect(mockGetSpace).toHaveBeenCalledTimes(1); expect(mockGetProposalsCount).toHaveBeenCalledTimes(1); }); @@ -203,7 +199,7 @@ describe('writer/proposal', () => { voting: { ...spacesGetSpaceFixtures.voting, period: VOTING_PERIOD } }); - await expect(writer.verify(inputWithVotingPeriod)).resolves.toBeDefined(); + await expect(writer.verify(inputWithVotingPeriod)).resolves.toBeUndefined(); expect(mockGetSpace).toHaveBeenCalledTimes(1); expect(mockGetProposalsCount).toHaveBeenCalledTimes(1); }); @@ -233,7 +229,9 @@ describe('writer/proposal', () => { const msg = JSON.parse(input.msg); msg.payload.start = msg.timestamp + VOTING_DELAY; - await expect(writer.verify({ ...input, msg: JSON.stringify(msg) })).resolves.toBeDefined(); + await expect( + writer.verify({ ...input, msg: JSON.stringify(msg) }) + ).resolves.toBeUndefined(); expect(mockGetSpace).toHaveBeenCalledTimes(1); expect(mockGetProposalsCount).toHaveBeenCalledTimes(1); }); @@ -298,7 +296,7 @@ describe('writer/proposal', () => { it('does not validate the space validation', async () => { expect.assertions(2); - await expect(writer.verify(input)).resolves.toBeDefined(); + await expect(writer.verify(input)).resolves.toBeUndefined(); expect(mockSnapshotUtilsValidate).toHaveBeenCalledTimes(0); }); }); @@ -307,7 +305,7 @@ describe('writer/proposal', () => { it('does not validate the space validation', async () => { expect.assertions(2); - await expect(writer.verify(input)).resolves.toBeDefined(); + await expect(writer.verify(input)).resolves.toBeUndefined(); expect(mockSnapshotUtilsValidate).toHaveBeenCalledTimes(0); }); }); @@ -323,19 +321,19 @@ describe('writer/proposal', () => { it('accepts a proposal with shutter privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: 'shutter' })) - ).resolves.toBeDefined(); + ).resolves.toBeUndefined(); }); it('accepts a proposal with undefined privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: undefined })) - ).resolves.toBeDefined(); + ).resolves.toBeUndefined(); }); it('accepts a proposal with no privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: '' })) - ).resolves.toBeDefined(); + ).resolves.toBeUndefined(); }); }); @@ -351,19 +349,19 @@ describe('writer/proposal', () => { it('accepts a proposal with shutter privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: 'shutter' })) - ).resolves.toBeDefined(); + ).resolves.toBeUndefined(); }); it('accepts a proposal with undefined privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: undefined })) - ).resolves.toBeDefined(); + ).resolves.toBeUndefined(); }); it('accepts a proposal with no privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: '' })) - ).resolves.toBeDefined(); + ).resolves.toBeUndefined(); }); }); @@ -384,13 +382,13 @@ describe('writer/proposal', () => { it('accepts a proposal with undefined privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: undefined })) - ).resolves.toBeDefined(); + ).resolves.toBeUndefined(); }); it('accepts a proposal with no privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: '' })) - ).resolves.toBeDefined(); + ).resolves.toBeUndefined(); }); }); @@ -405,13 +403,13 @@ describe('writer/proposal', () => { it('accepts a proposal with shutter privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: 'shutter' })) - ).resolves.toBeDefined(); + ).resolves.toBeUndefined(); }); it('accepts a proposal with undefined privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: undefined })) - ).resolves.toBeDefined(); + ).resolves.toBeUndefined(); }); it('rejects a proposal with privacy empty string', async () => { @@ -565,7 +563,7 @@ describe('writer/proposal', () => { it('verifies a valid input', async () => { expect.assertions(1); - await expect(writer.verify(input)).resolves.toBeDefined(); + await expect(writer.verify(input)).resolves.toBeUndefined(); }); }); }); From 27d36b5edeb609471641a5f8ec1e42622f47206d Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 17 Sep 2025 20:11:37 +0400 Subject: [PATCH 27/98] feat: refresh proposals vp_value_by_strategy async --- src/constants.ts | 4 +++- src/helpers/vpValue.ts | 50 ++++++++++++++++++++++++++++++++++++++++++ src/index.ts | 2 ++ src/writer/vote.ts | 4 ++-- 4 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 src/helpers/vpValue.ts diff --git a/src/constants.ts b/src/constants.ts index 53d57fd5..81d94da4 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,4 +1,6 @@ export const CB = { - UNELIGIBLE: -1, + INELIGIBLE: -1, PENDING_SYNC: -10 }; + +export const CURRENT_CB = parseInt(process.env.LAST_CB ?? '1'); diff --git a/src/helpers/vpValue.ts b/src/helpers/vpValue.ts new file mode 100644 index 00000000..cde0aa80 --- /dev/null +++ b/src/helpers/vpValue.ts @@ -0,0 +1,50 @@ +import { capture } from '@snapshot-labs/snapshot-sentry'; +import snapshot from '@snapshot-labs/snapshot.js'; +import { getVpValueByStrategy } from './entityValue'; +import db from './mysql'; +import { CB, CURRENT_CB } from '../constants'; + +type Proposal = { + id: string; + network: string; + start: number; + strategies: any[]; +}; + +const REFRESH_INTERVAL = 60 * 1000; + +async function getProposals(): Promise { + const query = + 'SELECT id, network, start, strategies FROM proposals WHERE cb = ? AND start < UNIX_TIMESTAMP() LIMIT 500'; + const proposals = await db.queryAsync(query, [CB.PENDING_SYNC]); + + return proposals.map((p: any) => { + p.strategies = JSON.parse(p.strategies); + return p; + }); +} + +async function refreshProposalVpValues(proposal: Proposal) { + try { + const values = await getVpValueByStrategy(proposal); + const query = 'UPDATE proposals SET vp_value_by_strategy = ?, cb = ? WHERE id = ? LIMIT 1'; + await db.queryAsync(query, [JSON.stringify(values), CURRENT_CB, proposal.id]); + } catch (e: any) { + capture(e); + } +} + +async function refreshProposals() { + const proposals = await getProposals(); + + for (const proposal of proposals) { + await refreshProposalVpValues(proposal); + } +} + +export default async function run() { + await refreshProposals(); + await snapshot.utils.sleep(REFRESH_INTERVAL); + + run(); +} diff --git a/src/index.ts b/src/index.ts index ee36c875..480e766d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -14,12 +14,14 @@ import { stop as stopStrategies } from './helpers/strategies'; import { trackTurboStatuses } from './helpers/turbo'; +import refreshVpValue from './helpers/vpValue'; const app = express(); async function startServer() { initLogger(app); refreshModeration(); + refreshVpValue(); await initializeStrategies(); refreshStrategies(); diff --git a/src/writer/vote.ts b/src/writer/vote.ts index dac25ea4..1c3e7dc0 100644 --- a/src/writer/vote.ts +++ b/src/writer/vote.ts @@ -1,5 +1,6 @@ import { capture } from '@snapshot-labs/snapshot-sentry'; import snapshot from '@snapshot-labs/snapshot.js'; +import { CURRENT_CB } from '../constants'; import { getProposal } from '../helpers/actions'; import { getVoteValue } from '../helpers/entityValue'; import log from '../helpers/log'; @@ -7,7 +8,6 @@ import db from '../helpers/mysql'; import { captureError, hasStrategyOverride, jsonParse } from '../helpers/utils'; import { updateProposalAndVotes } from '../scores'; -const LAST_CB = parseInt(process.env.LAST_CB ?? '1'); const scoreAPIUrl = process.env.SCORE_API_URL || 'https://score.snapshot.org'; // async function isLimitReached(space) { @@ -127,7 +127,7 @@ export async function action(body, ipfs, receipt, id, context): Promise { try { vpValue = getVoteValue(context.proposal, context.vp); - cb = LAST_CB; + cb = CURRENT_CB; } catch (e: any) { capture(e, { msg, proposalId, context }); } From 933bb27dad663b38fa708230d64d0ecd851d14c1 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Fri, 19 Sep 2025 20:26:46 +0400 Subject: [PATCH 28/98] fix: skip votes vp value computation when proposal score value is not ready yet --- src/writer/vote.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/writer/vote.ts b/src/writer/vote.ts index 1c3e7dc0..1b06eb74 100644 --- a/src/writer/vote.ts +++ b/src/writer/vote.ts @@ -1,6 +1,6 @@ import { capture } from '@snapshot-labs/snapshot-sentry'; import snapshot from '@snapshot-labs/snapshot.js'; -import { CURRENT_CB } from '../constants'; +import { CB, CURRENT_CB } from '../constants'; import { getProposal } from '../helpers/actions'; import { getVoteValue } from '../helpers/entityValue'; import log from '../helpers/log'; @@ -123,13 +123,15 @@ export async function action(body, ipfs, receipt, id, context): Promise { // Value is set on creation, and not updated on vote update // Value will be recomputed on proposal close let vpValue = 0; - let cb = 0; + let cb = CB.PENDING_SYNC; - try { - vpValue = getVoteValue(context.proposal, context.vp); - cb = CURRENT_CB; - } catch (e: any) { - capture(e, { msg, proposalId, context }); + if (context.proposal.cb >= 0) { + try { + vpValue = getVoteValue(context.proposal, context.vp); + cb = CURRENT_CB; + } catch (e: any) { + capture(e, { msg, proposalId, context }); + } } const params = { From 36cd96621005d91d4b8cd05d20ecc9af5cedf7a2 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Mon, 29 Sep 2025 16:20:30 +0400 Subject: [PATCH 29/98] fix: use only loop script to handle overlord logic --- src/constants.ts | 3 ++- src/helpers/vpValue.ts | 46 ++++++++++++++++++++++++++++++------------ src/writer/vote.ts | 23 +++------------------ 3 files changed, 38 insertions(+), 34 deletions(-) diff --git a/src/constants.ts b/src/constants.ts index 81d94da4..4ca6bf15 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,6 +1,7 @@ export const CB = { INELIGIBLE: -1, - PENDING_SYNC: -10 + PENDING_SYNC: -10, // waiting for value from overlord + PENDING_CLOSE: -20 // value from overlord set, waiting for proposal close for final computation }; export const CURRENT_CB = parseInt(process.env.LAST_CB ?? '1'); diff --git a/src/helpers/vpValue.ts b/src/helpers/vpValue.ts index cde0aa80..8dc26868 100644 --- a/src/helpers/vpValue.ts +++ b/src/helpers/vpValue.ts @@ -1,8 +1,8 @@ -import { capture } from '@snapshot-labs/snapshot-sentry'; +// import { capture } from '@snapshot-labs/snapshot-sentry'; import snapshot from '@snapshot-labs/snapshot.js'; import { getVpValueByStrategy } from './entityValue'; import db from './mysql'; -import { CB, CURRENT_CB } from '../constants'; +import { CB } from '../constants'; type Proposal = { id: string; @@ -12,11 +12,12 @@ type Proposal = { }; const REFRESH_INTERVAL = 60 * 1000; +const BATCH_SIZE = 100; async function getProposals(): Promise { const query = - 'SELECT id, network, start, strategies FROM proposals WHERE cb = ? AND start < UNIX_TIMESTAMP() LIMIT 500'; - const proposals = await db.queryAsync(query, [CB.PENDING_SYNC]); + 'SELECT id, network, start, strategies FROM proposals WHERE cb = ? AND start < UNIX_TIMESTAMP() LIMIT ?'; + const proposals = await db.queryAsync(query, [CB.PENDING_SYNC, BATCH_SIZE]); return proposals.map((p: any) => { p.strategies = JSON.parse(p.strategies); @@ -24,26 +25,45 @@ async function getProposals(): Promise { }); } -async function refreshProposalVpValues(proposal: Proposal) { +async function refreshProposalsVpValues(proposals: Proposal[]) { + const query: string[] = []; + const params: any[] = []; + + for (const proposal of proposals) { + await buildQuery(proposal, query, params); + } + + if (query.length) { + await db.queryAsync(query.join(';'), params); + } +} + +async function buildQuery(proposal: Proposal, query: string[], params: any[]) { try { const values = await getVpValueByStrategy(proposal); - const query = 'UPDATE proposals SET vp_value_by_strategy = ?, cb = ? WHERE id = ? LIMIT 1'; - await db.queryAsync(query, [JSON.stringify(values), CURRENT_CB, proposal.id]); - } catch (e: any) { - capture(e); + + query.push('UPDATE proposals SET vp_value_by_strategy = ?, cb = ? WHERE id = ? LIMIT 1'); + params.push(JSON.stringify(values), CB.PENDING_CLOSE, proposal.id); + } catch (e) { + // TODO: enable only after whole database is synced + // capture(e, { extra: { proposal } }); } } -async function refreshProposals() { +async function refreshPendingProposals() { const proposals = await getProposals(); - for (const proposal of proposals) { - await refreshProposalVpValues(proposal); + while (true) { + if (proposals.length === 0) break; + + await refreshProposalsVpValues(proposals); + + if (proposals.length < BATCH_SIZE) break; } } export default async function run() { - await refreshProposals(); + await refreshPendingProposals(); await snapshot.utils.sleep(REFRESH_INTERVAL); run(); diff --git a/src/writer/vote.ts b/src/writer/vote.ts index 1b06eb74..f3f1532c 100644 --- a/src/writer/vote.ts +++ b/src/writer/vote.ts @@ -1,8 +1,6 @@ -import { capture } from '@snapshot-labs/snapshot-sentry'; import snapshot from '@snapshot-labs/snapshot.js'; -import { CB, CURRENT_CB } from '../constants'; +import { CB } from '../constants'; import { getProposal } from '../helpers/actions'; -import { getVoteValue } from '../helpers/entityValue'; import log from '../helpers/log'; import db from '../helpers/mysql'; import { captureError, hasStrategyOverride, jsonParse } from '../helpers/utils'; @@ -119,21 +117,6 @@ export async function action(body, ipfs, receipt, id, context): Promise { const withOverride = hasStrategyOverride(context.proposal.strategies); if (vpState === 'final' && withOverride) vpState = 'pending'; - // Get proposal voting power value - // Value is set on creation, and not updated on vote update - // Value will be recomputed on proposal close - let vpValue = 0; - let cb = CB.PENDING_SYNC; - - if (context.proposal.cb >= 0) { - try { - vpValue = getVoteValue(context.proposal, context.vp); - cb = CURRENT_CB; - } catch (e: any) { - capture(e, { msg, proposalId, context }); - } - } - const params = { id, ipfs, @@ -148,8 +131,8 @@ export async function action(body, ipfs, receipt, id, context): Promise { vp: context.vp.vp, vp_by_strategy: JSON.stringify(context.vp.vp_by_strategy), vp_state: vpState, - vp_value: vpValue, - cb + vp_value: 0, + cb: CB.PENDING_SYNC }; // Check if voter already voted From 5a6534bca37f6f6144d6f5506db3241b86f25b13 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Mon, 29 Sep 2025 19:34:12 +0400 Subject: [PATCH 30/98] fix: process by most recent first --- src/helpers/vpValue.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/helpers/vpValue.ts b/src/helpers/vpValue.ts index 8dc26868..61593829 100644 --- a/src/helpers/vpValue.ts +++ b/src/helpers/vpValue.ts @@ -16,7 +16,7 @@ const BATCH_SIZE = 100; async function getProposals(): Promise { const query = - 'SELECT id, network, start, strategies FROM proposals WHERE cb = ? AND start < UNIX_TIMESTAMP() LIMIT ?'; + 'SELECT id, network, start, strategies FROM proposals WHERE cb = ? AND start < UNIX_TIMESTAMP() ORDER BY created DESC LIMIT ?'; const proposals = await db.queryAsync(query, [CB.PENDING_SYNC, BATCH_SIZE]); return proposals.map((p: any) => { From c70da7a6e979ff8636eec015ea7eccf3d4c06414 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Mon, 29 Sep 2025 20:36:47 +0400 Subject: [PATCH 31/98] refactor: better function name --- src/helpers/{vpValue.ts => proposalStrategiesValue.ts} | 0 src/index.ts | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename src/helpers/{vpValue.ts => proposalStrategiesValue.ts} (100%) diff --git a/src/helpers/vpValue.ts b/src/helpers/proposalStrategiesValue.ts similarity index 100% rename from src/helpers/vpValue.ts rename to src/helpers/proposalStrategiesValue.ts diff --git a/src/index.ts b/src/index.ts index 480e766d..3cf9d8f2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,6 +6,7 @@ import api from './api'; import log from './helpers/log'; import initMetrics from './helpers/metrics'; import refreshModeration from './helpers/moderation'; +import refreshProposalsVpValue from './helpers/proposalStrategiesValue'; import rateLimit from './helpers/rateLimit'; import shutter from './helpers/shutter'; import { @@ -14,14 +15,13 @@ import { stop as stopStrategies } from './helpers/strategies'; import { trackTurboStatuses } from './helpers/turbo'; -import refreshVpValue from './helpers/vpValue'; const app = express(); async function startServer() { initLogger(app); refreshModeration(); - refreshVpValue(); + refreshProposalsVpValue(); await initializeStrategies(); refreshStrategies(); From 49a9749367cc626359460c6083bd19eed6942ac0 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Mon, 29 Sep 2025 21:30:13 +0400 Subject: [PATCH 32/98] fix: fix loop using same data --- src/helpers/proposalStrategiesValue.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/helpers/proposalStrategiesValue.ts b/src/helpers/proposalStrategiesValue.ts index 61593829..ef574b08 100644 --- a/src/helpers/proposalStrategiesValue.ts +++ b/src/helpers/proposalStrategiesValue.ts @@ -51,9 +51,9 @@ async function buildQuery(proposal: Proposal, query: string[], params: any[]) { } async function refreshPendingProposals() { - const proposals = await getProposals(); - while (true) { + const proposals = await getProposals(); + if (proposals.length === 0) break; await refreshProposalsVpValues(proposals); From 57258536f53d4e8f4fbac7a497f2c3809bd4d0ef Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Mon, 29 Sep 2025 21:18:17 +0400 Subject: [PATCH 33/98] feat: add script to refresh votes' vp_value async --- src/helpers/entityValue.ts | 15 ++---- src/helpers/votesVpValue.ts | 75 +++++++++++++++++++++++++++ src/index.ts | 2 + test/unit/helpers/entityValue.test.ts | 61 +++++++++++++--------- 4 files changed, 118 insertions(+), 35 deletions(-) create mode 100644 src/helpers/votesVpValue.ts diff --git a/src/helpers/entityValue.ts b/src/helpers/entityValue.ts index 765ce1eb..0f354afe 100644 --- a/src/helpers/entityValue.ts +++ b/src/helpers/entityValue.ts @@ -1,9 +1,5 @@ import { jsonRpcRequest } from './utils'; -type Vote = { - vp_by_strategy: number[]; -}; - type Proposal = { network: string; strategies: any[]; @@ -33,15 +29,12 @@ export async function getVpValueByStrategy(proposal: Proposal): Promise sum + value * vote.vp_by_strategy[index], - 0 - ); + return vp_value_by_strategy.reduce((sum, value, index) => sum + value * vp_by_strategy[index], 0); } diff --git a/src/helpers/votesVpValue.ts b/src/helpers/votesVpValue.ts new file mode 100644 index 00000000..531632a9 --- /dev/null +++ b/src/helpers/votesVpValue.ts @@ -0,0 +1,75 @@ +// import { capture } from '@snapshot-labs/snapshot-sentry'; +import snapshot from '@snapshot-labs/snapshot.js'; +import { getVoteValue } from './entityValue'; +import db from './mysql'; +import { CB } from '../constants'; + +const REFRESH_INTERVAL = 60 * 1000; +const BATCH_SIZE = 100; + +type Datum = { + id: string; + vp_by_strategy: number[]; + vp_value_by_strategy: number[]; +}; + +async function getVotes(): Promise { + const query = ` + SELECT votes.id, votes.vp_by_strategy, proposals.vp_value_by_strategy + FROM votes + JOIN proposals ON votes.proposal = proposals.id + WHERE proposals.cb = ? AND votes.cb = ? + ORDER BY votes.created DESC + LIMIT ?`; + const results = await db.queryAsync(query, [CB.PENDING_CLOSE, CB.PENDING_SYNC, BATCH_SIZE]); + + return results.map((p: any) => { + p.vp_value_by_strategy = JSON.parse(p.vp_value_by_strategy); + p.vp_by_strategy = JSON.parse(p.vp_by_strategy); + return p; + }); +} + +async function refreshVotesVpValues(data: Datum[]) { + const query: string[] = []; + const params: any[] = []; + + for (const datum of data) { + buildQuery(datum, query, params); + } + + if (query.length) { + await db.queryAsync(query.join(';'), params); + } +} + +function buildQuery(datum: Datum, query: string[], params: any[]) { + try { + const value = getVoteValue(datum.vp_value_by_strategy, datum.vp_by_strategy); + + query.push('UPDATE votes SET vp_value = ?, cb = ? WHERE id = ? LIMIT 1'); + params.push(value, CB.PENDING_CLOSE, datum.id); + } catch (e) { + // TODO: enable only after whole database is synced + // capture(e, { extra: { proposal } }); + } +} + +async function refreshPendingVotes() { + while (true) { + const votes = await getVotes(); + + if (votes.length === 0) break; + + await refreshVotesVpValues(votes); + + if (votes.length < BATCH_SIZE) break; + } +} + +export default async function run() { + await refreshPendingVotes(); + await snapshot.utils.sleep(REFRESH_INTERVAL); + + run(); +} diff --git a/src/index.ts b/src/index.ts index 3cf9d8f2..a082bbee 100644 --- a/src/index.ts +++ b/src/index.ts @@ -15,6 +15,7 @@ import { stop as stopStrategies } from './helpers/strategies'; import { trackTurboStatuses } from './helpers/turbo'; +import refreshVotesVpValue from './helpers/votesVpValue'; const app = express(); @@ -22,6 +23,7 @@ async function startServer() { initLogger(app); refreshModeration(); refreshProposalsVpValue(); + refreshVotesVpValue(); await initializeStrategies(); refreshStrategies(); diff --git a/test/unit/helpers/entityValue.test.ts b/test/unit/helpers/entityValue.test.ts index 7dba57b1..fc02cd3c 100644 --- a/test/unit/helpers/entityValue.test.ts +++ b/test/unit/helpers/entityValue.test.ts @@ -2,69 +2,82 @@ import { getVoteValue } from '../../../src/helpers/entityValue'; describe('getVoteValue', () => { it('should calculate correct vote value with single strategy', () => { - const proposal = { vp_value_by_strategy: [2.5] }; - const vote = { vp_by_strategy: [100] }; + const vp_value_by_strategy = [2.5]; + const vp_by_strategy = [100]; - const result = getVoteValue(proposal, vote); + const result = getVoteValue(vp_value_by_strategy, vp_by_strategy); expect(result).toBe(250); }); it('should calculate correct vote value with multiple strategies', () => { - const proposal = { vp_value_by_strategy: [1.5, 3.0, 0.5] }; - const vote = { vp_by_strategy: [100, 50, 200] }; + const vp_value_by_strategy = [1.5, 3.0, 0.5]; + const vp_by_strategy = [100, 50, 200]; - const result = getVoteValue(proposal, vote); + const result = getVoteValue(vp_value_by_strategy, vp_by_strategy); expect(result).toBe(400); // (1.5 * 100) + (3.0 * 50) + (0.5 * 200) = 150 + 150 + 100 = 400 }); it('should return 0 when vote has no voting power', () => { - const proposal = { vp_value_by_strategy: [2.0, 1.5] }; - const vote = { vp_by_strategy: [0, 0] }; + const vp_value_by_strategy = [2.0, 1.5]; + const vp_by_strategy = [0, 0]; - const result = getVoteValue(proposal, vote); + const result = getVoteValue(vp_value_by_strategy, vp_by_strategy); expect(result).toBe(0); }); it('should return 0 when proposal has no value per strategy', () => { - const proposal = { vp_value_by_strategy: [0, 0] }; - const vote = { vp_by_strategy: [100, 50] }; + const vp_value_by_strategy = [0, 0]; + const vp_by_strategy = [100, 50]; - const result = getVoteValue(proposal, vote); + const result = getVoteValue(vp_value_by_strategy, vp_by_strategy); expect(result).toBe(0); }); it('should handle decimal values correctly', () => { - const proposal = { vp_value_by_strategy: [0.1, 0.25] }; - const vote = { vp_by_strategy: [10, 20] }; + const vp_value_by_strategy = [0.1, 0.25]; + const vp_by_strategy = [10, 20]; - const result = getVoteValue(proposal, vote); + const result = getVoteValue(vp_value_by_strategy, vp_by_strategy); expect(result).toBe(6); // (0.1 * 10) + (0.25 * 20) = 1 + 5 = 6 }); it('should throw error when strategy arrays have different lengths', () => { - const proposal = { vp_value_by_strategy: [1.0, 2.0] }; - const vote = { vp_by_strategy: [100] }; + const vp_value_by_strategy = [1.0, 2.0]; + const vp_by_strategy = [100]; - expect(() => getVoteValue(proposal, vote)).toThrow('invalid data to compute vote value'); + expect(() => getVoteValue(vp_value_by_strategy, vp_by_strategy)).toThrow( + 'invalid data to compute vote value' + ); }); it('should throw error when vote has more strategies than proposal', () => { - const proposal = { vp_value_by_strategy: [1.0] }; - const vote = { vp_by_strategy: [100, 50] }; + const vp_value_by_strategy = [1.0]; + const vp_by_strategy = [100, 50]; - expect(() => getVoteValue(proposal, vote)).toThrow('invalid data to compute vote value'); + expect(() => getVoteValue(vp_value_by_strategy, vp_by_strategy)).toThrow( + 'invalid data to compute vote value' + ); }); it('should handle empty arrays', () => { - const proposal = { vp_value_by_strategy: [] }; - const vote = { vp_by_strategy: [] }; + const vp_value_by_strategy = []; + const vp_by_strategy = []; - const result = getVoteValue(proposal, vote); + const result = getVoteValue(vp_value_by_strategy, vp_by_strategy); + + expect(result).toBe(0); + }); + + it('should return 0 when vp_value_by_strategy is empty', () => { + const vp_value_by_strategy = []; + const vp_by_strategy = [100, 50]; + + const result = getVoteValue(vp_value_by_strategy, vp_by_strategy); expect(result).toBe(0); }); From 690c0a56eded26ddd8cca30babe7dd725e558865 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 30 Sep 2025 00:11:08 +0400 Subject: [PATCH 34/98] fix: on new votes using overriding strategies, mark all votes to be resynced --- src/scores.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/scores.ts b/src/scores.ts index ec8c2d59..1ac5697e 100644 --- a/src/scores.ts +++ b/src/scores.ts @@ -1,4 +1,5 @@ import snapshot from '@snapshot-labs/snapshot.js'; +import { CB } from './constants'; import { getVoteValue } from './helpers/entityValue'; import log from './helpers/log'; import db from './helpers/mysql'; @@ -61,12 +62,13 @@ async function updateVotesVp(votes: any[], vpState: string, proposalId: string) let query = ''; votesInPage.forEach((vote: any) => { query += `UPDATE votes - SET vp = ?, vp_by_strategy = ?, vp_state = ?, vp_value = ? + SET vp = ?, vp_by_strategy = ?, vp_state = ?, vp_value = ?, cb = ? WHERE id = ? AND proposal = ? LIMIT 1; `; params.push(vote.balance); params.push(JSON.stringify(vote.scores)); params.push(vpState); params.push(vote.vp_value); + params.push(CB.PENDING_SYNC); params.push(vote.id); params.push(proposalId); }); From 486655d47be0de3f19bec8b822e3272b80c97637 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 30 Sep 2025 00:49:06 +0400 Subject: [PATCH 35/98] fix: add new CB value to trigger other fields computation --- src/constants.ts | 1 + src/helpers/proposalStrategiesValue.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/constants.ts b/src/constants.ts index 4ca6bf15..cf594b9b 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,6 +1,7 @@ export const CB = { INELIGIBLE: -1, PENDING_SYNC: -10, // waiting for value from overlord + PENDING_COMPUTE: -15, // value from overlord set, waiting for local computation or refresh PENDING_CLOSE: -20 // value from overlord set, waiting for proposal close for final computation }; diff --git a/src/helpers/proposalStrategiesValue.ts b/src/helpers/proposalStrategiesValue.ts index ef574b08..3b14eddd 100644 --- a/src/helpers/proposalStrategiesValue.ts +++ b/src/helpers/proposalStrategiesValue.ts @@ -43,7 +43,7 @@ async function buildQuery(proposal: Proposal, query: string[], params: any[]) { const values = await getVpValueByStrategy(proposal); query.push('UPDATE proposals SET vp_value_by_strategy = ?, cb = ? WHERE id = ? LIMIT 1'); - params.push(JSON.stringify(values), CB.PENDING_CLOSE, proposal.id); + params.push(JSON.stringify(values), CB.PENDING_COMPUTE, proposal.id); } catch (e) { // TODO: enable only after whole database is synced // capture(e, { extra: { proposal } }); From c94e6a831f668e7eba8ea9ced3229420eec8da73 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 30 Sep 2025 00:55:05 +0400 Subject: [PATCH 36/98] fix: use dedicated CB when value need refresh --- src/helpers/votesVpValue.ts | 8 ++++++-- src/scores.ts | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/helpers/votesVpValue.ts b/src/helpers/votesVpValue.ts index 531632a9..1e938443 100644 --- a/src/helpers/votesVpValue.ts +++ b/src/helpers/votesVpValue.ts @@ -18,10 +18,14 @@ async function getVotes(): Promise { SELECT votes.id, votes.vp_by_strategy, proposals.vp_value_by_strategy FROM votes JOIN proposals ON votes.proposal = proposals.id - WHERE proposals.cb = ? AND votes.cb = ? + WHERE proposals.cb IN (?) AND votes.cb = ? ORDER BY votes.created DESC LIMIT ?`; - const results = await db.queryAsync(query, [CB.PENDING_CLOSE, CB.PENDING_SYNC, BATCH_SIZE]); + const results = await db.queryAsync(query, [ + [CB.PENDING_CLOSE, CB.PENDING_COMPUTE], + CB.PENDING_SYNC, + BATCH_SIZE + ]); return results.map((p: any) => { p.vp_value_by_strategy = JSON.parse(p.vp_value_by_strategy); diff --git a/src/scores.ts b/src/scores.ts index 1ac5697e..3b23d889 100644 --- a/src/scores.ts +++ b/src/scores.ts @@ -68,7 +68,7 @@ async function updateVotesVp(votes: any[], vpState: string, proposalId: string) params.push(JSON.stringify(vote.scores)); params.push(vpState); params.push(vote.vp_value); - params.push(CB.PENDING_SYNC); + params.push(CB.PENDING_COMPUTE); params.push(vote.id); params.push(proposalId); }); From 76ee137701ff2f42bc9201cf7c1fc9653a4654b1 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 30 Sep 2025 01:19:02 +0400 Subject: [PATCH 37/98] fix: finalize votes CB --- src/helpers/votesVpValue.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/helpers/votesVpValue.ts b/src/helpers/votesVpValue.ts index 1e938443..01da8014 100644 --- a/src/helpers/votesVpValue.ts +++ b/src/helpers/votesVpValue.ts @@ -2,28 +2,29 @@ import snapshot from '@snapshot-labs/snapshot.js'; import { getVoteValue } from './entityValue'; import db from './mysql'; -import { CB } from '../constants'; +import { CB, CURRENT_CB } from '../constants'; const REFRESH_INTERVAL = 60 * 1000; const BATCH_SIZE = 100; type Datum = { id: string; + vp_state: string; vp_by_strategy: number[]; vp_value_by_strategy: number[]; }; async function getVotes(): Promise { const query = ` - SELECT votes.id, votes.vp_by_strategy, proposals.vp_value_by_strategy + SELECT votes.id, votes.vp_state, votes.vp_by_strategy, proposals.vp_value_by_strategy FROM votes JOIN proposals ON votes.proposal = proposals.id - WHERE proposals.cb IN (?) AND votes.cb = ? + WHERE proposals.cb IN (?) AND votes.cb IN (?) ORDER BY votes.created DESC LIMIT ?`; const results = await db.queryAsync(query, [ [CB.PENDING_CLOSE, CB.PENDING_COMPUTE], - CB.PENDING_SYNC, + [CB.PENDING_SYNC, CB.PENDING_COMPUTE], BATCH_SIZE ]); @@ -52,7 +53,7 @@ function buildQuery(datum: Datum, query: string[], params: any[]) { const value = getVoteValue(datum.vp_value_by_strategy, datum.vp_by_strategy); query.push('UPDATE votes SET vp_value = ?, cb = ? WHERE id = ? LIMIT 1'); - params.push(value, CB.PENDING_CLOSE, datum.id); + params.push(value, datum.vp_state === 'final' ? CURRENT_CB : CB.PENDING_CLOSE, datum.id); } catch (e) { // TODO: enable only after whole database is synced // capture(e, { extra: { proposal } }); From c79c4cfe1c9ebe1cfdac5845766942e58474f606 Mon Sep 17 00:00:00 2001 From: Wan <495709+wa0x6e@users.noreply.github.com> Date: Tue, 30 Sep 2025 21:16:53 +0900 Subject: [PATCH 38/98] Update src/helpers/proposalStrategiesValue.ts Co-authored-by: Less --- src/helpers/proposalStrategiesValue.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/helpers/proposalStrategiesValue.ts b/src/helpers/proposalStrategiesValue.ts index 3b14eddd..b72a3435 100644 --- a/src/helpers/proposalStrategiesValue.ts +++ b/src/helpers/proposalStrategiesValue.ts @@ -11,7 +11,7 @@ type Proposal = { strategies: any[]; }; -const REFRESH_INTERVAL = 60 * 1000; +const REFRESH_INTERVAL = 10 * 1000; const BATCH_SIZE = 100; async function getProposals(): Promise { From e5595f3b87a02c1b45682992849723b4b85e3541 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 30 Sep 2025 16:21:38 +0400 Subject: [PATCH 39/98] fix: use static CB value for closed status --- src/constants.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/constants.ts b/src/constants.ts index cf594b9b..897b69ff 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -2,7 +2,6 @@ export const CB = { INELIGIBLE: -1, PENDING_SYNC: -10, // waiting for value from overlord PENDING_COMPUTE: -15, // value from overlord set, waiting for local computation or refresh - PENDING_CLOSE: -20 // value from overlord set, waiting for proposal close for final computation + PENDING_CLOSE: -20, // value from overlord set, waiting for proposal close for final computation + CLOSED: 1 }; - -export const CURRENT_CB = parseInt(process.env.LAST_CB ?? '1'); From 862cddf97ab24d0116fa6f7668e20501a71751e0 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 30 Sep 2025 16:30:16 +0400 Subject: [PATCH 40/98] refactoring: code improvement --- src/helpers/proposalStrategiesValue.ts | 42 +++++++++++++------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/helpers/proposalStrategiesValue.ts b/src/helpers/proposalStrategiesValue.ts index b72a3435..900b5ad0 100644 --- a/src/helpers/proposalStrategiesValue.ts +++ b/src/helpers/proposalStrategiesValue.ts @@ -15,8 +15,13 @@ const REFRESH_INTERVAL = 10 * 1000; const BATCH_SIZE = 100; async function getProposals(): Promise { - const query = - 'SELECT id, network, start, strategies FROM proposals WHERE cb = ? AND start < UNIX_TIMESTAMP() ORDER BY created DESC LIMIT ?'; + const query = ` + SELECT id, network, start, strategies + FROM proposals + WHERE cb = ? AND start < UNIX_TIMESTAMP() + ORDER BY created DESC + LIMIT ? + `; const proposals = await db.queryAsync(query, [CB.PENDING_SYNC, BATCH_SIZE]); return proposals.map((p: any) => { @@ -25,12 +30,22 @@ async function getProposals(): Promise { }); } -async function refreshProposalsVpValues(proposals: Proposal[]) { +async function refreshVpByStrategy(proposals: Proposal[]) { const query: string[] = []; const params: any[] = []; for (const proposal of proposals) { - await buildQuery(proposal, query, params); + try { + const values = await getVpValueByStrategy(proposal); + + query.push('UPDATE proposals SET vp_value_by_strategy = ?, cb = ? WHERE id = ? LIMIT 1'); + params.push(JSON.stringify(values), CB.PENDING_COMPUTE, proposal.id); + } catch (e) { + // TODO: switch to capture only after whole database is synced + // to avoid quota issues + // capture(e, { extra: { proposal } }); + console.log(e); + } } if (query.length) { @@ -38,32 +53,17 @@ async function refreshProposalsVpValues(proposals: Proposal[]) { } } -async function buildQuery(proposal: Proposal, query: string[], params: any[]) { - try { - const values = await getVpValueByStrategy(proposal); - - query.push('UPDATE proposals SET vp_value_by_strategy = ?, cb = ? WHERE id = ? LIMIT 1'); - params.push(JSON.stringify(values), CB.PENDING_COMPUTE, proposal.id); - } catch (e) { - // TODO: enable only after whole database is synced - // capture(e, { extra: { proposal } }); - } -} - -async function refreshPendingProposals() { +export default async function run() { while (true) { const proposals = await getProposals(); if (proposals.length === 0) break; - await refreshProposalsVpValues(proposals); + await refreshVpByStrategy(proposals); if (proposals.length < BATCH_SIZE) break; } -} -export default async function run() { - await refreshPendingProposals(); await snapshot.utils.sleep(REFRESH_INTERVAL); run(); From 2be6513899acf9269c3f3a938a8c44df7de9b9c8 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 30 Sep 2025 16:35:30 +0400 Subject: [PATCH 41/98] fix: better constant name --- src/constants.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/constants.ts b/src/constants.ts index 897b69ff..e22d63e2 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -3,5 +3,5 @@ export const CB = { PENDING_SYNC: -10, // waiting for value from overlord PENDING_COMPUTE: -15, // value from overlord set, waiting for local computation or refresh PENDING_CLOSE: -20, // value from overlord set, waiting for proposal close for final computation - CLOSED: 1 + FINAL: 1 }; From 3ca345c5dcced0127f5e765f424680af9c640cc2 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 30 Sep 2025 16:55:33 +0400 Subject: [PATCH 42/98] fix: process older proposals first --- src/helpers/proposalStrategiesValue.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/helpers/proposalStrategiesValue.ts b/src/helpers/proposalStrategiesValue.ts index 96bfc219..7426538e 100644 --- a/src/helpers/proposalStrategiesValue.ts +++ b/src/helpers/proposalStrategiesValue.ts @@ -11,7 +11,7 @@ type Proposal = { }; const REFRESH_INTERVAL = 10 * 1000; -const BATCH_SIZE = 25; +const BATCH_SIZE = 10; async function getProposals(): Promise { const query = ` From 5e2da2b6352fe84b1af044eeec8b8dac2e0c1195 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 30 Sep 2025 17:12:41 +0400 Subject: [PATCH 43/98] fix: send request in batch --- src/helpers/proposalStrategiesValue.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/helpers/proposalStrategiesValue.ts b/src/helpers/proposalStrategiesValue.ts index 7426538e..96bfc219 100644 --- a/src/helpers/proposalStrategiesValue.ts +++ b/src/helpers/proposalStrategiesValue.ts @@ -11,7 +11,7 @@ type Proposal = { }; const REFRESH_INTERVAL = 10 * 1000; -const BATCH_SIZE = 10; +const BATCH_SIZE = 25; async function getProposals(): Promise { const query = ` From a23d2eeb68818963f7276334ebcea492e42f39f4 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 1 Oct 2025 01:38:07 +0400 Subject: [PATCH 44/98] fix: update to match proposal script convention --- src/helpers/votesVpValue.ts | 43 +++++++++++++++---------------------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/src/helpers/votesVpValue.ts b/src/helpers/votesVpValue.ts index 01da8014..8f03e9fd 100644 --- a/src/helpers/votesVpValue.ts +++ b/src/helpers/votesVpValue.ts @@ -2,9 +2,9 @@ import snapshot from '@snapshot-labs/snapshot.js'; import { getVoteValue } from './entityValue'; import db from './mysql'; -import { CB, CURRENT_CB } from '../constants'; +import { CB } from '../constants'; -const REFRESH_INTERVAL = 60 * 1000; +const REFRESH_INTERVAL = 10 * 1000; const BATCH_SIZE = 100; type Datum = { @@ -20,18 +20,18 @@ async function getVotes(): Promise { FROM votes JOIN proposals ON votes.proposal = proposals.id WHERE proposals.cb IN (?) AND votes.cb IN (?) - ORDER BY votes.created DESC + ORDER BY votes.created ASC LIMIT ?`; const results = await db.queryAsync(query, [ - [CB.PENDING_CLOSE, CB.PENDING_COMPUTE], + [CB.PENDING_CLOSE, CB.PENDING_COMPUTE, CB.FINAL], [CB.PENDING_SYNC, CB.PENDING_COMPUTE], BATCH_SIZE ]); - return results.map((p: any) => { - p.vp_value_by_strategy = JSON.parse(p.vp_value_by_strategy); - p.vp_by_strategy = JSON.parse(p.vp_by_strategy); - return p; + return results.map((r: any) => { + r.vp_value_by_strategy = JSON.parse(r.vp_value_by_strategy); + r.vp_by_strategy = JSON.parse(r.vp_by_strategy); + return r; }); } @@ -40,7 +40,14 @@ async function refreshVotesVpValues(data: Datum[]) { const params: any[] = []; for (const datum of data) { - buildQuery(datum, query, params); + try { + const value = getVoteValue(datum.vp_value_by_strategy, datum.vp_by_strategy); + + query.push('UPDATE votes SET vp_value = ?, cb = ? WHERE id = ? LIMIT 1'); + params.push(value, datum.vp_state === 'final' ? CB.FINAL : CB.PENDING_CLOSE, datum.id); + } catch (e) { + console.log(e); + } } if (query.length) { @@ -48,19 +55,7 @@ async function refreshVotesVpValues(data: Datum[]) { } } -function buildQuery(datum: Datum, query: string[], params: any[]) { - try { - const value = getVoteValue(datum.vp_value_by_strategy, datum.vp_by_strategy); - - query.push('UPDATE votes SET vp_value = ?, cb = ? WHERE id = ? LIMIT 1'); - params.push(value, datum.vp_state === 'final' ? CURRENT_CB : CB.PENDING_CLOSE, datum.id); - } catch (e) { - // TODO: enable only after whole database is synced - // capture(e, { extra: { proposal } }); - } -} - -async function refreshPendingVotes() { +export default async function run() { while (true) { const votes = await getVotes(); @@ -70,10 +65,6 @@ async function refreshPendingVotes() { if (votes.length < BATCH_SIZE) break; } -} - -export default async function run() { - await refreshPendingVotes(); await snapshot.utils.sleep(REFRESH_INTERVAL); run(); From 0004af2e1adb6bd38742bff603f22d8b177ade32 Mon Sep 17 00:00:00 2001 From: Wan <495709+wa0x6e@users.noreply.github.com> Date: Wed, 1 Oct 2025 06:42:53 +0900 Subject: [PATCH 45/98] Update src/helpers/votesVpValue.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/helpers/votesVpValue.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/helpers/votesVpValue.ts b/src/helpers/votesVpValue.ts index 8f03e9fd..c79c7cf0 100644 --- a/src/helpers/votesVpValue.ts +++ b/src/helpers/votesVpValue.ts @@ -57,15 +57,15 @@ async function refreshVotesVpValues(data: Datum[]) { export default async function run() { while (true) { - const votes = await getVotes(); + while (true) { + const votes = await getVotes(); - if (votes.length === 0) break; + if (votes.length === 0) break; - await refreshVotesVpValues(votes); + await refreshVotesVpValues(votes); - if (votes.length < BATCH_SIZE) break; + if (votes.length < BATCH_SIZE) break; + } + await snapshot.utils.sleep(REFRESH_INTERVAL); } - await snapshot.utils.sleep(REFRESH_INTERVAL); - - run(); } From b7f366ae951300adc7997e84194a14be2cc8d5fc Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 1 Oct 2025 01:54:32 +0400 Subject: [PATCH 46/98] chore: remove unused import --- src/helpers/votesVpValue.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/helpers/votesVpValue.ts b/src/helpers/votesVpValue.ts index c79c7cf0..9fbec673 100644 --- a/src/helpers/votesVpValue.ts +++ b/src/helpers/votesVpValue.ts @@ -1,4 +1,3 @@ -// import { capture } from '@snapshot-labs/snapshot-sentry'; import snapshot from '@snapshot-labs/snapshot.js'; import { getVoteValue } from './entityValue'; import db from './mysql'; From ef9498aa5629db8b4f32edd72e0c4cbc22b4a55b Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 1 Oct 2025 02:28:14 +0400 Subject: [PATCH 47/98] perf: better loop --- src/helpers/votesVpValue.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/helpers/votesVpValue.ts b/src/helpers/votesVpValue.ts index 9fbec673..d67c939a 100644 --- a/src/helpers/votesVpValue.ts +++ b/src/helpers/votesVpValue.ts @@ -56,15 +56,14 @@ async function refreshVotesVpValues(data: Datum[]) { export default async function run() { while (true) { - while (true) { - const votes = await getVotes(); - - if (votes.length === 0) break; + const votes = await getVotes(); + if (votes.length) { await refreshVotesVpValues(votes); + } - if (votes.length < BATCH_SIZE) break; + if (votes.length < BATCH_SIZE) { + await snapshot.utils.sleep(REFRESH_INTERVAL); } - await snapshot.utils.sleep(REFRESH_INTERVAL); } } From 7c0cc53baf64ae866e9fc27848ccbd92c7a44d25 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 1 Oct 2025 03:13:02 +0400 Subject: [PATCH 48/98] fix: mark votes as errored --- src/helpers/votesVpValue.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/helpers/votesVpValue.ts b/src/helpers/votesVpValue.ts index d67c939a..2f42a68b 100644 --- a/src/helpers/votesVpValue.ts +++ b/src/helpers/votesVpValue.ts @@ -39,13 +39,15 @@ async function refreshVotesVpValues(data: Datum[]) { const params: any[] = []; for (const datum of data) { + query.push('UPDATE votes SET vp_value = ?, cb = ? WHERE id = ? LIMIT 1'); + try { const value = getVoteValue(datum.vp_value_by_strategy, datum.vp_by_strategy); - query.push('UPDATE votes SET vp_value = ?, cb = ? WHERE id = ? LIMIT 1'); params.push(value, datum.vp_state === 'final' ? CB.FINAL : CB.PENDING_CLOSE, datum.id); } catch (e) { console.log(e); + params.push(0, CB.INELIGIBLE, datum.id); } } From 259473414afc2b145956ab79051de98a80b9cd73 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Mon, 4 Aug 2025 00:08:40 +0400 Subject: [PATCH 49/98] feat: set proposal vp value on proposal creation chore: fix types --- src/helpers/vpValue.ts | 58 ++++++++++++++++++++++++++++++++++++++++++ src/writer/proposal.ts | 3 +++ 2 files changed, 61 insertions(+) create mode 100644 src/helpers/vpValue.ts diff --git a/src/helpers/vpValue.ts b/src/helpers/vpValue.ts new file mode 100644 index 00000000..c0e09e3d --- /dev/null +++ b/src/helpers/vpValue.ts @@ -0,0 +1,58 @@ +import hubDB from './mysql'; +import { fetchWithKeepAlive } from './utils'; + +type Proposal = { + id: string; + network: string; + strategies: any[]; + start: number; +}; + +const OVERLORD_URL = 'https://overlord.snapshot.org'; +const CB_LAST = 5; +const CB_ERROR = 0; + +export async function setProposalVpValue(proposal: Proposal) { + if (proposal.start > Date.now()) { + return; + } + + try { + const vpValue = await getVpValue(proposal); + + await hubDB.queryAsync('UPDATE proposals SET vp_value = ?, cb = ? WHERE id = ? LIMIT 1', [ + vpValue, + CB_LAST, + proposal.id + ]); + } catch { + await hubDB.queryAsync('UPDATE proposals SET cb = ? WHERE id = ? LIMIT 1', [ + CB_ERROR, + proposal.id + ]); + return; + } +} + +async function getVpValue(proposal: Proposal) { + const init = { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + jsonrpc: '2.0', + method: 'get_vp_value_by_strategy', + params: { + network: proposal.network, + strategies: proposal.strategies, + snapshot: proposal.start + }, + id: Math.random().toString(36).substring(7) + }) + }; + const res = await fetchWithKeepAlive(OVERLORD_URL, init); + const { result } = await res.json(); + return result; +} diff --git a/src/writer/proposal.ts b/src/writer/proposal.ts index db3a7b2b..6be22c0b 100644 --- a/src/writer/proposal.ts +++ b/src/writer/proposal.ts @@ -11,6 +11,7 @@ import db from '../helpers/mysql'; import { getLimits, getSpaceType } from '../helpers/options'; import { validateSpaceSettings } from '../helpers/spaceValidation'; import { captureError, getQuorum, jsonParse, validateChoices } from '../helpers/utils'; +import { setProposalVpValue } from '../helpers/vpValue'; const scoreAPIUrl = process.env.SCORE_API_URL || 'https://score.snapshot.org'; const broviderUrl = process.env.BROVIDER_URL || 'https://rpc.snapshot.org'; @@ -315,4 +316,6 @@ export async function action(body, ipfs, receipt, id): Promise { `; await db.queryAsync(query, [proposal, space, author, space]); + + setProposalVpValue({ ...proposal, strategies: spaceSettings.strategies }); } From 764e7440dd315426b151ce4d7e555e19fd3f4b6e Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Mon, 4 Aug 2025 00:08:40 +0400 Subject: [PATCH 50/98] feat: set proposal strategies value on proposal creation fix: fix botched merge conflict chore: remove typo fix: fix typescript error fix: fix lint error fix: delete unused file --- .../{vpValue.ts => strategiesValue.ts} | 28 +-------------- src/writer/proposal.ts | 35 +++++++++++++++---- test/integration/writer/proposal.test.ts | 4 +-- 3 files changed, 31 insertions(+), 36 deletions(-) rename src/helpers/{vpValue.ts => strategiesValue.ts} (54%) diff --git a/src/helpers/vpValue.ts b/src/helpers/strategiesValue.ts similarity index 54% rename from src/helpers/vpValue.ts rename to src/helpers/strategiesValue.ts index c0e09e3d..38805fd6 100644 --- a/src/helpers/vpValue.ts +++ b/src/helpers/strategiesValue.ts @@ -1,40 +1,14 @@ -import hubDB from './mysql'; import { fetchWithKeepAlive } from './utils'; type Proposal = { - id: string; network: string; strategies: any[]; start: number; }; const OVERLORD_URL = 'https://overlord.snapshot.org'; -const CB_LAST = 5; -const CB_ERROR = 0; -export async function setProposalVpValue(proposal: Proposal) { - if (proposal.start > Date.now()) { - return; - } - - try { - const vpValue = await getVpValue(proposal); - - await hubDB.queryAsync('UPDATE proposals SET vp_value = ?, cb = ? WHERE id = ? LIMIT 1', [ - vpValue, - CB_LAST, - proposal.id - ]); - } catch { - await hubDB.queryAsync('UPDATE proposals SET cb = ? WHERE id = ? LIMIT 1', [ - CB_ERROR, - proposal.id - ]); - return; - } -} - -async function getVpValue(proposal: Proposal) { +export default async function getStrategiesValue(proposal: Proposal): Promise { const init = { method: 'POST', headers: { diff --git a/src/writer/proposal.ts b/src/writer/proposal.ts index 6be22c0b..99d31c5e 100644 --- a/src/writer/proposal.ts +++ b/src/writer/proposal.ts @@ -9,9 +9,8 @@ import { containsFlaggedLinks, flaggedAddresses } from '../helpers/moderation'; import { isMalicious } from '../helpers/monitoring'; import db from '../helpers/mysql'; import { getLimits, getSpaceType } from '../helpers/options'; -import { validateSpaceSettings } from '../helpers/spaceValidation'; +import getStrategiesValue from '../helpers/strategiesValue'; import { captureError, getQuorum, jsonParse, validateChoices } from '../helpers/utils'; -import { setProposalVpValue } from '../helpers/vpValue'; const scoreAPIUrl = process.env.SCORE_API_URL || 'https://score.snapshot.org'; const broviderUrl = process.env.BROVIDER_URL || 'https://rpc.snapshot.org'; @@ -239,9 +238,34 @@ export async function verify(body): Promise { if (msg.payload.choices.length > choicesLimit) { return Promise.reject(`number of choices can not exceed ${choicesLimit}`); } + + let strategiesValue: number[] = []; + + try { + strategiesValue = await getStrategiesValue({ + network: space.network, + start: msg.payload.start, + strategies: space.strategies + }); + + // Handle unlikely case where strategies value array length does not match strategies length + if (strategiesValue.length !== space.strategies.length) { + capture(new Error('Strategies value length mismatch'), { + space: space.id, + strategiesLength: space.strategies.length, + strategiesValue: JSON.stringify(strategiesValue) + }); + return Promise.reject('failed to get strategies value'); + } + } catch (e: any) { + console.log('unable to get strategies value', e.message); + return Promise.reject('failed to get strategies value'); + } + + return { strategiesValue }; } -export async function action(body, ipfs, receipt, id): Promise { +export async function action(body, ipfs, receipt, id, context): Promise { const msg = jsonParse(body.msg); const space = msg.space; @@ -299,8 +323,7 @@ export async function action(body, ipfs, receipt, id): Promise { scores_state: 'pending', scores_total: 0, scores_updated: 0, - scores_total_value: 0, - vp_value_by_strategy: JSON.stringify([]), + vp_value_by_strategy: JSON.stringify(context.strategiesValue), votes: 0, validation, flagged: +containsFlaggedLinks(msg.payload.body), @@ -316,6 +339,4 @@ export async function action(body, ipfs, receipt, id): Promise { `; await db.queryAsync(query, [proposal, space, author, space]); - - setProposalVpValue({ ...proposal, strategies: spaceSettings.strategies }); } diff --git a/test/integration/writer/proposal.test.ts b/test/integration/writer/proposal.test.ts index 80df065c..520ac5d8 100644 --- a/test/integration/writer/proposal.test.ts +++ b/test/integration/writer/proposal.test.ts @@ -45,7 +45,7 @@ describe('writer/proposal', () => { expect.hasAssertions(); mockContainsFlaggedLinks.mockReturnValueOnce(true); const id = '0x01-flagged'; - expect(await action(input, 'ipfs', 'receipt', id)).toBeUndefined(); + expect(await action(input, 'ipfs', 'receipt', id, {})).toBeUndefined(); expect(mockContainsFlaggedLinks).toBeCalledTimes(1); const [proposal] = await db.queryAsync('SELECT * FROM proposals WHERE id = ?', [id]); @@ -57,7 +57,7 @@ describe('writer/proposal', () => { it('creates and does not flag proposal', async () => { expect.hasAssertions(); const id = '0x02-non-flagged'; - expect(await action(input, 'ipfs', 'receipt', id)).toBeUndefined(); + expect(await action(input, 'ipfs', 'receipt', id, {})).toBeUndefined(); expect(mockContainsFlaggedLinks).toBeCalledTimes(1); const [proposal] = await db.queryAsync('SELECT * FROM proposals WHERE id = ?', [id]); From a4c07c42cb684492140b60731c0126a2db2a049f Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 6 Aug 2025 00:34:03 +0400 Subject: [PATCH 51/98] fix: set `cb` column --- .env.example | 11 +++++------ src/writer/proposal.ts | 4 +++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.env.example b/.env.example index 04fbdf4c..294931c7 100644 --- a/.env.example +++ b/.env.example @@ -10,14 +10,13 @@ SIDEKICK_URL=https://sh5.co PINEAPPLE_URL=https://.../upload # optional SCORE_API_URL=https://score.snapshot.org SCHNAPS_API_URL=https://schnaps.snapshot.box -# If you need unlimited access to score-api, use `https://score.snapshot.org?apiKey=...` -RATE_LIMIT_DATABASE_URL= # optional +# If you need unlimted access to score-api, use `https://score.snapshot.org?apiKey=...` +RATE_LIMIT_DATABASE_URL= # optional RATE_LIMIT_KEYS_PREFIX=snapshot-sequencer: # optional -BROVIDER_URL=https://rpc.snapshot.org # optional -STARKNET_RPC_URL= # optional +BROVIDER_URL=https://rpc.snapshot.org # optional +STARKNET_RPC_URL= # optional # Secret for laser AUTH_SECRET=1dfd84a695705665668c260222ded178d1f1d62d251d7bee8148428dac6d0487 # optional WALLETCONNECT_PROJECT_ID=e6454bd61aba40b786e866a69bd4c5c6 REOWN_SECRET= # optional -OVERLORD_URL=https://overlord.snapshot.box -LAST_CB=1 +LAST_CB=0 diff --git a/src/writer/proposal.ts b/src/writer/proposal.ts index 99d31c5e..9e997688 100644 --- a/src/writer/proposal.ts +++ b/src/writer/proposal.ts @@ -14,6 +14,7 @@ import { captureError, getQuorum, jsonParse, validateChoices } from '../helpers/ const scoreAPIUrl = process.env.SCORE_API_URL || 'https://score.snapshot.org'; const broviderUrl = process.env.BROVIDER_URL || 'https://rpc.snapshot.org'; +const LAST_CB = process.env.LAST_CB ?? 0; export const getProposalsCount = async (space, author) => { const query = ` @@ -323,11 +324,12 @@ export async function action(body, ipfs, receipt, id, context): Promise { scores_state: 'pending', scores_total: 0, scores_updated: 0, + scores_total_value: 0, vp_value_by_strategy: JSON.stringify(context.strategiesValue), votes: 0, validation, flagged: +containsFlaggedLinks(msg.payload.body), - cb: CB.PENDING_SYNC + cb: LAST_CB }; const query = ` From 18e7e5fcd3d02158a58e5d436f321052d624f365 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 6 Aug 2025 00:40:49 +0400 Subject: [PATCH 52/98] fix: default last_cb to 1 --- .env.example | 2 +- src/writer/proposal.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 294931c7..fb0da2e8 100644 --- a/.env.example +++ b/.env.example @@ -19,4 +19,4 @@ STARKNET_RPC_URL= # optional AUTH_SECRET=1dfd84a695705665668c260222ded178d1f1d62d251d7bee8148428dac6d0487 # optional WALLETCONNECT_PROJECT_ID=e6454bd61aba40b786e866a69bd4c5c6 REOWN_SECRET= # optional -LAST_CB=0 +LAST_CB=1 diff --git a/src/writer/proposal.ts b/src/writer/proposal.ts index 9e997688..712e8980 100644 --- a/src/writer/proposal.ts +++ b/src/writer/proposal.ts @@ -14,7 +14,7 @@ import { captureError, getQuorum, jsonParse, validateChoices } from '../helpers/ const scoreAPIUrl = process.env.SCORE_API_URL || 'https://score.snapshot.org'; const broviderUrl = process.env.BROVIDER_URL || 'https://rpc.snapshot.org'; -const LAST_CB = process.env.LAST_CB ?? 0; +const LAST_CB = parseInt(process.env.LAST_CB ?? '1'); export const getProposalsCount = async (space, author) => { const query = ` From 4ffbd36a08fabc69d1b5a181141eb0f93e476cee Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 6 Aug 2025 00:45:11 +0400 Subject: [PATCH 53/98] fix: round strategies value up to 9 decimals --- src/writer/proposal.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/writer/proposal.ts b/src/writer/proposal.ts index 712e8980..26dad679 100644 --- a/src/writer/proposal.ts +++ b/src/writer/proposal.ts @@ -258,6 +258,8 @@ export async function verify(body): Promise { }); return Promise.reject('failed to get strategies value'); } + + strategiesValue = strategiesValue.map(value => parseFloat(value.toFixed(9))); } catch (e: any) { console.log('unable to get strategies value', e.message); return Promise.reject('failed to get strategies value'); From 132912c98cdcd627cf9ec4a20b8d7d704282c6fb Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 6 Aug 2025 00:47:44 +0400 Subject: [PATCH 54/98] refactor: move precision to a const --- src/writer/proposal.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/writer/proposal.ts b/src/writer/proposal.ts index 26dad679..942fe1bf 100644 --- a/src/writer/proposal.ts +++ b/src/writer/proposal.ts @@ -15,6 +15,7 @@ import { captureError, getQuorum, jsonParse, validateChoices } from '../helpers/ const scoreAPIUrl = process.env.SCORE_API_URL || 'https://score.snapshot.org'; const broviderUrl = process.env.BROVIDER_URL || 'https://rpc.snapshot.org'; const LAST_CB = parseInt(process.env.LAST_CB ?? '1'); +const STRATEGIES_VALUE_PRECISION = 9; // Precision for strategies value export const getProposalsCount = async (space, author) => { const query = ` @@ -259,7 +260,9 @@ export async function verify(body): Promise { return Promise.reject('failed to get strategies value'); } - strategiesValue = strategiesValue.map(value => parseFloat(value.toFixed(9))); + strategiesValue = strategiesValue.map(value => + parseFloat(value.toFixed(STRATEGIES_VALUE_PRECISION)) + ); } catch (e: any) { console.log('unable to get strategies value', e.message); return Promise.reject('failed to get strategies value'); From 0f0f6410663b72ac1c69f5587291731eb091fe92 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 6 Aug 2025 00:51:22 +0400 Subject: [PATCH 55/98] chore: fix typo --- .env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.example b/.env.example index fb0da2e8..22ff6a63 100644 --- a/.env.example +++ b/.env.example @@ -10,7 +10,7 @@ SIDEKICK_URL=https://sh5.co PINEAPPLE_URL=https://.../upload # optional SCORE_API_URL=https://score.snapshot.org SCHNAPS_API_URL=https://schnaps.snapshot.box -# If you need unlimted access to score-api, use `https://score.snapshot.org?apiKey=...` +# If you need unlimited access to score-api, use `https://score.snapshot.org?apiKey=...` RATE_LIMIT_DATABASE_URL= # optional RATE_LIMIT_KEYS_PREFIX=snapshot-sequencer: # optional BROVIDER_URL=https://rpc.snapshot.org # optional From a9b066fd62d7436271ae4c3270343b6187902841 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 6 Aug 2025 00:56:04 +0400 Subject: [PATCH 56/98] test: fix test test: fix tests --- test/integration/writer/proposal.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/writer/proposal.test.ts b/test/integration/writer/proposal.test.ts index 520ac5d8..42d34922 100644 --- a/test/integration/writer/proposal.test.ts +++ b/test/integration/writer/proposal.test.ts @@ -45,7 +45,7 @@ describe('writer/proposal', () => { expect.hasAssertions(); mockContainsFlaggedLinks.mockReturnValueOnce(true); const id = '0x01-flagged'; - expect(await action(input, 'ipfs', 'receipt', id, {})).toBeUndefined(); + expect(await action(input, 'ipfs', 'receipt', id, { strategiesValue: [] })).toBeUndefined(); expect(mockContainsFlaggedLinks).toBeCalledTimes(1); const [proposal] = await db.queryAsync('SELECT * FROM proposals WHERE id = ?', [id]); @@ -57,7 +57,7 @@ describe('writer/proposal', () => { it('creates and does not flag proposal', async () => { expect.hasAssertions(); const id = '0x02-non-flagged'; - expect(await action(input, 'ipfs', 'receipt', id, {})).toBeUndefined(); + expect(await action(input, 'ipfs', 'receipt', id, { strategiesValue: [] })).toBeUndefined(); expect(mockContainsFlaggedLinks).toBeCalledTimes(1); const [proposal] = await db.queryAsync('SELECT * FROM proposals WHERE id = ?', [id]); From bb35683249950a9e264ff081c28d21336bd9f8d2 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 6 Aug 2025 01:41:10 +0400 Subject: [PATCH 57/98] test: fix tests --- src/helpers/strategiesValue.ts | 11 ++ src/writer/proposal.ts | 10 -- test/integration/ingestor.test.ts | 13 +- test/integration/writer/proposal.test.ts | 5 + test/unit/writer/proposal.test.ts | 171 +++++++++++++++++++---- 5 files changed, 166 insertions(+), 44 deletions(-) diff --git a/src/helpers/strategiesValue.ts b/src/helpers/strategiesValue.ts index 38805fd6..0a8123b0 100644 --- a/src/helpers/strategiesValue.ts +++ b/src/helpers/strategiesValue.ts @@ -1,3 +1,4 @@ +import { capture } from '@snapshot-labs/snapshot-sentry'; import { fetchWithKeepAlive } from './utils'; type Proposal = { @@ -28,5 +29,15 @@ export default async function getStrategiesValue(proposal: Proposal): Promise { strategies: space.strategies }); - // Handle unlikely case where strategies value array length does not match strategies length - if (strategiesValue.length !== space.strategies.length) { - capture(new Error('Strategies value length mismatch'), { - space: space.id, - strategiesLength: space.strategies.length, - strategiesValue: JSON.stringify(strategiesValue) - }); - return Promise.reject('failed to get strategies value'); - } - strategiesValue = strategiesValue.map(value => parseFloat(value.toFixed(STRATEGIES_VALUE_PRECISION)) ); diff --git a/test/integration/ingestor.test.ts b/test/integration/ingestor.test.ts index 82fff3c9..dcd1071c 100644 --- a/test/integration/ingestor.test.ts +++ b/test/integration/ingestor.test.ts @@ -112,16 +112,9 @@ jest.mock('@snapshot-labs/pineapple', () => { }; }); -jest.mock('../../src/helpers/strategies', () => ({ - getStrategies: jest.fn(() => ({ - 'erc20-balance-of': { id: 'erc20-balance-of', override: false, disabled: false }, - 'contract-call': { id: 'contract-call', override: true, disabled: false }, - delegation: { id: 'delegation', override: false, disabled: false }, - whitelist: { id: 'whitelist', override: false, disabled: false } - })), - initialize: jest.fn().mockResolvedValue(undefined), - run: jest.fn(), - stop: jest.fn() +jest.mock('../../src/helpers/strategiesValue', () => ({ + __esModule: true, + default: jest.fn(() => Promise.resolve([])) })); const proposalRequest = { diff --git a/test/integration/writer/proposal.test.ts b/test/integration/writer/proposal.test.ts index 42d34922..4b8ecdf6 100644 --- a/test/integration/writer/proposal.test.ts +++ b/test/integration/writer/proposal.test.ts @@ -19,6 +19,11 @@ jest.mock('../../../src/helpers/moderation', () => { }; }); +jest.mock('../../../src/helpers/strategiesValue', () => ({ + __esModule: true, + default: jest.fn(() => Promise.resolve([])) +})); + const getSpaceMock = jest.spyOn(actionHelper, 'getSpace'); getSpaceMock.mockResolvedValue(spacesGetSpaceFixtures); diff --git a/test/unit/writer/proposal.test.ts b/test/unit/writer/proposal.test.ts index 9c98d93e..a8900c37 100644 --- a/test/unit/writer/proposal.test.ts +++ b/test/unit/writer/proposal.test.ts @@ -98,10 +98,10 @@ jest.mock('../../../src/helpers/moderation', () => { }; }); -// Get the mocked function after the mock is created -const { validateSpaceSettings: mockValidateSpaceSettings } = jest.requireMock( - '../../../src/helpers/spaceValidation' -); +jest.mock('../../../src/helpers/strategiesValue', () => ({ + __esModule: true, + default: jest.fn(() => Promise.resolve([])) +})); const mockGetProposalsCount = jest.spyOn(writer, 'getProposalsCount'); mockGetProposalsCount.mockResolvedValue([ @@ -170,11 +170,50 @@ describe('writer/proposal', () => { msg.payload.type = 'basic'; msg.payload.choices = ['For', 'Against', 'Abstain']; - await expect(writer.verify({ ...input, msg: JSON.stringify(msg) })).resolves.toBeUndefined(); + await expect(writer.verify({ ...input, msg: JSON.stringify(msg) })).resolves.toBeDefined(); expect(mockGetSpace).toHaveBeenCalledTimes(1); expect(mockGetProposalsCount).toHaveBeenCalledTimes(1); }); + describe('when the space has enabled the ticket validation strategy', () => { + it('rejects if the space does not have voting validation', async () => { + expect.assertions(2); + mockGetSpace.mockResolvedValueOnce({ + ...spacesGetSpaceFixtures, + strategies: [{ name: 'ticket' }], + voteValidation: undefined + }); + + await expect(writer.verify(input)).rejects.toMatch('ticket'); + expect(mockGetSpace).toHaveBeenCalledTimes(1); + }); + + it('rejects if the space voting validation is ', async () => { + expect.assertions(2); + mockGetSpace.mockResolvedValueOnce({ + ...spacesGetSpaceFixtures, + strategies: [{ name: 'ticket' }], + voteValidation: { name: 'any' } + }); + + await expect(writer.verify(input)).rejects.toMatch('ticket'); + expect(mockGetSpace).toHaveBeenCalledTimes(1); + }); + + it('does not reject if the space voting validation is anything else valid than ', async () => { + expect.assertions(3); + mockGetSpace.mockResolvedValueOnce({ + ...spacesGetSpaceFixtures, + strategies: [{ name: 'ticket' }], + voteValidation: { name: 'gitcoin' } + }); + + await expect(writer.verify(input)).resolves.toBeDefined(); + expect(mockGetSpace).toHaveBeenCalledTimes(1); + expect(mockGetProposalsCount).toHaveBeenCalledTimes(1); + }); + }); + describe('when the space has set a voting period', () => { const VOTING_PERIOD = 120; const msg = JSON.parse(input.msg); @@ -199,7 +238,7 @@ describe('writer/proposal', () => { voting: { ...spacesGetSpaceFixtures.voting, period: VOTING_PERIOD } }); - await expect(writer.verify(inputWithVotingPeriod)).resolves.toBeUndefined(); + await expect(writer.verify(inputWithVotingPeriod)).resolves.toBeDefined(); expect(mockGetSpace).toHaveBeenCalledTimes(1); expect(mockGetProposalsCount).toHaveBeenCalledTimes(1); }); @@ -231,7 +270,7 @@ describe('writer/proposal', () => { await expect( writer.verify({ ...input, msg: JSON.stringify(msg) }) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); expect(mockGetSpace).toHaveBeenCalledTimes(1); expect(mockGetProposalsCount).toHaveBeenCalledTimes(1); }); @@ -296,7 +335,7 @@ describe('writer/proposal', () => { it('does not validate the space validation', async () => { expect.assertions(2); - await expect(writer.verify(input)).resolves.toBeUndefined(); + await expect(writer.verify(input)).resolves.toBeDefined(); expect(mockSnapshotUtilsValidate).toHaveBeenCalledTimes(0); }); }); @@ -305,7 +344,7 @@ describe('writer/proposal', () => { it('does not validate the space validation', async () => { expect.assertions(2); - await expect(writer.verify(input)).resolves.toBeUndefined(); + await expect(writer.verify(input)).resolves.toBeDefined(); expect(mockSnapshotUtilsValidate).toHaveBeenCalledTimes(0); }); }); @@ -321,19 +360,19 @@ describe('writer/proposal', () => { it('accepts a proposal with shutter privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: 'shutter' })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); it('accepts a proposal with undefined privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: undefined })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); it('accepts a proposal with no privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: '' })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); }); @@ -349,19 +388,19 @@ describe('writer/proposal', () => { it('accepts a proposal with shutter privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: 'shutter' })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); it('accepts a proposal with undefined privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: undefined })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); it('accepts a proposal with no privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: '' })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); }); @@ -382,13 +421,13 @@ describe('writer/proposal', () => { it('accepts a proposal with undefined privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: undefined })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); it('accepts a proposal with no privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: '' })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); }); @@ -403,13 +442,13 @@ describe('writer/proposal', () => { it('accepts a proposal with shutter privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: 'shutter' })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); it('accepts a proposal with undefined privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: undefined })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); it('rejects a proposal with privacy empty string', async () => { @@ -542,10 +581,94 @@ describe('writer/proposal', () => { ...spacesGetSpaceFixtures }); - await expect(writer.verify(input)).rejects.toMatch( - 'invalid space settings: space validation failed' - ); - expect(mockGetSpace).toHaveBeenCalledTimes(1); + it('rejects if the network does not exist', async () => { + expect.assertions(2); + mockGetSpace.mockResolvedValueOnce({ + ...spacesGetSpaceFixtures, + network: '123abc' + }); + + await expect(writer.verify(input)).rejects.toMatch( + 'invalid space settings: network not allowed' + ); + expect(mockGetSpace).toHaveBeenCalledTimes(1); + }); + + it('rejects if using a non-premium network', async () => { + expect.assertions(2); + mockGetSpace.mockResolvedValueOnce({ + ...spacesGetSpaceFixtures, + network: '56' // Using BSC network, which is not in the premium list + }); + + await expect(writer.verify(input)).rejects.toMatch('space is using a non-premium network'); + expect(mockGetSpace).toHaveBeenCalledTimes(1); + }); + + it('should not reject if using a premium network for turbo space', async () => { + expect.assertions(3); + mockGetSpace.mockResolvedValueOnce({ + ...spacesGetSpaceFixtures, + network: '56', // Using Ethereum mainnet, which is in the premium list + turbo: '1' + }); + + await expect(writer.verify(input)).resolves.toBeDefined(); + expect(mockGetSpace).toHaveBeenCalledTimes(1); + expect(mockGetProposalsCount).toHaveBeenCalledTimes(1); + }); + + it('rejects if strategies use a non-premium network', async () => { + expect.assertions(2); + mockGetSpace.mockResolvedValueOnce({ + ...spacesGetSpaceFixtures, + strategies: [{ name: 'erc20-balance-of', network: '56' }] // Non-premium network + }); + + await expect(writer.verify(input)).rejects.toMatch('space is using a non-premium network'); + expect(mockGetSpace).toHaveBeenCalledTimes(1); + }); + + it('rejects if missing proposal validation', async () => { + expect.assertions(2); + mockGetSpace.mockResolvedValueOnce({ + ...spacesGetSpaceFixtures, + validation: { name: 'any' } + }); + + await expect(writer.verify(input)).rejects.toMatch( + 'invalid space settings: space missing proposal validation' + ); + expect(mockGetSpace).toHaveBeenCalledTimes(1); + }); + + it('rejects if missing vote validation with ticket strategy', async () => { + expect.assertions(2); + mockGetSpace.mockResolvedValueOnce({ + ...spacesGetSpaceFixtures, + validation: { name: 'any' }, + strategies: [{ name: 'ticket' }] + }); + + await expect(writer.verify(input)).rejects.toMatch( + 'invalid space settings: space with ticket requires voting validation' + ); + expect(mockGetSpace).toHaveBeenCalledTimes(1); + }); + + it('rejects if the space was deleted', async () => { + expect.assertions(2); + mockGetSpace.mockResolvedValueOnce({ + ...spacesGetSpaceFixtures, + filters: { onlyMembers: true }, + deleted: true + }); + + await expect(writer.verify(input)).rejects.toMatch( + 'invalid space settings: space deleted, contact admin' + ); + expect(mockGetSpace).toHaveBeenCalledTimes(1); + }); }); describe('when only members can propose', () => { @@ -563,7 +686,7 @@ describe('writer/proposal', () => { it('verifies a valid input', async () => { expect.assertions(1); - await expect(writer.verify(input)).resolves.toBeUndefined(); + await expect(writer.verify(input)).resolves.toBeDefined(); }); }); }); From 1a633966d8e85434c86d6986b383c60596a3e9ff Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 6 Aug 2025 22:02:41 +0400 Subject: [PATCH 58/98] refactor: use entityValue as value logic handler --- src/helpers/entityValue.ts | 59 +++++++++++------------- src/helpers/strategiesValue.ts | 43 ----------------- src/writer/proposal.ts | 7 +-- test/integration/ingestor.test.ts | 4 +- test/integration/writer/proposal.test.ts | 4 +- test/unit/writer/proposal.test.ts | 4 +- 6 files changed, 35 insertions(+), 86 deletions(-) delete mode 100644 src/helpers/strategiesValue.ts diff --git a/src/helpers/entityValue.ts b/src/helpers/entityValue.ts index 765ce1eb..dda03547 100644 --- a/src/helpers/entityValue.ts +++ b/src/helpers/entityValue.ts @@ -1,8 +1,5 @@ -import { jsonRpcRequest } from './utils'; - -type Vote = { - vp_by_strategy: number[]; -}; +import { capture } from '@snapshot-labs/snapshot-sentry'; +import { fetchWithKeepAlive } from './utils'; type Proposal = { network: string; @@ -10,38 +7,38 @@ type Proposal = { start: number; }; -const OVERLORD_URL = process.env.OVERLORD_URL ?? 'https://overlord.snapshot.box'; -// Round strategy values to 9 decimal places +const OVERLORD_URL = 'https://overlord.snapshot.org'; const STRATEGIES_VALUE_PRECISION = 9; -export async function getVpValueByStrategy(proposal: Proposal): Promise { - const result: number[] = await jsonRpcRequest(OVERLORD_URL, 'get_value_by_strategy', { - network: proposal.network, - strategies: proposal.strategies, - snapshot: proposal.start // Expecting timestamp and not block number - }); +export async function getStrategiesValue(proposal: Proposal): Promise { + const init = { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + jsonrpc: '2.0', + method: 'get_vp_value_by_strategy', + params: { + network: proposal.network, + strategies: proposal.strategies, + snapshot: proposal.start + }, + id: Math.random().toString(36).substring(7) + }) + }; + const res = await fetchWithKeepAlive(OVERLORD_URL, init); + const { result } = await res.json(); // Handle unlikely case where strategies value array length does not match strategies length if (result.length !== proposal.strategies.length) { - throw new Error('Strategies value length mismatch'); + capture(new Error('Strategies value length mismatch'), { + strategiesLength: proposal.strategies.length, + result: JSON.stringify(result) + }); + return Promise.reject('failed to get strategies value'); } return result.map(value => parseFloat(value.toFixed(STRATEGIES_VALUE_PRECISION))); } - -/** - * Calculates the total vote value based on the voting power and the proposal's value per strategy. - * @returns The total vote value, in the currency unit specified by the proposal's vp_value_by_strategy values - **/ -export function getVoteValue(proposal: { vp_value_by_strategy: number[] }, vote: Vote): number { - if (!proposal.vp_value_by_strategy.length) return 0; - - if (proposal.vp_value_by_strategy.length !== vote.vp_by_strategy.length) { - throw new Error('invalid data to compute vote value'); - } - - return proposal.vp_value_by_strategy.reduce( - (sum, value, index) => sum + value * vote.vp_by_strategy[index], - 0 - ); -} diff --git a/src/helpers/strategiesValue.ts b/src/helpers/strategiesValue.ts deleted file mode 100644 index 0a8123b0..00000000 --- a/src/helpers/strategiesValue.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { capture } from '@snapshot-labs/snapshot-sentry'; -import { fetchWithKeepAlive } from './utils'; - -type Proposal = { - network: string; - strategies: any[]; - start: number; -}; - -const OVERLORD_URL = 'https://overlord.snapshot.org'; - -export default async function getStrategiesValue(proposal: Proposal): Promise { - const init = { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json' - }, - body: JSON.stringify({ - jsonrpc: '2.0', - method: 'get_vp_value_by_strategy', - params: { - network: proposal.network, - strategies: proposal.strategies, - snapshot: proposal.start - }, - id: Math.random().toString(36).substring(7) - }) - }; - const res = await fetchWithKeepAlive(OVERLORD_URL, init); - const { result } = await res.json(); - - // Handle unlikely case where strategies value array length does not match strategies length - if (result.length !== proposal.strategies.length) { - capture(new Error('Strategies value length mismatch'), { - strategiesLength: proposal.strategies.length, - result: JSON.stringify(result) - }); - return Promise.reject('failed to get strategies value'); - } - - return result; -} diff --git a/src/writer/proposal.ts b/src/writer/proposal.ts index 508c3ac7..1d696109 100644 --- a/src/writer/proposal.ts +++ b/src/writer/proposal.ts @@ -4,18 +4,17 @@ import networks from '@snapshot-labs/snapshot.js/src/networks.json'; import { uniq } from 'lodash'; import { CB } from '../constants'; import { getPremiumNetworkIds, getSpace } from '../helpers/actions'; +import { getStrategiesValue } from '../helpers/entityValue'; import log from '../helpers/log'; import { containsFlaggedLinks, flaggedAddresses } from '../helpers/moderation'; import { isMalicious } from '../helpers/monitoring'; import db from '../helpers/mysql'; import { getLimits, getSpaceType } from '../helpers/options'; -import getStrategiesValue from '../helpers/strategiesValue'; import { captureError, getQuorum, jsonParse, validateChoices } from '../helpers/utils'; const scoreAPIUrl = process.env.SCORE_API_URL || 'https://score.snapshot.org'; const broviderUrl = process.env.BROVIDER_URL || 'https://rpc.snapshot.org'; const LAST_CB = parseInt(process.env.LAST_CB ?? '1'); -const STRATEGIES_VALUE_PRECISION = 9; // Precision for strategies value export const getProposalsCount = async (space, author) => { const query = ` @@ -249,10 +248,6 @@ export async function verify(body): Promise { start: msg.payload.start, strategies: space.strategies }); - - strategiesValue = strategiesValue.map(value => - parseFloat(value.toFixed(STRATEGIES_VALUE_PRECISION)) - ); } catch (e: any) { console.log('unable to get strategies value', e.message); return Promise.reject('failed to get strategies value'); diff --git a/test/integration/ingestor.test.ts b/test/integration/ingestor.test.ts index dcd1071c..e75021c0 100644 --- a/test/integration/ingestor.test.ts +++ b/test/integration/ingestor.test.ts @@ -112,9 +112,9 @@ jest.mock('@snapshot-labs/pineapple', () => { }; }); -jest.mock('../../src/helpers/strategiesValue', () => ({ +jest.mock('../../src/helpers/entityValue', () => ({ __esModule: true, - default: jest.fn(() => Promise.resolve([])) + getStrategiesValue: jest.fn(() => Promise.resolve([])) })); const proposalRequest = { diff --git a/test/integration/writer/proposal.test.ts b/test/integration/writer/proposal.test.ts index 4b8ecdf6..831fdf58 100644 --- a/test/integration/writer/proposal.test.ts +++ b/test/integration/writer/proposal.test.ts @@ -19,9 +19,9 @@ jest.mock('../../../src/helpers/moderation', () => { }; }); -jest.mock('../../../src/helpers/strategiesValue', () => ({ +jest.mock('../../../src/helpers/entityValue', () => ({ __esModule: true, - default: jest.fn(() => Promise.resolve([])) + getStrategiesValue: jest.fn(() => Promise.resolve([])) })); const getSpaceMock = jest.spyOn(actionHelper, 'getSpace'); diff --git a/test/unit/writer/proposal.test.ts b/test/unit/writer/proposal.test.ts index a8900c37..8f191ac6 100644 --- a/test/unit/writer/proposal.test.ts +++ b/test/unit/writer/proposal.test.ts @@ -98,9 +98,9 @@ jest.mock('../../../src/helpers/moderation', () => { }; }); -jest.mock('../../../src/helpers/strategiesValue', () => ({ +jest.mock('../../../src/helpers/entityValue', () => ({ __esModule: true, - default: jest.fn(() => Promise.resolve([])) + getStrategiesValue: jest.fn(() => Promise.resolve([])) })); const mockGetProposalsCount = jest.spyOn(writer, 'getProposalsCount'); From 88a3b8cc8b8cdfd8c9317d494480a5a10520a9dc Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 6 Aug 2025 22:15:57 +0400 Subject: [PATCH 59/98] fix: use randomUUID to generate random number --- src/helpers/entityValue.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/helpers/entityValue.ts b/src/helpers/entityValue.ts index dda03547..9524cc50 100644 --- a/src/helpers/entityValue.ts +++ b/src/helpers/entityValue.ts @@ -1,3 +1,4 @@ +import { randomUUID } from 'crypto'; import { capture } from '@snapshot-labs/snapshot-sentry'; import { fetchWithKeepAlive } from './utils'; @@ -25,7 +26,7 @@ export async function getStrategiesValue(proposal: Proposal): Promise strategies: proposal.strategies, snapshot: proposal.start }, - id: Math.random().toString(36).substring(7) + id: randomUUID() }) }; const res = await fetchWithKeepAlive(OVERLORD_URL, init); From 6a898320445ffb388e6be96f1d1548b2ff07a231 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 6 Aug 2025 22:17:50 +0400 Subject: [PATCH 60/98] chore: use consistent logging --- src/writer/proposal.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/writer/proposal.ts b/src/writer/proposal.ts index 1d696109..fa1a84e7 100644 --- a/src/writer/proposal.ts +++ b/src/writer/proposal.ts @@ -249,7 +249,7 @@ export async function verify(body): Promise { strategies: space.strategies }); } catch (e: any) { - console.log('unable to get strategies value', e.message); + log.warn('unable to get strategies value', e.message); return Promise.reject('failed to get strategies value'); } From b50edfc85db981cce660aa1aad8e78347e15d9a4 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 6 Aug 2025 23:22:50 +0400 Subject: [PATCH 61/98] fix: handle overlord error --- src/helpers/entityValue.ts | 68 ++++++++++++++++++++++++++++++++------ 1 file changed, 57 insertions(+), 11 deletions(-) diff --git a/src/helpers/entityValue.ts b/src/helpers/entityValue.ts index 9524cc50..41e3a158 100644 --- a/src/helpers/entityValue.ts +++ b/src/helpers/entityValue.ts @@ -29,17 +29,63 @@ export async function getStrategiesValue(proposal: Proposal): Promise id: randomUUID() }) }; - const res = await fetchWithKeepAlive(OVERLORD_URL, init); - const { result } = await res.json(); - - // Handle unlikely case where strategies value array length does not match strategies length - if (result.length !== proposal.strategies.length) { - capture(new Error('Strategies value length mismatch'), { - strategiesLength: proposal.strategies.length, - result: JSON.stringify(result) + + try { + const res = await fetchWithKeepAlive(OVERLORD_URL, init); + + // Check HTTP status + if (!res.ok) { + capture(new Error('HTTP error response'), { + status: res.status, + statusText: res.statusText, + url: OVERLORD_URL + }); + return Promise.reject(`HTTP error: ${res.status} ${res.statusText}`); + } + + const response = await res.json(); + + // Handle JSON-RPC error response + if (response.error) { + capture(new Error('JSON-RPC error response'), { + error: response.error, + request: { + network: proposal.network, + strategiesLength: proposal.strategies.length, + snapshot: proposal.start + } + }); + return Promise.reject( + `JSON-RPC error: ${response.error.message || response.error.code || 'Unknown error'}` + ); + } + + const { result } = response; + + // Handle unlikely case where strategies value array length does not match strategies length + if (result.length !== proposal.strategies.length) { + capture(new Error('Strategies value length mismatch'), { + strategiesLength: proposal.strategies.length, + result: JSON.stringify(result) + }); + return Promise.reject('failed to get strategies value'); + } + + return result.map(value => parseFloat(value.toFixed(STRATEGIES_VALUE_PRECISION))); + } catch (error) { + capture(new Error('Network or parsing error'), { + error: error instanceof Error ? error.message : String(error), + url: OVERLORD_URL, + request: { + network: proposal.network, + strategiesLength: proposal.strategies.length, + snapshot: proposal.start + } }); - return Promise.reject('failed to get strategies value'); + return Promise.reject( + `Failed to fetch strategies value: ${ + error instanceof Error ? error.message : 'Unknown error' + }` + ); } - - return result.map(value => parseFloat(value.toFixed(STRATEGIES_VALUE_PRECISION))); } From b9163abddb84b6b4f27d07dd489afffc634ea22f Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 6 Aug 2025 23:27:26 +0400 Subject: [PATCH 62/98] fix: more precise rounding --- src/helpers/entityValue.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/helpers/entityValue.ts b/src/helpers/entityValue.ts index 41e3a158..283765b9 100644 --- a/src/helpers/entityValue.ts +++ b/src/helpers/entityValue.ts @@ -10,6 +10,7 @@ type Proposal = { const OVERLORD_URL = 'https://overlord.snapshot.org'; const STRATEGIES_VALUE_PRECISION = 9; +const PRECISION_MULTIPLIER = Math.pow(10, STRATEGIES_VALUE_PRECISION); export async function getStrategiesValue(proposal: Proposal): Promise { const init = { @@ -71,7 +72,7 @@ export async function getStrategiesValue(proposal: Proposal): Promise return Promise.reject('failed to get strategies value'); } - return result.map(value => parseFloat(value.toFixed(STRATEGIES_VALUE_PRECISION))); + return result.map(value => Math.round(value * PRECISION_MULTIPLIER) / PRECISION_MULTIPLIER); } catch (error) { capture(new Error('Network or parsing error'), { error: error instanceof Error ? error.message : String(error), From 54730753574a5e69fa54d45827711481411770cc Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Thu, 7 Aug 2025 19:43:30 +0400 Subject: [PATCH 63/98] refactor: DRY json rpc request fetcher --- src/helpers/entityValue.ts | 91 ++++++-------------------------------- src/helpers/utils.ts | 4 +- 2 files changed, 15 insertions(+), 80 deletions(-) diff --git a/src/helpers/entityValue.ts b/src/helpers/entityValue.ts index 283765b9..6c07125b 100644 --- a/src/helpers/entityValue.ts +++ b/src/helpers/entityValue.ts @@ -1,6 +1,4 @@ -import { randomUUID } from 'crypto'; -import { capture } from '@snapshot-labs/snapshot-sentry'; -import { fetchWithKeepAlive } from './utils'; +import { jsonRpcRequest } from './utils'; type Proposal = { network: string; @@ -9,84 +7,21 @@ type Proposal = { }; const OVERLORD_URL = 'https://overlord.snapshot.org'; +// Round strategy values to 9 decimal places const STRATEGIES_VALUE_PRECISION = 9; const PRECISION_MULTIPLIER = Math.pow(10, STRATEGIES_VALUE_PRECISION); export async function getStrategiesValue(proposal: Proposal): Promise { - const init = { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json' - }, - body: JSON.stringify({ - jsonrpc: '2.0', - method: 'get_vp_value_by_strategy', - params: { - network: proposal.network, - strategies: proposal.strategies, - snapshot: proposal.start - }, - id: randomUUID() - }) - }; - - try { - const res = await fetchWithKeepAlive(OVERLORD_URL, init); - - // Check HTTP status - if (!res.ok) { - capture(new Error('HTTP error response'), { - status: res.status, - statusText: res.statusText, - url: OVERLORD_URL - }); - return Promise.reject(`HTTP error: ${res.status} ${res.statusText}`); - } - - const response = await res.json(); - - // Handle JSON-RPC error response - if (response.error) { - capture(new Error('JSON-RPC error response'), { - error: response.error, - request: { - network: proposal.network, - strategiesLength: proposal.strategies.length, - snapshot: proposal.start - } - }); - return Promise.reject( - `JSON-RPC error: ${response.error.message || response.error.code || 'Unknown error'}` - ); - } - - const { result } = response; - - // Handle unlikely case where strategies value array length does not match strategies length - if (result.length !== proposal.strategies.length) { - capture(new Error('Strategies value length mismatch'), { - strategiesLength: proposal.strategies.length, - result: JSON.stringify(result) - }); - return Promise.reject('failed to get strategies value'); - } - - return result.map(value => Math.round(value * PRECISION_MULTIPLIER) / PRECISION_MULTIPLIER); - } catch (error) { - capture(new Error('Network or parsing error'), { - error: error instanceof Error ? error.message : String(error), - url: OVERLORD_URL, - request: { - network: proposal.network, - strategiesLength: proposal.strategies.length, - snapshot: proposal.start - } - }); - return Promise.reject( - `Failed to fetch strategies value: ${ - error instanceof Error ? error.message : 'Unknown error' - }` - ); + const result: number[] = await jsonRpcRequest(OVERLORD_URL, 'get_vp_value_by_strategy', { + network: proposal.network, + strategies: proposal.strategies, + snapshot: proposal.start + }); + + // Handle unlikely case where strategies value array length does not match strategies length + if (result.length !== proposal.strategies.length) { + throw new Error('Strategies value length mismatch'); } + + return result.map(value => Math.round(value * PRECISION_MULTIPLIER) / PRECISION_MULTIPLIER); } diff --git a/src/helpers/utils.ts b/src/helpers/utils.ts index 5720d097..bf246eeb 100644 --- a/src/helpers/utils.ts +++ b/src/helpers/utils.ts @@ -1,4 +1,4 @@ -import { createHash, randomInt } from 'crypto'; +import { createHash, randomUUID } from 'crypto'; import http from 'http'; import https from 'https'; import { URL } from 'url'; @@ -73,7 +73,7 @@ export async function jsonRpcRequest(url: string, method: string, params: any): jsonrpc: '2.0', method, params, - id: randomInt(10000) + id: randomUUID() }) }; From b98aa04fb82f43985f923ad5bdbdf65889aadcdb Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Thu, 7 Aug 2025 20:06:29 +0400 Subject: [PATCH 64/98] refactor: simplify rounding --- src/helpers/entityValue.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/helpers/entityValue.ts b/src/helpers/entityValue.ts index 6c07125b..5c6fd80d 100644 --- a/src/helpers/entityValue.ts +++ b/src/helpers/entityValue.ts @@ -9,7 +9,6 @@ type Proposal = { const OVERLORD_URL = 'https://overlord.snapshot.org'; // Round strategy values to 9 decimal places const STRATEGIES_VALUE_PRECISION = 9; -const PRECISION_MULTIPLIER = Math.pow(10, STRATEGIES_VALUE_PRECISION); export async function getStrategiesValue(proposal: Proposal): Promise { const result: number[] = await jsonRpcRequest(OVERLORD_URL, 'get_vp_value_by_strategy', { @@ -23,5 +22,5 @@ export async function getStrategiesValue(proposal: Proposal): Promise throw new Error('Strategies value length mismatch'); } - return result.map(value => Math.round(value * PRECISION_MULTIPLIER) / PRECISION_MULTIPLIER); + return result.map(value => parseFloat(value.toFixed(STRATEGIES_VALUE_PRECISION))); } From 4e55ece0ada4f5eb3a42d0de655dd5ecf4b119ef Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Thu, 7 Aug 2025 20:28:03 +0400 Subject: [PATCH 65/98] fix: allow customize overlord url via env var --- .env.example | 1 + src/helpers/entityValue.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 22ff6a63..1bd3b482 100644 --- a/.env.example +++ b/.env.example @@ -19,4 +19,5 @@ STARKNET_RPC_URL= # optional AUTH_SECRET=1dfd84a695705665668c260222ded178d1f1d62d251d7bee8148428dac6d0487 # optional WALLETCONNECT_PROJECT_ID=e6454bd61aba40b786e866a69bd4c5c6 REOWN_SECRET= # optional +OVERLORD_URL=https://overlord.snapshot.org LAST_CB=1 diff --git a/src/helpers/entityValue.ts b/src/helpers/entityValue.ts index 5c6fd80d..66478445 100644 --- a/src/helpers/entityValue.ts +++ b/src/helpers/entityValue.ts @@ -6,7 +6,7 @@ type Proposal = { start: number; }; -const OVERLORD_URL = 'https://overlord.snapshot.org'; +const OVERLORD_URL = process.env.OVERLORD_URL ?? 'https://overlord.snapshot.org'; // Round strategy values to 9 decimal places const STRATEGIES_VALUE_PRECISION = 9; From 110893a099af0b26e3d86ff85e0b33b68c00d6e6 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Thu, 7 Aug 2025 20:32:06 +0400 Subject: [PATCH 66/98] chore: remove white spaces --- .env.example | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 1bd3b482..7a9ab964 100644 --- a/.env.example +++ b/.env.example @@ -11,10 +11,10 @@ PINEAPPLE_URL=https://.../upload # optional SCORE_API_URL=https://score.snapshot.org SCHNAPS_API_URL=https://schnaps.snapshot.box # If you need unlimited access to score-api, use `https://score.snapshot.org?apiKey=...` -RATE_LIMIT_DATABASE_URL= # optional +RATE_LIMIT_DATABASE_URL= # optional RATE_LIMIT_KEYS_PREFIX=snapshot-sequencer: # optional -BROVIDER_URL=https://rpc.snapshot.org # optional -STARKNET_RPC_URL= # optional +BROVIDER_URL=https://rpc.snapshot.org # optional +STARKNET_RPC_URL= # optional # Secret for laser AUTH_SECRET=1dfd84a695705665668c260222ded178d1f1d62d251d7bee8148428dac6d0487 # optional WALLETCONNECT_PROJECT_ID=e6454bd61aba40b786e866a69bd4c5c6 From 169e3426bbdba6453185c0e5ed04640e9a895b0c Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 16 Sep 2025 14:48:52 +0400 Subject: [PATCH 67/98] fix: fix types --- src/helpers/entityValue.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/helpers/entityValue.ts b/src/helpers/entityValue.ts index 66478445..bed9a1ac 100644 --- a/src/helpers/entityValue.ts +++ b/src/helpers/entityValue.ts @@ -24,3 +24,20 @@ export async function getStrategiesValue(proposal: Proposal): Promise return result.map(value => parseFloat(value.toFixed(STRATEGIES_VALUE_PRECISION))); } + +/** + * Calculates the total vote value based on the voting power and the proposal's value per strategy. + * @returns The total vote value, in the currency unit specified by the proposal's vp_value_by_strategy values + **/ +export function getVoteValue(proposal: { vp_value_by_strategy: number[] }, vote: Vote): number { + if (!proposal.vp_value_by_strategy.length) return 0; + + if (proposal.vp_value_by_strategy.length !== vote.vp_by_strategy.length) { + throw new Error('invalid data to compute vote value'); + } + + return proposal.vp_value_by_strategy.reduce( + (sum, value, index) => sum + value * vote.vp_by_strategy[index], + 0 + ); +} From 0c010207596e34695a1d526767dbf94ca93ff34f Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 16 Sep 2025 15:16:21 +0400 Subject: [PATCH 68/98] fix: update overlord url --- .env.example | 2 +- src/helpers/entityValue.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 7a9ab964..04fbdf4c 100644 --- a/.env.example +++ b/.env.example @@ -19,5 +19,5 @@ STARKNET_RPC_URL= # optional AUTH_SECRET=1dfd84a695705665668c260222ded178d1f1d62d251d7bee8148428dac6d0487 # optional WALLETCONNECT_PROJECT_ID=e6454bd61aba40b786e866a69bd4c5c6 REOWN_SECRET= # optional -OVERLORD_URL=https://overlord.snapshot.org +OVERLORD_URL=https://overlord.snapshot.box LAST_CB=1 diff --git a/src/helpers/entityValue.ts b/src/helpers/entityValue.ts index bed9a1ac..99944e1b 100644 --- a/src/helpers/entityValue.ts +++ b/src/helpers/entityValue.ts @@ -6,7 +6,7 @@ type Proposal = { start: number; }; -const OVERLORD_URL = process.env.OVERLORD_URL ?? 'https://overlord.snapshot.org'; +const OVERLORD_URL = process.env.OVERLORD_URL ?? 'https://overlord.snapshot.box'; // Round strategy values to 9 decimal places const STRATEGIES_VALUE_PRECISION = 9; From fba0e955613872999b0501b4ad2ac812dc257b80 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 16 Sep 2025 15:27:54 +0400 Subject: [PATCH 69/98] refactor: more consistent function name --- src/helpers/entityValue.ts | 2 +- src/writer/proposal.ts | 4 ++-- test/integration/ingestor.test.ts | 2 +- test/integration/writer/proposal.test.ts | 2 +- test/unit/writer/proposal.test.ts | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/helpers/entityValue.ts b/src/helpers/entityValue.ts index 99944e1b..757d2d0c 100644 --- a/src/helpers/entityValue.ts +++ b/src/helpers/entityValue.ts @@ -10,7 +10,7 @@ const OVERLORD_URL = process.env.OVERLORD_URL ?? 'https://overlord.snapshot.box' // Round strategy values to 9 decimal places const STRATEGIES_VALUE_PRECISION = 9; -export async function getStrategiesValue(proposal: Proposal): Promise { +export async function getVpValueByStrategy(proposal: Proposal): Promise { const result: number[] = await jsonRpcRequest(OVERLORD_URL, 'get_vp_value_by_strategy', { network: proposal.network, strategies: proposal.strategies, diff --git a/src/writer/proposal.ts b/src/writer/proposal.ts index fa1a84e7..c8c74901 100644 --- a/src/writer/proposal.ts +++ b/src/writer/proposal.ts @@ -4,7 +4,7 @@ import networks from '@snapshot-labs/snapshot.js/src/networks.json'; import { uniq } from 'lodash'; import { CB } from '../constants'; import { getPremiumNetworkIds, getSpace } from '../helpers/actions'; -import { getStrategiesValue } from '../helpers/entityValue'; +import { getVpValueByStrategy } from '../helpers/entityValue'; import log from '../helpers/log'; import { containsFlaggedLinks, flaggedAddresses } from '../helpers/moderation'; import { isMalicious } from '../helpers/monitoring'; @@ -243,7 +243,7 @@ export async function verify(body): Promise { let strategiesValue: number[] = []; try { - strategiesValue = await getStrategiesValue({ + strategiesValue = await getVpValueByStrategy({ network: space.network, start: msg.payload.start, strategies: space.strategies diff --git a/test/integration/ingestor.test.ts b/test/integration/ingestor.test.ts index e75021c0..9f507078 100644 --- a/test/integration/ingestor.test.ts +++ b/test/integration/ingestor.test.ts @@ -114,7 +114,7 @@ jest.mock('@snapshot-labs/pineapple', () => { jest.mock('../../src/helpers/entityValue', () => ({ __esModule: true, - getStrategiesValue: jest.fn(() => Promise.resolve([])) + getVpValueByStrategy: jest.fn(() => Promise.resolve([])) })); const proposalRequest = { diff --git a/test/integration/writer/proposal.test.ts b/test/integration/writer/proposal.test.ts index 831fdf58..011bd0e4 100644 --- a/test/integration/writer/proposal.test.ts +++ b/test/integration/writer/proposal.test.ts @@ -21,7 +21,7 @@ jest.mock('../../../src/helpers/moderation', () => { jest.mock('../../../src/helpers/entityValue', () => ({ __esModule: true, - getStrategiesValue: jest.fn(() => Promise.resolve([])) + getVpValueByStrategy: jest.fn(() => Promise.resolve([])) })); const getSpaceMock = jest.spyOn(actionHelper, 'getSpace'); diff --git a/test/unit/writer/proposal.test.ts b/test/unit/writer/proposal.test.ts index 8f191ac6..ae698366 100644 --- a/test/unit/writer/proposal.test.ts +++ b/test/unit/writer/proposal.test.ts @@ -100,7 +100,7 @@ jest.mock('../../../src/helpers/moderation', () => { jest.mock('../../../src/helpers/entityValue', () => ({ __esModule: true, - getStrategiesValue: jest.fn(() => Promise.resolve([])) + getVpValueByStrategy: jest.fn(() => Promise.resolve([])) })); const mockGetProposalsCount = jest.spyOn(writer, 'getProposalsCount'); From ee86a37ea9009842b3b6e1c8d36fcd40aaf2f0a2 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 16 Sep 2025 15:39:14 +0400 Subject: [PATCH 70/98] fix: fix invalid method name --- src/helpers/entityValue.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/helpers/entityValue.ts b/src/helpers/entityValue.ts index 757d2d0c..b15662b7 100644 --- a/src/helpers/entityValue.ts +++ b/src/helpers/entityValue.ts @@ -11,7 +11,7 @@ const OVERLORD_URL = process.env.OVERLORD_URL ?? 'https://overlord.snapshot.box' const STRATEGIES_VALUE_PRECISION = 9; export async function getVpValueByStrategy(proposal: Proposal): Promise { - const result: number[] = await jsonRpcRequest(OVERLORD_URL, 'get_vp_value_by_strategy', { + const result: number[] = await jsonRpcRequest(OVERLORD_URL, 'get_value_by_strategy', { network: proposal.network, strategies: proposal.strategies, snapshot: proposal.start From f46a8a1cd4190786b5696a613e84d71ab42e5d69 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 16 Sep 2025 15:44:04 +0400 Subject: [PATCH 71/98] fix: send JSON-RPC id as number --- src/helpers/utils.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/helpers/utils.ts b/src/helpers/utils.ts index bf246eeb..5720d097 100644 --- a/src/helpers/utils.ts +++ b/src/helpers/utils.ts @@ -1,4 +1,4 @@ -import { createHash, randomUUID } from 'crypto'; +import { createHash, randomInt } from 'crypto'; import http from 'http'; import https from 'https'; import { URL } from 'url'; @@ -73,7 +73,7 @@ export async function jsonRpcRequest(url: string, method: string, params: any): jsonrpc: '2.0', method, params, - id: randomUUID() + id: randomInt(10000) }) }; From 57ef9b1db3f4544c2b5c65025cbf77e6f7402451 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 16 Sep 2025 16:42:53 +0400 Subject: [PATCH 72/98] fix: skip vp value fetching for future proposals --- src/helpers/entityValue.ts | 2 +- src/writer/proposal.ts | 23 +++++++++++++---------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/helpers/entityValue.ts b/src/helpers/entityValue.ts index b15662b7..a6274a52 100644 --- a/src/helpers/entityValue.ts +++ b/src/helpers/entityValue.ts @@ -14,7 +14,7 @@ export async function getVpValueByStrategy(proposal: Proposal): Promise { return Promise.reject('wrong proposal format'); } - const tsInt = (Date.now() / 1e3).toFixed(); + const tsInt = +(Date.now() / 1e3).toFixed(); if (msg.payload.end <= tsInt) { return Promise.reject('proposal end date must be in the future'); } @@ -242,15 +242,18 @@ export async function verify(body): Promise { let strategiesValue: number[] = []; - try { - strategiesValue = await getVpValueByStrategy({ - network: space.network, - start: msg.payload.start, - strategies: space.strategies - }); - } catch (e: any) { - log.warn('unable to get strategies value', e.message); - return Promise.reject('failed to get strategies value'); + // Token value are not available yet for future proposals + if (msg.payload.start <= tsInt) { + try { + strategiesValue = await getVpValueByStrategy({ + network: space.network, + start: msg.payload.start, + strategies: space.strategies + }); + } catch (e: any) { + log.warn('unable to get strategies value', e.message); + return Promise.reject('failed to get strategies value'); + } } return { strategiesValue }; From f651c5477139ecc2d8de6386e18c01879413b9ba Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 17 Sep 2025 18:16:00 +0400 Subject: [PATCH 73/98] fix: remove the getValue setting on proposal creation it should be done async, via a separate infinite loop script --- src/constants.ts | 8 ++--- src/writer/proposal.ts | 28 +++-------------- test/integration/ingestor.test.ts | 13 ++++++-- test/integration/writer/proposal.test.ts | 9 ++---- test/unit/writer/proposal.test.ts | 40 ++++++++++++------------ 5 files changed, 38 insertions(+), 60 deletions(-) diff --git a/src/constants.ts b/src/constants.ts index 1d20dfb0..53d57fd5 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,8 +1,4 @@ export const CB = { - FINAL: 1, - PENDING_SYNC: 0, // Default db value, waiting from value from overlord - PENDING_COMPUTE: -1, - PENDING_CLOSE: -2, - INELIGIBLE: -10, // Payload format, can not compute - ERROR_SYNC: -11 // Sync error from overlord, waiting for retry + UNELIGIBLE: -1, + PENDING_SYNC: -10 }; diff --git a/src/writer/proposal.ts b/src/writer/proposal.ts index 4bf48c8c..1eb111fb 100644 --- a/src/writer/proposal.ts +++ b/src/writer/proposal.ts @@ -4,7 +4,6 @@ import networks from '@snapshot-labs/snapshot.js/src/networks.json'; import { uniq } from 'lodash'; import { CB } from '../constants'; import { getPremiumNetworkIds, getSpace } from '../helpers/actions'; -import { getVpValueByStrategy } from '../helpers/entityValue'; import log from '../helpers/log'; import { containsFlaggedLinks, flaggedAddresses } from '../helpers/moderation'; import { isMalicious } from '../helpers/monitoring'; @@ -14,7 +13,6 @@ import { captureError, getQuorum, jsonParse, validateChoices } from '../helpers/ const scoreAPIUrl = process.env.SCORE_API_URL || 'https://score.snapshot.org'; const broviderUrl = process.env.BROVIDER_URL || 'https://rpc.snapshot.org'; -const LAST_CB = parseInt(process.env.LAST_CB ?? '1'); export const getProposalsCount = async (space, author) => { const query = ` @@ -105,7 +103,7 @@ export async function verify(body): Promise { return Promise.reject('wrong proposal format'); } - const tsInt = +(Date.now() / 1e3).toFixed(); + const tsInt = (Date.now() / 1e3).toFixed(); if (msg.payload.end <= tsInt) { return Promise.reject('proposal end date must be in the future'); } @@ -239,27 +237,9 @@ export async function verify(body): Promise { if (msg.payload.choices.length > choicesLimit) { return Promise.reject(`number of choices can not exceed ${choicesLimit}`); } - - let strategiesValue: number[] = []; - - // Token value are not available yet for future proposals - if (msg.payload.start <= tsInt) { - try { - strategiesValue = await getVpValueByStrategy({ - network: space.network, - start: msg.payload.start, - strategies: space.strategies - }); - } catch (e: any) { - log.warn('unable to get strategies value', e.message); - return Promise.reject('failed to get strategies value'); - } - } - - return { strategiesValue }; } -export async function action(body, ipfs, receipt, id, context): Promise { +export async function action(body, ipfs, receipt, id): Promise { const msg = jsonParse(body.msg); const space = msg.space; @@ -318,11 +298,11 @@ export async function action(body, ipfs, receipt, id, context): Promise { scores_total: 0, scores_updated: 0, scores_total_value: 0, - vp_value_by_strategy: JSON.stringify(context.strategiesValue), + vp_value_by_strategy: JSON.stringify([]), votes: 0, validation, flagged: +containsFlaggedLinks(msg.payload.body), - cb: LAST_CB + cb: CB.PENDING_SYNC }; const query = ` diff --git a/test/integration/ingestor.test.ts b/test/integration/ingestor.test.ts index 9f507078..82fff3c9 100644 --- a/test/integration/ingestor.test.ts +++ b/test/integration/ingestor.test.ts @@ -112,9 +112,16 @@ jest.mock('@snapshot-labs/pineapple', () => { }; }); -jest.mock('../../src/helpers/entityValue', () => ({ - __esModule: true, - getVpValueByStrategy: jest.fn(() => Promise.resolve([])) +jest.mock('../../src/helpers/strategies', () => ({ + getStrategies: jest.fn(() => ({ + 'erc20-balance-of': { id: 'erc20-balance-of', override: false, disabled: false }, + 'contract-call': { id: 'contract-call', override: true, disabled: false }, + delegation: { id: 'delegation', override: false, disabled: false }, + whitelist: { id: 'whitelist', override: false, disabled: false } + })), + initialize: jest.fn().mockResolvedValue(undefined), + run: jest.fn(), + stop: jest.fn() })); const proposalRequest = { diff --git a/test/integration/writer/proposal.test.ts b/test/integration/writer/proposal.test.ts index 011bd0e4..80df065c 100644 --- a/test/integration/writer/proposal.test.ts +++ b/test/integration/writer/proposal.test.ts @@ -19,11 +19,6 @@ jest.mock('../../../src/helpers/moderation', () => { }; }); -jest.mock('../../../src/helpers/entityValue', () => ({ - __esModule: true, - getVpValueByStrategy: jest.fn(() => Promise.resolve([])) -})); - const getSpaceMock = jest.spyOn(actionHelper, 'getSpace'); getSpaceMock.mockResolvedValue(spacesGetSpaceFixtures); @@ -50,7 +45,7 @@ describe('writer/proposal', () => { expect.hasAssertions(); mockContainsFlaggedLinks.mockReturnValueOnce(true); const id = '0x01-flagged'; - expect(await action(input, 'ipfs', 'receipt', id, { strategiesValue: [] })).toBeUndefined(); + expect(await action(input, 'ipfs', 'receipt', id)).toBeUndefined(); expect(mockContainsFlaggedLinks).toBeCalledTimes(1); const [proposal] = await db.queryAsync('SELECT * FROM proposals WHERE id = ?', [id]); @@ -62,7 +57,7 @@ describe('writer/proposal', () => { it('creates and does not flag proposal', async () => { expect.hasAssertions(); const id = '0x02-non-flagged'; - expect(await action(input, 'ipfs', 'receipt', id, { strategiesValue: [] })).toBeUndefined(); + expect(await action(input, 'ipfs', 'receipt', id)).toBeUndefined(); expect(mockContainsFlaggedLinks).toBeCalledTimes(1); const [proposal] = await db.queryAsync('SELECT * FROM proposals WHERE id = ?', [id]); diff --git a/test/unit/writer/proposal.test.ts b/test/unit/writer/proposal.test.ts index ae698366..40acbd09 100644 --- a/test/unit/writer/proposal.test.ts +++ b/test/unit/writer/proposal.test.ts @@ -98,10 +98,10 @@ jest.mock('../../../src/helpers/moderation', () => { }; }); -jest.mock('../../../src/helpers/entityValue', () => ({ - __esModule: true, - getVpValueByStrategy: jest.fn(() => Promise.resolve([])) -})); +// Get the mocked function after the mock is created +const { validateSpaceSettings: mockValidateSpaceSettings } = jest.requireMock( + '../../../src/helpers/spaceValidation' +); const mockGetProposalsCount = jest.spyOn(writer, 'getProposalsCount'); mockGetProposalsCount.mockResolvedValue([ @@ -170,7 +170,7 @@ describe('writer/proposal', () => { msg.payload.type = 'basic'; msg.payload.choices = ['For', 'Against', 'Abstain']; - await expect(writer.verify({ ...input, msg: JSON.stringify(msg) })).resolves.toBeDefined(); + await expect(writer.verify({ ...input, msg: JSON.stringify(msg) })).resolves.toBeUndefined(); expect(mockGetSpace).toHaveBeenCalledTimes(1); expect(mockGetProposalsCount).toHaveBeenCalledTimes(1); }); @@ -238,7 +238,7 @@ describe('writer/proposal', () => { voting: { ...spacesGetSpaceFixtures.voting, period: VOTING_PERIOD } }); - await expect(writer.verify(inputWithVotingPeriod)).resolves.toBeDefined(); + await expect(writer.verify(inputWithVotingPeriod)).resolves.toBeUndefined(); expect(mockGetSpace).toHaveBeenCalledTimes(1); expect(mockGetProposalsCount).toHaveBeenCalledTimes(1); }); @@ -270,7 +270,7 @@ describe('writer/proposal', () => { await expect( writer.verify({ ...input, msg: JSON.stringify(msg) }) - ).resolves.toBeDefined(); + ).resolves.toBeUndefined(); expect(mockGetSpace).toHaveBeenCalledTimes(1); expect(mockGetProposalsCount).toHaveBeenCalledTimes(1); }); @@ -335,7 +335,7 @@ describe('writer/proposal', () => { it('does not validate the space validation', async () => { expect.assertions(2); - await expect(writer.verify(input)).resolves.toBeDefined(); + await expect(writer.verify(input)).resolves.toBeUndefined(); expect(mockSnapshotUtilsValidate).toHaveBeenCalledTimes(0); }); }); @@ -344,7 +344,7 @@ describe('writer/proposal', () => { it('does not validate the space validation', async () => { expect.assertions(2); - await expect(writer.verify(input)).resolves.toBeDefined(); + await expect(writer.verify(input)).resolves.toBeUndefined(); expect(mockSnapshotUtilsValidate).toHaveBeenCalledTimes(0); }); }); @@ -360,19 +360,19 @@ describe('writer/proposal', () => { it('accepts a proposal with shutter privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: 'shutter' })) - ).resolves.toBeDefined(); + ).resolves.toBeUndefined(); }); it('accepts a proposal with undefined privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: undefined })) - ).resolves.toBeDefined(); + ).resolves.toBeUndefined(); }); it('accepts a proposal with no privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: '' })) - ).resolves.toBeDefined(); + ).resolves.toBeUndefined(); }); }); @@ -388,19 +388,19 @@ describe('writer/proposal', () => { it('accepts a proposal with shutter privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: 'shutter' })) - ).resolves.toBeDefined(); + ).resolves.toBeUndefined(); }); it('accepts a proposal with undefined privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: undefined })) - ).resolves.toBeDefined(); + ).resolves.toBeUndefined(); }); it('accepts a proposal with no privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: '' })) - ).resolves.toBeDefined(); + ).resolves.toBeUndefined(); }); }); @@ -421,13 +421,13 @@ describe('writer/proposal', () => { it('accepts a proposal with undefined privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: undefined })) - ).resolves.toBeDefined(); + ).resolves.toBeUndefined(); }); it('accepts a proposal with no privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: '' })) - ).resolves.toBeDefined(); + ).resolves.toBeUndefined(); }); }); @@ -442,13 +442,13 @@ describe('writer/proposal', () => { it('accepts a proposal with shutter privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: 'shutter' })) - ).resolves.toBeDefined(); + ).resolves.toBeUndefined(); }); it('accepts a proposal with undefined privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: undefined })) - ).resolves.toBeDefined(); + ).resolves.toBeUndefined(); }); it('rejects a proposal with privacy empty string', async () => { @@ -686,7 +686,7 @@ describe('writer/proposal', () => { it('verifies a valid input', async () => { expect.assertions(1); - await expect(writer.verify(input)).resolves.toBeDefined(); + await expect(writer.verify(input)).resolves.toBeUndefined(); }); }); }); From 487d90d66828e77332da8a0cc48e5b3bc080daf5 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 17 Sep 2025 20:11:37 +0400 Subject: [PATCH 74/98] feat: refresh proposals vp_value_by_strategy async --- src/constants.ts | 4 +++- src/helpers/vpValue.ts | 50 ++++++++++++++++++++++++++++++++++++++++++ src/index.ts | 3 ++- src/writer/vote.ts | 15 ++++++++++++- 4 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 src/helpers/vpValue.ts diff --git a/src/constants.ts b/src/constants.ts index 53d57fd5..81d94da4 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,4 +1,6 @@ export const CB = { - UNELIGIBLE: -1, + INELIGIBLE: -1, PENDING_SYNC: -10 }; + +export const CURRENT_CB = parseInt(process.env.LAST_CB ?? '1'); diff --git a/src/helpers/vpValue.ts b/src/helpers/vpValue.ts new file mode 100644 index 00000000..cde0aa80 --- /dev/null +++ b/src/helpers/vpValue.ts @@ -0,0 +1,50 @@ +import { capture } from '@snapshot-labs/snapshot-sentry'; +import snapshot from '@snapshot-labs/snapshot.js'; +import { getVpValueByStrategy } from './entityValue'; +import db from './mysql'; +import { CB, CURRENT_CB } from '../constants'; + +type Proposal = { + id: string; + network: string; + start: number; + strategies: any[]; +}; + +const REFRESH_INTERVAL = 60 * 1000; + +async function getProposals(): Promise { + const query = + 'SELECT id, network, start, strategies FROM proposals WHERE cb = ? AND start < UNIX_TIMESTAMP() LIMIT 500'; + const proposals = await db.queryAsync(query, [CB.PENDING_SYNC]); + + return proposals.map((p: any) => { + p.strategies = JSON.parse(p.strategies); + return p; + }); +} + +async function refreshProposalVpValues(proposal: Proposal) { + try { + const values = await getVpValueByStrategy(proposal); + const query = 'UPDATE proposals SET vp_value_by_strategy = ?, cb = ? WHERE id = ? LIMIT 1'; + await db.queryAsync(query, [JSON.stringify(values), CURRENT_CB, proposal.id]); + } catch (e: any) { + capture(e); + } +} + +async function refreshProposals() { + const proposals = await getProposals(); + + for (const proposal of proposals) { + await refreshProposalVpValues(proposal); + } +} + +export default async function run() { + await refreshProposals(); + await snapshot.utils.sleep(REFRESH_INTERVAL); + + run(); +} diff --git a/src/index.ts b/src/index.ts index 3cf9d8f2..294cbbd2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -15,13 +15,14 @@ import { stop as stopStrategies } from './helpers/strategies'; import { trackTurboStatuses } from './helpers/turbo'; +import refreshVpValue from './helpers/vpValue'; const app = express(); async function startServer() { initLogger(app); refreshModeration(); - refreshProposalsVpValue(); + refreshVpValue(); await initializeStrategies(); refreshStrategies(); diff --git a/src/writer/vote.ts b/src/writer/vote.ts index d11b948f..d047a6f7 100644 --- a/src/writer/vote.ts +++ b/src/writer/vote.ts @@ -1,5 +1,5 @@ import snapshot from '@snapshot-labs/snapshot.js'; -import { CB } from '../constants'; +import { CURRENT_CB } from '../constants'; import { getProposal } from '../helpers/actions'; import log from '../helpers/log'; import db from '../helpers/mysql'; @@ -117,6 +117,19 @@ export async function action(body, ipfs, receipt, id, context): Promise { const withOverride = hasStrategyOverride(context.proposal.strategies); if (vpState === 'final' && withOverride) vpState = 'pending'; + // Get proposal voting power value + // Value is set on creation, and not updated on vote update + // Value will be recomputed on proposal close + let vpValue = 0; + let cb = 0; + + try { + vpValue = getVoteValue(context.proposal, context.vp); + cb = CURRENT_CB; + } catch (e: any) { + capture(e, { msg, proposalId, context }); + } + const params = { id, ipfs, From 555bb9f37dbf82d85cdf512c09de00645c34ca56 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Fri, 19 Sep 2025 20:26:46 +0400 Subject: [PATCH 75/98] fix: skip votes vp value computation when proposal score value is not ready yet --- src/writer/vote.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/writer/vote.ts b/src/writer/vote.ts index d047a6f7..0387278a 100644 --- a/src/writer/vote.ts +++ b/src/writer/vote.ts @@ -1,5 +1,5 @@ import snapshot from '@snapshot-labs/snapshot.js'; -import { CURRENT_CB } from '../constants'; +import { CB, CURRENT_CB } from '../constants'; import { getProposal } from '../helpers/actions'; import log from '../helpers/log'; import db from '../helpers/mysql'; @@ -121,13 +121,15 @@ export async function action(body, ipfs, receipt, id, context): Promise { // Value is set on creation, and not updated on vote update // Value will be recomputed on proposal close let vpValue = 0; - let cb = 0; + let cb = CB.PENDING_SYNC; - try { - vpValue = getVoteValue(context.proposal, context.vp); - cb = CURRENT_CB; - } catch (e: any) { - capture(e, { msg, proposalId, context }); + if (context.proposal.cb >= 0) { + try { + vpValue = getVoteValue(context.proposal, context.vp); + cb = CURRENT_CB; + } catch (e: any) { + capture(e, { msg, proposalId, context }); + } } const params = { From 3bd4e96060186ca8aeb1c085ac7e4af2c847b3c6 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Mon, 29 Sep 2025 16:20:30 +0400 Subject: [PATCH 76/98] fix: use only loop script to handle overlord logic --- src/constants.ts | 3 ++- src/helpers/vpValue.ts | 46 ++++++++++++++++++++++++++++++------------ src/writer/vote.ts | 19 ++--------------- 3 files changed, 37 insertions(+), 31 deletions(-) diff --git a/src/constants.ts b/src/constants.ts index 81d94da4..4ca6bf15 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,6 +1,7 @@ export const CB = { INELIGIBLE: -1, - PENDING_SYNC: -10 + PENDING_SYNC: -10, // waiting for value from overlord + PENDING_CLOSE: -20 // value from overlord set, waiting for proposal close for final computation }; export const CURRENT_CB = parseInt(process.env.LAST_CB ?? '1'); diff --git a/src/helpers/vpValue.ts b/src/helpers/vpValue.ts index cde0aa80..8dc26868 100644 --- a/src/helpers/vpValue.ts +++ b/src/helpers/vpValue.ts @@ -1,8 +1,8 @@ -import { capture } from '@snapshot-labs/snapshot-sentry'; +// import { capture } from '@snapshot-labs/snapshot-sentry'; import snapshot from '@snapshot-labs/snapshot.js'; import { getVpValueByStrategy } from './entityValue'; import db from './mysql'; -import { CB, CURRENT_CB } from '../constants'; +import { CB } from '../constants'; type Proposal = { id: string; @@ -12,11 +12,12 @@ type Proposal = { }; const REFRESH_INTERVAL = 60 * 1000; +const BATCH_SIZE = 100; async function getProposals(): Promise { const query = - 'SELECT id, network, start, strategies FROM proposals WHERE cb = ? AND start < UNIX_TIMESTAMP() LIMIT 500'; - const proposals = await db.queryAsync(query, [CB.PENDING_SYNC]); + 'SELECT id, network, start, strategies FROM proposals WHERE cb = ? AND start < UNIX_TIMESTAMP() LIMIT ?'; + const proposals = await db.queryAsync(query, [CB.PENDING_SYNC, BATCH_SIZE]); return proposals.map((p: any) => { p.strategies = JSON.parse(p.strategies); @@ -24,26 +25,45 @@ async function getProposals(): Promise { }); } -async function refreshProposalVpValues(proposal: Proposal) { +async function refreshProposalsVpValues(proposals: Proposal[]) { + const query: string[] = []; + const params: any[] = []; + + for (const proposal of proposals) { + await buildQuery(proposal, query, params); + } + + if (query.length) { + await db.queryAsync(query.join(';'), params); + } +} + +async function buildQuery(proposal: Proposal, query: string[], params: any[]) { try { const values = await getVpValueByStrategy(proposal); - const query = 'UPDATE proposals SET vp_value_by_strategy = ?, cb = ? WHERE id = ? LIMIT 1'; - await db.queryAsync(query, [JSON.stringify(values), CURRENT_CB, proposal.id]); - } catch (e: any) { - capture(e); + + query.push('UPDATE proposals SET vp_value_by_strategy = ?, cb = ? WHERE id = ? LIMIT 1'); + params.push(JSON.stringify(values), CB.PENDING_CLOSE, proposal.id); + } catch (e) { + // TODO: enable only after whole database is synced + // capture(e, { extra: { proposal } }); } } -async function refreshProposals() { +async function refreshPendingProposals() { const proposals = await getProposals(); - for (const proposal of proposals) { - await refreshProposalVpValues(proposal); + while (true) { + if (proposals.length === 0) break; + + await refreshProposalsVpValues(proposals); + + if (proposals.length < BATCH_SIZE) break; } } export default async function run() { - await refreshProposals(); + await refreshPendingProposals(); await snapshot.utils.sleep(REFRESH_INTERVAL); run(); diff --git a/src/writer/vote.ts b/src/writer/vote.ts index 0387278a..f3f1532c 100644 --- a/src/writer/vote.ts +++ b/src/writer/vote.ts @@ -1,5 +1,5 @@ import snapshot from '@snapshot-labs/snapshot.js'; -import { CB, CURRENT_CB } from '../constants'; +import { CB } from '../constants'; import { getProposal } from '../helpers/actions'; import log from '../helpers/log'; import db from '../helpers/mysql'; @@ -117,21 +117,6 @@ export async function action(body, ipfs, receipt, id, context): Promise { const withOverride = hasStrategyOverride(context.proposal.strategies); if (vpState === 'final' && withOverride) vpState = 'pending'; - // Get proposal voting power value - // Value is set on creation, and not updated on vote update - // Value will be recomputed on proposal close - let vpValue = 0; - let cb = CB.PENDING_SYNC; - - if (context.proposal.cb >= 0) { - try { - vpValue = getVoteValue(context.proposal, context.vp); - cb = CURRENT_CB; - } catch (e: any) { - capture(e, { msg, proposalId, context }); - } - } - const params = { id, ipfs, @@ -147,7 +132,7 @@ export async function action(body, ipfs, receipt, id, context): Promise { vp_by_strategy: JSON.stringify(context.vp.vp_by_strategy), vp_state: vpState, vp_value: 0, - cb: CB.PENDING_COMPUTE + cb: CB.PENDING_SYNC }; // Check if voter already voted From 33ffe9107a10a5c737a71d339e979202b42d37e2 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Mon, 29 Sep 2025 19:34:12 +0400 Subject: [PATCH 77/98] fix: process by most recent first --- src/helpers/vpValue.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/helpers/vpValue.ts b/src/helpers/vpValue.ts index 8dc26868..61593829 100644 --- a/src/helpers/vpValue.ts +++ b/src/helpers/vpValue.ts @@ -16,7 +16,7 @@ const BATCH_SIZE = 100; async function getProposals(): Promise { const query = - 'SELECT id, network, start, strategies FROM proposals WHERE cb = ? AND start < UNIX_TIMESTAMP() LIMIT ?'; + 'SELECT id, network, start, strategies FROM proposals WHERE cb = ? AND start < UNIX_TIMESTAMP() ORDER BY created DESC LIMIT ?'; const proposals = await db.queryAsync(query, [CB.PENDING_SYNC, BATCH_SIZE]); return proposals.map((p: any) => { From ce652ab8d958876d3c2f60e39b09a96fac423652 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Mon, 29 Sep 2025 20:36:47 +0400 Subject: [PATCH 78/98] refactor: better function name --- src/helpers/proposalStrategiesValue.ts | 66 ++++++++++++------------ src/helpers/vpValue.ts | 70 -------------------------- src/index.ts | 3 +- 3 files changed, 34 insertions(+), 105 deletions(-) delete mode 100644 src/helpers/vpValue.ts diff --git a/src/helpers/proposalStrategiesValue.ts b/src/helpers/proposalStrategiesValue.ts index 96bfc219..61593829 100644 --- a/src/helpers/proposalStrategiesValue.ts +++ b/src/helpers/proposalStrategiesValue.ts @@ -1,3 +1,4 @@ +// import { capture } from '@snapshot-labs/snapshot-sentry'; import snapshot from '@snapshot-labs/snapshot.js'; import { getVpValueByStrategy } from './entityValue'; import db from './mysql'; @@ -10,17 +11,12 @@ type Proposal = { strategies: any[]; }; -const REFRESH_INTERVAL = 10 * 1000; -const BATCH_SIZE = 25; +const REFRESH_INTERVAL = 60 * 1000; +const BATCH_SIZE = 100; async function getProposals(): Promise { - const query = ` - SELECT id, network, start, strategies - FROM proposals - WHERE cb = ? AND start < UNIX_TIMESTAMP() - ORDER BY created ASC - LIMIT ? - `; + const query = + 'SELECT id, network, start, strategies FROM proposals WHERE cb = ? AND start < UNIX_TIMESTAMP() ORDER BY created DESC LIMIT ?'; const proposals = await db.queryAsync(query, [CB.PENDING_SYNC, BATCH_SIZE]); return proposals.map((p: any) => { @@ -29,25 +25,12 @@ async function getProposals(): Promise { }); } -async function refreshVpByStrategy(proposals: Proposal[]) { +async function refreshProposalsVpValues(proposals: Proposal[]) { const query: string[] = []; const params: any[] = []; - const results = await Promise.all( - proposals.map(async proposal => { - try { - const values = await getVpValueByStrategy(proposal); - return { proposal, values, cb: CB.PENDING_COMPUTE }; - } catch (e) { - console.log(e); - return { proposal, values: [], cb: CB.ERROR_SYNC }; - } - }) - ); - - for (const result of results) { - query.push('UPDATE proposals SET vp_value_by_strategy = ?, cb = ? WHERE id = ? LIMIT 1'); - params.push(JSON.stringify(result.values), result.cb, result.proposal.id); + for (const proposal of proposals) { + await buildQuery(proposal, query, params); } if (query.length) { @@ -55,16 +38,33 @@ async function refreshVpByStrategy(proposals: Proposal[]) { } } -export default async function run() { +async function buildQuery(proposal: Proposal, query: string[], params: any[]) { + try { + const values = await getVpValueByStrategy(proposal); + + query.push('UPDATE proposals SET vp_value_by_strategy = ?, cb = ? WHERE id = ? LIMIT 1'); + params.push(JSON.stringify(values), CB.PENDING_CLOSE, proposal.id); + } catch (e) { + // TODO: enable only after whole database is synced + // capture(e, { extra: { proposal } }); + } +} + +async function refreshPendingProposals() { + const proposals = await getProposals(); + while (true) { - const proposals = await getProposals(); + if (proposals.length === 0) break; - if (proposals.length) { - await refreshVpByStrategy(proposals); - } + await refreshProposalsVpValues(proposals); - if (proposals.length < BATCH_SIZE) { - await snapshot.utils.sleep(REFRESH_INTERVAL); - } + if (proposals.length < BATCH_SIZE) break; } } + +export default async function run() { + await refreshPendingProposals(); + await snapshot.utils.sleep(REFRESH_INTERVAL); + + run(); +} diff --git a/src/helpers/vpValue.ts b/src/helpers/vpValue.ts deleted file mode 100644 index 61593829..00000000 --- a/src/helpers/vpValue.ts +++ /dev/null @@ -1,70 +0,0 @@ -// import { capture } from '@snapshot-labs/snapshot-sentry'; -import snapshot from '@snapshot-labs/snapshot.js'; -import { getVpValueByStrategy } from './entityValue'; -import db from './mysql'; -import { CB } from '../constants'; - -type Proposal = { - id: string; - network: string; - start: number; - strategies: any[]; -}; - -const REFRESH_INTERVAL = 60 * 1000; -const BATCH_SIZE = 100; - -async function getProposals(): Promise { - const query = - 'SELECT id, network, start, strategies FROM proposals WHERE cb = ? AND start < UNIX_TIMESTAMP() ORDER BY created DESC LIMIT ?'; - const proposals = await db.queryAsync(query, [CB.PENDING_SYNC, BATCH_SIZE]); - - return proposals.map((p: any) => { - p.strategies = JSON.parse(p.strategies); - return p; - }); -} - -async function refreshProposalsVpValues(proposals: Proposal[]) { - const query: string[] = []; - const params: any[] = []; - - for (const proposal of proposals) { - await buildQuery(proposal, query, params); - } - - if (query.length) { - await db.queryAsync(query.join(';'), params); - } -} - -async function buildQuery(proposal: Proposal, query: string[], params: any[]) { - try { - const values = await getVpValueByStrategy(proposal); - - query.push('UPDATE proposals SET vp_value_by_strategy = ?, cb = ? WHERE id = ? LIMIT 1'); - params.push(JSON.stringify(values), CB.PENDING_CLOSE, proposal.id); - } catch (e) { - // TODO: enable only after whole database is synced - // capture(e, { extra: { proposal } }); - } -} - -async function refreshPendingProposals() { - const proposals = await getProposals(); - - while (true) { - if (proposals.length === 0) break; - - await refreshProposalsVpValues(proposals); - - if (proposals.length < BATCH_SIZE) break; - } -} - -export default async function run() { - await refreshPendingProposals(); - await snapshot.utils.sleep(REFRESH_INTERVAL); - - run(); -} diff --git a/src/index.ts b/src/index.ts index 294cbbd2..3cf9d8f2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -15,14 +15,13 @@ import { stop as stopStrategies } from './helpers/strategies'; import { trackTurboStatuses } from './helpers/turbo'; -import refreshVpValue from './helpers/vpValue'; const app = express(); async function startServer() { initLogger(app); refreshModeration(); - refreshVpValue(); + refreshProposalsVpValue(); await initializeStrategies(); refreshStrategies(); From 3f2a49ee0c5e21a87c0aa0b27c81cea1911b1359 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Mon, 29 Sep 2025 21:18:17 +0400 Subject: [PATCH 79/98] feat: add script to refresh votes' vp_value async --- src/helpers/entityValue.ts | 11 ++-- src/helpers/votesVpValue.ts | 75 +++++++++++++++++++++++++++ src/index.ts | 2 + test/unit/helpers/entityValue.test.ts | 61 +++++++++++++--------- 4 files changed, 118 insertions(+), 31 deletions(-) create mode 100644 src/helpers/votesVpValue.ts diff --git a/src/helpers/entityValue.ts b/src/helpers/entityValue.ts index a6274a52..0f354afe 100644 --- a/src/helpers/entityValue.ts +++ b/src/helpers/entityValue.ts @@ -29,15 +29,12 @@ export async function getVpValueByStrategy(proposal: Proposal): Promise sum + value * vote.vp_by_strategy[index], - 0 - ); + return vp_value_by_strategy.reduce((sum, value, index) => sum + value * vp_by_strategy[index], 0); } diff --git a/src/helpers/votesVpValue.ts b/src/helpers/votesVpValue.ts new file mode 100644 index 00000000..531632a9 --- /dev/null +++ b/src/helpers/votesVpValue.ts @@ -0,0 +1,75 @@ +// import { capture } from '@snapshot-labs/snapshot-sentry'; +import snapshot from '@snapshot-labs/snapshot.js'; +import { getVoteValue } from './entityValue'; +import db from './mysql'; +import { CB } from '../constants'; + +const REFRESH_INTERVAL = 60 * 1000; +const BATCH_SIZE = 100; + +type Datum = { + id: string; + vp_by_strategy: number[]; + vp_value_by_strategy: number[]; +}; + +async function getVotes(): Promise { + const query = ` + SELECT votes.id, votes.vp_by_strategy, proposals.vp_value_by_strategy + FROM votes + JOIN proposals ON votes.proposal = proposals.id + WHERE proposals.cb = ? AND votes.cb = ? + ORDER BY votes.created DESC + LIMIT ?`; + const results = await db.queryAsync(query, [CB.PENDING_CLOSE, CB.PENDING_SYNC, BATCH_SIZE]); + + return results.map((p: any) => { + p.vp_value_by_strategy = JSON.parse(p.vp_value_by_strategy); + p.vp_by_strategy = JSON.parse(p.vp_by_strategy); + return p; + }); +} + +async function refreshVotesVpValues(data: Datum[]) { + const query: string[] = []; + const params: any[] = []; + + for (const datum of data) { + buildQuery(datum, query, params); + } + + if (query.length) { + await db.queryAsync(query.join(';'), params); + } +} + +function buildQuery(datum: Datum, query: string[], params: any[]) { + try { + const value = getVoteValue(datum.vp_value_by_strategy, datum.vp_by_strategy); + + query.push('UPDATE votes SET vp_value = ?, cb = ? WHERE id = ? LIMIT 1'); + params.push(value, CB.PENDING_CLOSE, datum.id); + } catch (e) { + // TODO: enable only after whole database is synced + // capture(e, { extra: { proposal } }); + } +} + +async function refreshPendingVotes() { + while (true) { + const votes = await getVotes(); + + if (votes.length === 0) break; + + await refreshVotesVpValues(votes); + + if (votes.length < BATCH_SIZE) break; + } +} + +export default async function run() { + await refreshPendingVotes(); + await snapshot.utils.sleep(REFRESH_INTERVAL); + + run(); +} diff --git a/src/index.ts b/src/index.ts index 3cf9d8f2..a082bbee 100644 --- a/src/index.ts +++ b/src/index.ts @@ -15,6 +15,7 @@ import { stop as stopStrategies } from './helpers/strategies'; import { trackTurboStatuses } from './helpers/turbo'; +import refreshVotesVpValue from './helpers/votesVpValue'; const app = express(); @@ -22,6 +23,7 @@ async function startServer() { initLogger(app); refreshModeration(); refreshProposalsVpValue(); + refreshVotesVpValue(); await initializeStrategies(); refreshStrategies(); diff --git a/test/unit/helpers/entityValue.test.ts b/test/unit/helpers/entityValue.test.ts index 7dba57b1..fc02cd3c 100644 --- a/test/unit/helpers/entityValue.test.ts +++ b/test/unit/helpers/entityValue.test.ts @@ -2,69 +2,82 @@ import { getVoteValue } from '../../../src/helpers/entityValue'; describe('getVoteValue', () => { it('should calculate correct vote value with single strategy', () => { - const proposal = { vp_value_by_strategy: [2.5] }; - const vote = { vp_by_strategy: [100] }; + const vp_value_by_strategy = [2.5]; + const vp_by_strategy = [100]; - const result = getVoteValue(proposal, vote); + const result = getVoteValue(vp_value_by_strategy, vp_by_strategy); expect(result).toBe(250); }); it('should calculate correct vote value with multiple strategies', () => { - const proposal = { vp_value_by_strategy: [1.5, 3.0, 0.5] }; - const vote = { vp_by_strategy: [100, 50, 200] }; + const vp_value_by_strategy = [1.5, 3.0, 0.5]; + const vp_by_strategy = [100, 50, 200]; - const result = getVoteValue(proposal, vote); + const result = getVoteValue(vp_value_by_strategy, vp_by_strategy); expect(result).toBe(400); // (1.5 * 100) + (3.0 * 50) + (0.5 * 200) = 150 + 150 + 100 = 400 }); it('should return 0 when vote has no voting power', () => { - const proposal = { vp_value_by_strategy: [2.0, 1.5] }; - const vote = { vp_by_strategy: [0, 0] }; + const vp_value_by_strategy = [2.0, 1.5]; + const vp_by_strategy = [0, 0]; - const result = getVoteValue(proposal, vote); + const result = getVoteValue(vp_value_by_strategy, vp_by_strategy); expect(result).toBe(0); }); it('should return 0 when proposal has no value per strategy', () => { - const proposal = { vp_value_by_strategy: [0, 0] }; - const vote = { vp_by_strategy: [100, 50] }; + const vp_value_by_strategy = [0, 0]; + const vp_by_strategy = [100, 50]; - const result = getVoteValue(proposal, vote); + const result = getVoteValue(vp_value_by_strategy, vp_by_strategy); expect(result).toBe(0); }); it('should handle decimal values correctly', () => { - const proposal = { vp_value_by_strategy: [0.1, 0.25] }; - const vote = { vp_by_strategy: [10, 20] }; + const vp_value_by_strategy = [0.1, 0.25]; + const vp_by_strategy = [10, 20]; - const result = getVoteValue(proposal, vote); + const result = getVoteValue(vp_value_by_strategy, vp_by_strategy); expect(result).toBe(6); // (0.1 * 10) + (0.25 * 20) = 1 + 5 = 6 }); it('should throw error when strategy arrays have different lengths', () => { - const proposal = { vp_value_by_strategy: [1.0, 2.0] }; - const vote = { vp_by_strategy: [100] }; + const vp_value_by_strategy = [1.0, 2.0]; + const vp_by_strategy = [100]; - expect(() => getVoteValue(proposal, vote)).toThrow('invalid data to compute vote value'); + expect(() => getVoteValue(vp_value_by_strategy, vp_by_strategy)).toThrow( + 'invalid data to compute vote value' + ); }); it('should throw error when vote has more strategies than proposal', () => { - const proposal = { vp_value_by_strategy: [1.0] }; - const vote = { vp_by_strategy: [100, 50] }; + const vp_value_by_strategy = [1.0]; + const vp_by_strategy = [100, 50]; - expect(() => getVoteValue(proposal, vote)).toThrow('invalid data to compute vote value'); + expect(() => getVoteValue(vp_value_by_strategy, vp_by_strategy)).toThrow( + 'invalid data to compute vote value' + ); }); it('should handle empty arrays', () => { - const proposal = { vp_value_by_strategy: [] }; - const vote = { vp_by_strategy: [] }; + const vp_value_by_strategy = []; + const vp_by_strategy = []; - const result = getVoteValue(proposal, vote); + const result = getVoteValue(vp_value_by_strategy, vp_by_strategy); + + expect(result).toBe(0); + }); + + it('should return 0 when vp_value_by_strategy is empty', () => { + const vp_value_by_strategy = []; + const vp_by_strategy = [100, 50]; + + const result = getVoteValue(vp_value_by_strategy, vp_by_strategy); expect(result).toBe(0); }); From f08703edcf71c539a664832d5963db7a147691be Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 30 Sep 2025 00:11:08 +0400 Subject: [PATCH 80/98] fix: on new votes using overriding strategies, mark all votes to be resynced --- src/scores.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/scores.ts b/src/scores.ts index ec8c2d59..1ac5697e 100644 --- a/src/scores.ts +++ b/src/scores.ts @@ -1,4 +1,5 @@ import snapshot from '@snapshot-labs/snapshot.js'; +import { CB } from './constants'; import { getVoteValue } from './helpers/entityValue'; import log from './helpers/log'; import db from './helpers/mysql'; @@ -61,12 +62,13 @@ async function updateVotesVp(votes: any[], vpState: string, proposalId: string) let query = ''; votesInPage.forEach((vote: any) => { query += `UPDATE votes - SET vp = ?, vp_by_strategy = ?, vp_state = ?, vp_value = ? + SET vp = ?, vp_by_strategy = ?, vp_state = ?, vp_value = ?, cb = ? WHERE id = ? AND proposal = ? LIMIT 1; `; params.push(vote.balance); params.push(JSON.stringify(vote.scores)); params.push(vpState); params.push(vote.vp_value); + params.push(CB.PENDING_SYNC); params.push(vote.id); params.push(proposalId); }); From d8801456706a2e3b8e5dc0fb1fa3250b943752b1 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Mon, 29 Sep 2025 21:30:13 +0400 Subject: [PATCH 81/98] fix: fix loop using same data --- src/helpers/proposalStrategiesValue.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/helpers/proposalStrategiesValue.ts b/src/helpers/proposalStrategiesValue.ts index 61593829..ef574b08 100644 --- a/src/helpers/proposalStrategiesValue.ts +++ b/src/helpers/proposalStrategiesValue.ts @@ -51,9 +51,9 @@ async function buildQuery(proposal: Proposal, query: string[], params: any[]) { } async function refreshPendingProposals() { - const proposals = await getProposals(); - while (true) { + const proposals = await getProposals(); + if (proposals.length === 0) break; await refreshProposalsVpValues(proposals); From 68e6f59a266eff07892321186812bc8965374cd1 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 30 Sep 2025 00:49:06 +0400 Subject: [PATCH 82/98] fix: add new CB value to trigger other fields computation --- src/constants.ts | 1 + src/helpers/proposalStrategiesValue.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/constants.ts b/src/constants.ts index 4ca6bf15..cf594b9b 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,6 +1,7 @@ export const CB = { INELIGIBLE: -1, PENDING_SYNC: -10, // waiting for value from overlord + PENDING_COMPUTE: -15, // value from overlord set, waiting for local computation or refresh PENDING_CLOSE: -20 // value from overlord set, waiting for proposal close for final computation }; diff --git a/src/helpers/proposalStrategiesValue.ts b/src/helpers/proposalStrategiesValue.ts index ef574b08..3b14eddd 100644 --- a/src/helpers/proposalStrategiesValue.ts +++ b/src/helpers/proposalStrategiesValue.ts @@ -43,7 +43,7 @@ async function buildQuery(proposal: Proposal, query: string[], params: any[]) { const values = await getVpValueByStrategy(proposal); query.push('UPDATE proposals SET vp_value_by_strategy = ?, cb = ? WHERE id = ? LIMIT 1'); - params.push(JSON.stringify(values), CB.PENDING_CLOSE, proposal.id); + params.push(JSON.stringify(values), CB.PENDING_COMPUTE, proposal.id); } catch (e) { // TODO: enable only after whole database is synced // capture(e, { extra: { proposal } }); From 4f0e151613ada0da9d275b1c8809503f280a7dd8 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 30 Sep 2025 00:55:05 +0400 Subject: [PATCH 83/98] fix: use dedicated CB when value need refresh --- src/helpers/votesVpValue.ts | 8 ++++++-- src/scores.ts | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/helpers/votesVpValue.ts b/src/helpers/votesVpValue.ts index 531632a9..1e938443 100644 --- a/src/helpers/votesVpValue.ts +++ b/src/helpers/votesVpValue.ts @@ -18,10 +18,14 @@ async function getVotes(): Promise { SELECT votes.id, votes.vp_by_strategy, proposals.vp_value_by_strategy FROM votes JOIN proposals ON votes.proposal = proposals.id - WHERE proposals.cb = ? AND votes.cb = ? + WHERE proposals.cb IN (?) AND votes.cb = ? ORDER BY votes.created DESC LIMIT ?`; - const results = await db.queryAsync(query, [CB.PENDING_CLOSE, CB.PENDING_SYNC, BATCH_SIZE]); + const results = await db.queryAsync(query, [ + [CB.PENDING_CLOSE, CB.PENDING_COMPUTE], + CB.PENDING_SYNC, + BATCH_SIZE + ]); return results.map((p: any) => { p.vp_value_by_strategy = JSON.parse(p.vp_value_by_strategy); diff --git a/src/scores.ts b/src/scores.ts index 1ac5697e..3b23d889 100644 --- a/src/scores.ts +++ b/src/scores.ts @@ -68,7 +68,7 @@ async function updateVotesVp(votes: any[], vpState: string, proposalId: string) params.push(JSON.stringify(vote.scores)); params.push(vpState); params.push(vote.vp_value); - params.push(CB.PENDING_SYNC); + params.push(CB.PENDING_COMPUTE); params.push(vote.id); params.push(proposalId); }); From 96aff861f26929fc539d7a7429dd86459d221b01 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 30 Sep 2025 01:19:02 +0400 Subject: [PATCH 84/98] fix: finalize votes CB --- src/helpers/votesVpValue.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/helpers/votesVpValue.ts b/src/helpers/votesVpValue.ts index 1e938443..01da8014 100644 --- a/src/helpers/votesVpValue.ts +++ b/src/helpers/votesVpValue.ts @@ -2,28 +2,29 @@ import snapshot from '@snapshot-labs/snapshot.js'; import { getVoteValue } from './entityValue'; import db from './mysql'; -import { CB } from '../constants'; +import { CB, CURRENT_CB } from '../constants'; const REFRESH_INTERVAL = 60 * 1000; const BATCH_SIZE = 100; type Datum = { id: string; + vp_state: string; vp_by_strategy: number[]; vp_value_by_strategy: number[]; }; async function getVotes(): Promise { const query = ` - SELECT votes.id, votes.vp_by_strategy, proposals.vp_value_by_strategy + SELECT votes.id, votes.vp_state, votes.vp_by_strategy, proposals.vp_value_by_strategy FROM votes JOIN proposals ON votes.proposal = proposals.id - WHERE proposals.cb IN (?) AND votes.cb = ? + WHERE proposals.cb IN (?) AND votes.cb IN (?) ORDER BY votes.created DESC LIMIT ?`; const results = await db.queryAsync(query, [ [CB.PENDING_CLOSE, CB.PENDING_COMPUTE], - CB.PENDING_SYNC, + [CB.PENDING_SYNC, CB.PENDING_COMPUTE], BATCH_SIZE ]); @@ -52,7 +53,7 @@ function buildQuery(datum: Datum, query: string[], params: any[]) { const value = getVoteValue(datum.vp_value_by_strategy, datum.vp_by_strategy); query.push('UPDATE votes SET vp_value = ?, cb = ? WHERE id = ? LIMIT 1'); - params.push(value, CB.PENDING_CLOSE, datum.id); + params.push(value, datum.vp_state === 'final' ? CURRENT_CB : CB.PENDING_CLOSE, datum.id); } catch (e) { // TODO: enable only after whole database is synced // capture(e, { extra: { proposal } }); From 7f8f3b96f9fb72272530b12ab7af9bde481da710 Mon Sep 17 00:00:00 2001 From: Wan <495709+wa0x6e@users.noreply.github.com> Date: Tue, 30 Sep 2025 21:16:53 +0900 Subject: [PATCH 85/98] Update src/helpers/proposalStrategiesValue.ts Co-authored-by: Less --- src/helpers/proposalStrategiesValue.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/helpers/proposalStrategiesValue.ts b/src/helpers/proposalStrategiesValue.ts index 3b14eddd..b72a3435 100644 --- a/src/helpers/proposalStrategiesValue.ts +++ b/src/helpers/proposalStrategiesValue.ts @@ -11,7 +11,7 @@ type Proposal = { strategies: any[]; }; -const REFRESH_INTERVAL = 60 * 1000; +const REFRESH_INTERVAL = 10 * 1000; const BATCH_SIZE = 100; async function getProposals(): Promise { From 93f6792a78a7a89011e4e54ebe2c4fe0e15decdc Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 30 Sep 2025 16:21:38 +0400 Subject: [PATCH 86/98] fix: use static CB value for closed status --- src/constants.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/constants.ts b/src/constants.ts index cf594b9b..897b69ff 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -2,7 +2,6 @@ export const CB = { INELIGIBLE: -1, PENDING_SYNC: -10, // waiting for value from overlord PENDING_COMPUTE: -15, // value from overlord set, waiting for local computation or refresh - PENDING_CLOSE: -20 // value from overlord set, waiting for proposal close for final computation + PENDING_CLOSE: -20, // value from overlord set, waiting for proposal close for final computation + CLOSED: 1 }; - -export const CURRENT_CB = parseInt(process.env.LAST_CB ?? '1'); From 7607541c366f4dde9a9a3c0289342abb06ff3059 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 30 Sep 2025 16:30:16 +0400 Subject: [PATCH 87/98] refactoring: code improvement --- src/helpers/proposalStrategiesValue.ts | 42 +++++++++++++------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/helpers/proposalStrategiesValue.ts b/src/helpers/proposalStrategiesValue.ts index b72a3435..900b5ad0 100644 --- a/src/helpers/proposalStrategiesValue.ts +++ b/src/helpers/proposalStrategiesValue.ts @@ -15,8 +15,13 @@ const REFRESH_INTERVAL = 10 * 1000; const BATCH_SIZE = 100; async function getProposals(): Promise { - const query = - 'SELECT id, network, start, strategies FROM proposals WHERE cb = ? AND start < UNIX_TIMESTAMP() ORDER BY created DESC LIMIT ?'; + const query = ` + SELECT id, network, start, strategies + FROM proposals + WHERE cb = ? AND start < UNIX_TIMESTAMP() + ORDER BY created DESC + LIMIT ? + `; const proposals = await db.queryAsync(query, [CB.PENDING_SYNC, BATCH_SIZE]); return proposals.map((p: any) => { @@ -25,12 +30,22 @@ async function getProposals(): Promise { }); } -async function refreshProposalsVpValues(proposals: Proposal[]) { +async function refreshVpByStrategy(proposals: Proposal[]) { const query: string[] = []; const params: any[] = []; for (const proposal of proposals) { - await buildQuery(proposal, query, params); + try { + const values = await getVpValueByStrategy(proposal); + + query.push('UPDATE proposals SET vp_value_by_strategy = ?, cb = ? WHERE id = ? LIMIT 1'); + params.push(JSON.stringify(values), CB.PENDING_COMPUTE, proposal.id); + } catch (e) { + // TODO: switch to capture only after whole database is synced + // to avoid quota issues + // capture(e, { extra: { proposal } }); + console.log(e); + } } if (query.length) { @@ -38,32 +53,17 @@ async function refreshProposalsVpValues(proposals: Proposal[]) { } } -async function buildQuery(proposal: Proposal, query: string[], params: any[]) { - try { - const values = await getVpValueByStrategy(proposal); - - query.push('UPDATE proposals SET vp_value_by_strategy = ?, cb = ? WHERE id = ? LIMIT 1'); - params.push(JSON.stringify(values), CB.PENDING_COMPUTE, proposal.id); - } catch (e) { - // TODO: enable only after whole database is synced - // capture(e, { extra: { proposal } }); - } -} - -async function refreshPendingProposals() { +export default async function run() { while (true) { const proposals = await getProposals(); if (proposals.length === 0) break; - await refreshProposalsVpValues(proposals); + await refreshVpByStrategy(proposals); if (proposals.length < BATCH_SIZE) break; } -} -export default async function run() { - await refreshPendingProposals(); await snapshot.utils.sleep(REFRESH_INTERVAL); run(); From 610e24ee91c06afab17acf974cbdd7a1b38f43a4 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 30 Sep 2025 16:35:30 +0400 Subject: [PATCH 88/98] fix: better constant name --- src/constants.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/constants.ts b/src/constants.ts index 897b69ff..e22d63e2 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -3,5 +3,5 @@ export const CB = { PENDING_SYNC: -10, // waiting for value from overlord PENDING_COMPUTE: -15, // value from overlord set, waiting for local computation or refresh PENDING_CLOSE: -20, // value from overlord set, waiting for proposal close for final computation - CLOSED: 1 + FINAL: 1 }; From 4059f3bf7f2558bcb3ae858f50693214bf3765b3 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 30 Sep 2025 16:55:33 +0400 Subject: [PATCH 89/98] fix: process older proposals first --- src/helpers/proposalStrategiesValue.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/helpers/proposalStrategiesValue.ts b/src/helpers/proposalStrategiesValue.ts index 900b5ad0..a1f10a94 100644 --- a/src/helpers/proposalStrategiesValue.ts +++ b/src/helpers/proposalStrategiesValue.ts @@ -12,14 +12,14 @@ type Proposal = { }; const REFRESH_INTERVAL = 10 * 1000; -const BATCH_SIZE = 100; +const BATCH_SIZE = 10; async function getProposals(): Promise { const query = ` SELECT id, network, start, strategies FROM proposals WHERE cb = ? AND start < UNIX_TIMESTAMP() - ORDER BY created DESC + ORDER BY created ASC LIMIT ? `; const proposals = await db.queryAsync(query, [CB.PENDING_SYNC, BATCH_SIZE]); From db63fcbdc17fea4696af917c92ed763a4a28de1c Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 30 Sep 2025 17:12:41 +0400 Subject: [PATCH 90/98] fix: send request in batch --- src/constants.ts | 1 + src/helpers/proposalStrategiesValue.ts | 27 ++++++++++++++------------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/constants.ts b/src/constants.ts index e22d63e2..b37e7f51 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,5 +1,6 @@ export const CB = { INELIGIBLE: -1, + SYNC_ERROR: -2, // Sync error from overlord PENDING_SYNC: -10, // waiting for value from overlord PENDING_COMPUTE: -15, // value from overlord set, waiting for local computation or refresh PENDING_CLOSE: -20, // value from overlord set, waiting for proposal close for final computation diff --git a/src/helpers/proposalStrategiesValue.ts b/src/helpers/proposalStrategiesValue.ts index a1f10a94..724524ae 100644 --- a/src/helpers/proposalStrategiesValue.ts +++ b/src/helpers/proposalStrategiesValue.ts @@ -12,7 +12,7 @@ type Proposal = { }; const REFRESH_INTERVAL = 10 * 1000; -const BATCH_SIZE = 10; +const BATCH_SIZE = 25; async function getProposals(): Promise { const query = ` @@ -34,18 +34,21 @@ async function refreshVpByStrategy(proposals: Proposal[]) { const query: string[] = []; const params: any[] = []; - for (const proposal of proposals) { - try { - const values = await getVpValueByStrategy(proposal); + const results = await Promise.all( + proposals.map(async proposal => { + try { + const values = await getVpValueByStrategy(proposal); + return { proposal, values, cb: CB.PENDING_COMPUTE }; + } catch (e) { + console.log(e); + return { proposal, values: [], cb: CB.SYNC_ERROR }; + } + }) + ); - query.push('UPDATE proposals SET vp_value_by_strategy = ?, cb = ? WHERE id = ? LIMIT 1'); - params.push(JSON.stringify(values), CB.PENDING_COMPUTE, proposal.id); - } catch (e) { - // TODO: switch to capture only after whole database is synced - // to avoid quota issues - // capture(e, { extra: { proposal } }); - console.log(e); - } + for (const result of results) { + query.push('UPDATE proposals SET vp_value_by_strategy = ?, cb = ? WHERE id = ? LIMIT 1'); + params.push(JSON.stringify(result.values), result.cb, result.proposal.id); } if (query.length) { From 7d9fda2b92c4822be87328b1df9600be89f255c7 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Tue, 30 Sep 2025 20:48:32 +0400 Subject: [PATCH 91/98] fix: update CB values --- src/constants.ts | 12 ++++++------ src/helpers/proposalStrategiesValue.ts | 2 +- src/writer/vote.ts | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/constants.ts b/src/constants.ts index b37e7f51..1d20dfb0 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,8 +1,8 @@ export const CB = { - INELIGIBLE: -1, - SYNC_ERROR: -2, // Sync error from overlord - PENDING_SYNC: -10, // waiting for value from overlord - PENDING_COMPUTE: -15, // value from overlord set, waiting for local computation or refresh - PENDING_CLOSE: -20, // value from overlord set, waiting for proposal close for final computation - FINAL: 1 + FINAL: 1, + PENDING_SYNC: 0, // Default db value, waiting from value from overlord + PENDING_COMPUTE: -1, + PENDING_CLOSE: -2, + INELIGIBLE: -10, // Payload format, can not compute + ERROR_SYNC: -11 // Sync error from overlord, waiting for retry }; diff --git a/src/helpers/proposalStrategiesValue.ts b/src/helpers/proposalStrategiesValue.ts index 724524ae..0ef2d2b7 100644 --- a/src/helpers/proposalStrategiesValue.ts +++ b/src/helpers/proposalStrategiesValue.ts @@ -41,7 +41,7 @@ async function refreshVpByStrategy(proposals: Proposal[]) { return { proposal, values, cb: CB.PENDING_COMPUTE }; } catch (e) { console.log(e); - return { proposal, values: [], cb: CB.SYNC_ERROR }; + return { proposal, values: [], cb: CB.ERROR_SYNC }; } }) ); diff --git a/src/writer/vote.ts b/src/writer/vote.ts index f3f1532c..d11b948f 100644 --- a/src/writer/vote.ts +++ b/src/writer/vote.ts @@ -132,7 +132,7 @@ export async function action(body, ipfs, receipt, id, context): Promise { vp_by_strategy: JSON.stringify(context.vp.vp_by_strategy), vp_state: vpState, vp_value: 0, - cb: CB.PENDING_SYNC + cb: CB.PENDING_COMPUTE }; // Check if voter already voted From b2707a361ee4d9362ce185434c629960c52c4581 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 1 Oct 2025 01:38:07 +0400 Subject: [PATCH 92/98] fix: update to match proposal script convention --- src/helpers/votesVpValue.ts | 43 +++++++++++++++---------------------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/src/helpers/votesVpValue.ts b/src/helpers/votesVpValue.ts index 01da8014..8f03e9fd 100644 --- a/src/helpers/votesVpValue.ts +++ b/src/helpers/votesVpValue.ts @@ -2,9 +2,9 @@ import snapshot from '@snapshot-labs/snapshot.js'; import { getVoteValue } from './entityValue'; import db from './mysql'; -import { CB, CURRENT_CB } from '../constants'; +import { CB } from '../constants'; -const REFRESH_INTERVAL = 60 * 1000; +const REFRESH_INTERVAL = 10 * 1000; const BATCH_SIZE = 100; type Datum = { @@ -20,18 +20,18 @@ async function getVotes(): Promise { FROM votes JOIN proposals ON votes.proposal = proposals.id WHERE proposals.cb IN (?) AND votes.cb IN (?) - ORDER BY votes.created DESC + ORDER BY votes.created ASC LIMIT ?`; const results = await db.queryAsync(query, [ - [CB.PENDING_CLOSE, CB.PENDING_COMPUTE], + [CB.PENDING_CLOSE, CB.PENDING_COMPUTE, CB.FINAL], [CB.PENDING_SYNC, CB.PENDING_COMPUTE], BATCH_SIZE ]); - return results.map((p: any) => { - p.vp_value_by_strategy = JSON.parse(p.vp_value_by_strategy); - p.vp_by_strategy = JSON.parse(p.vp_by_strategy); - return p; + return results.map((r: any) => { + r.vp_value_by_strategy = JSON.parse(r.vp_value_by_strategy); + r.vp_by_strategy = JSON.parse(r.vp_by_strategy); + return r; }); } @@ -40,7 +40,14 @@ async function refreshVotesVpValues(data: Datum[]) { const params: any[] = []; for (const datum of data) { - buildQuery(datum, query, params); + try { + const value = getVoteValue(datum.vp_value_by_strategy, datum.vp_by_strategy); + + query.push('UPDATE votes SET vp_value = ?, cb = ? WHERE id = ? LIMIT 1'); + params.push(value, datum.vp_state === 'final' ? CB.FINAL : CB.PENDING_CLOSE, datum.id); + } catch (e) { + console.log(e); + } } if (query.length) { @@ -48,19 +55,7 @@ async function refreshVotesVpValues(data: Datum[]) { } } -function buildQuery(datum: Datum, query: string[], params: any[]) { - try { - const value = getVoteValue(datum.vp_value_by_strategy, datum.vp_by_strategy); - - query.push('UPDATE votes SET vp_value = ?, cb = ? WHERE id = ? LIMIT 1'); - params.push(value, datum.vp_state === 'final' ? CURRENT_CB : CB.PENDING_CLOSE, datum.id); - } catch (e) { - // TODO: enable only after whole database is synced - // capture(e, { extra: { proposal } }); - } -} - -async function refreshPendingVotes() { +export default async function run() { while (true) { const votes = await getVotes(); @@ -70,10 +65,6 @@ async function refreshPendingVotes() { if (votes.length < BATCH_SIZE) break; } -} - -export default async function run() { - await refreshPendingVotes(); await snapshot.utils.sleep(REFRESH_INTERVAL); run(); From 53751bcc12fa8fbc9baf7f9eaf9760f28e1ea917 Mon Sep 17 00:00:00 2001 From: Wan <495709+wa0x6e@users.noreply.github.com> Date: Wed, 1 Oct 2025 06:42:53 +0900 Subject: [PATCH 93/98] Update src/helpers/votesVpValue.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/helpers/votesVpValue.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/helpers/votesVpValue.ts b/src/helpers/votesVpValue.ts index 8f03e9fd..c79c7cf0 100644 --- a/src/helpers/votesVpValue.ts +++ b/src/helpers/votesVpValue.ts @@ -57,15 +57,15 @@ async function refreshVotesVpValues(data: Datum[]) { export default async function run() { while (true) { - const votes = await getVotes(); + while (true) { + const votes = await getVotes(); - if (votes.length === 0) break; + if (votes.length === 0) break; - await refreshVotesVpValues(votes); + await refreshVotesVpValues(votes); - if (votes.length < BATCH_SIZE) break; + if (votes.length < BATCH_SIZE) break; + } + await snapshot.utils.sleep(REFRESH_INTERVAL); } - await snapshot.utils.sleep(REFRESH_INTERVAL); - - run(); } From 0849f02b3a2bd1c37eafaba62033e42a7416d275 Mon Sep 17 00:00:00 2001 From: Wan <495709+wa0x6e@users.noreply.github.com> Date: Wed, 1 Oct 2025 06:42:22 +0900 Subject: [PATCH 94/98] Update src/helpers/proposalStrategiesValue.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/helpers/proposalStrategiesValue.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/helpers/proposalStrategiesValue.ts b/src/helpers/proposalStrategiesValue.ts index 0ef2d2b7..c66d9666 100644 --- a/src/helpers/proposalStrategiesValue.ts +++ b/src/helpers/proposalStrategiesValue.ts @@ -58,16 +58,16 @@ async function refreshVpByStrategy(proposals: Proposal[]) { export default async function run() { while (true) { - const proposals = await getProposals(); + while (true) { + const proposals = await getProposals(); - if (proposals.length === 0) break; + if (proposals.length === 0) break; - await refreshVpByStrategy(proposals); + await refreshVpByStrategy(proposals); - if (proposals.length < BATCH_SIZE) break; - } - - await snapshot.utils.sleep(REFRESH_INTERVAL); + if (proposals.length < BATCH_SIZE) break; + } - run(); + await snapshot.utils.sleep(REFRESH_INTERVAL); + } } From 51a3e6b45e7cdbb741ad2119da36574a8b25f0e0 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 1 Oct 2025 01:54:32 +0400 Subject: [PATCH 95/98] chore: remove unused import --- src/helpers/votesVpValue.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/helpers/votesVpValue.ts b/src/helpers/votesVpValue.ts index c79c7cf0..9fbec673 100644 --- a/src/helpers/votesVpValue.ts +++ b/src/helpers/votesVpValue.ts @@ -1,4 +1,3 @@ -// import { capture } from '@snapshot-labs/snapshot-sentry'; import snapshot from '@snapshot-labs/snapshot.js'; import { getVoteValue } from './entityValue'; import db from './mysql'; From 81a1c9781ad7627e48c503181a713a6b6a9775fe Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 1 Oct 2025 02:46:13 +0400 Subject: [PATCH 96/98] feat: compute proposal's scores_total_value --- src/helpers/entityValue.ts | 47 ++++++++++++++++++++ src/helpers/proposalsScoresValue.ts | 69 +++++++++++++++++++++++++++++ src/index.ts | 2 + 3 files changed, 118 insertions(+) create mode 100644 src/helpers/proposalsScoresValue.ts diff --git a/src/helpers/entityValue.ts b/src/helpers/entityValue.ts index 0f354afe..e1f113e8 100644 --- a/src/helpers/entityValue.ts +++ b/src/helpers/entityValue.ts @@ -38,3 +38,50 @@ export function getVoteValue(vp_value_by_strategy: number[], vp_by_strategy: num return vp_value_by_strategy.reduce((sum, value, index) => sum + value * vp_by_strategy[index], 0); } + +/** + * Calculates the proposal total value based on all votes' total voting power and the proposal's value per strategy. + * @returns The total value of the given proposal's votes, in the currency unit specified by the proposal's vp_value_by_strategy values + */ +export function getProposalValue( + scores_by_strategy: number[][], + vp_value_by_strategy: number[] +): number { + if ( + !scores_by_strategy.length || + !scores_by_strategy[0]?.length || + !vp_value_by_strategy.length + ) { + return 0; + } + + // Validate that all voteScores arrays have the same length as vp_value_by_strategy + for (const voteScores of scores_by_strategy) { + if (voteScores.length !== vp_value_by_strategy.length) { + throw new Error( + 'Array size mismatch: voteScores length does not match vp_value_by_strategy length' + ); + } + } + + let totalValue = 0; + for (let strategyIndex = 0; strategyIndex < vp_value_by_strategy.length; strategyIndex++) { + const strategyTotal = scores_by_strategy.reduce((sum, voteScores) => { + const score = voteScores[strategyIndex]; + if (typeof score !== 'number') { + throw new Error(`Invalid score value: expected number, got ${typeof score}`); + } + return sum + score; + }, 0); + + if (typeof vp_value_by_strategy[strategyIndex] !== 'number') { + throw new Error( + `Invalid vp_value: expected number, got ${typeof vp_value_by_strategy[strategyIndex]}` + ); + } + + totalValue += strategyTotal * vp_value_by_strategy[strategyIndex]; + } + + return totalValue; +} diff --git a/src/helpers/proposalsScoresValue.ts b/src/helpers/proposalsScoresValue.ts new file mode 100644 index 00000000..c65ef99b --- /dev/null +++ b/src/helpers/proposalsScoresValue.ts @@ -0,0 +1,69 @@ +import { capture } from '@snapshot-labs/snapshot-sentry'; +import snapshot from '@snapshot-labs/snapshot.js'; +import { getProposalValue } from './entityValue'; +import db from './mysql'; +import { CB } from '../constants'; + +type Proposal = { + id: string; + vp_value_by_strategy: number[]; + scores_by_strategy: number[][]; +}; + +const REFRESH_INTERVAL = 10 * 1000; +const BATCH_SIZE = 25; + +async function getProposals(): Promise { + const query = ` + SELECT id, vp_value_by_strategy, scores_by_strategy + FROM proposals + WHERE cb = ? AND end < UNIX_TIMESTAMP() AND scores_state = ? + ORDER BY created ASC + LIMIT ? + `; + const proposals = await db.queryAsync(query, [CB.PENDING_CLOSE, 'final', BATCH_SIZE]); + + return proposals.map((p: any) => { + p.vp_value_by_strategy = JSON.parse(p.vp_value_by_strategy); + p.scores_by_strategy = JSON.parse(p.scores_by_strategy); + return p; + }); +} + +async function refreshScoresTotal(proposals: Proposal[]) { + const query: string[] = []; + const params: any[] = []; + + proposals.map(proposal => { + query.push('UPDATE proposals SET scores_total_value = ?, cb = ? WHERE id = ? LIMIT 1'); + + try { + const scores_total_value = getProposalValue( + proposal.scores_by_strategy, + proposal.vp_value_by_strategy + ); + params.push(scores_total_value, CB.FINAL, proposal.id); + } catch (e) { + capture(e); + params.push(0, CB.INELIGIBLE, proposal.id); + } + }); + + if (query.length) { + await db.queryAsync(query.join(';'), params); + } +} + +export default async function run() { + while (true) { + const proposals = await getProposals(); + + if (proposals.length) { + await refreshScoresTotal(proposals); + } + + if (proposals.length < BATCH_SIZE) { + await snapshot.utils.sleep(REFRESH_INTERVAL); + } + } +} diff --git a/src/index.ts b/src/index.ts index a082bbee..bfbaeeec 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,6 +6,7 @@ import api from './api'; import log from './helpers/log'; import initMetrics from './helpers/metrics'; import refreshModeration from './helpers/moderation'; +import refreshProposalsScoresValue from './helpers/proposalsScoresValue'; import refreshProposalsVpValue from './helpers/proposalStrategiesValue'; import rateLimit from './helpers/rateLimit'; import shutter from './helpers/shutter'; @@ -23,6 +24,7 @@ async function startServer() { initLogger(app); refreshModeration(); refreshProposalsVpValue(); + refreshProposalsScoresValue(); refreshVotesVpValue(); await initializeStrategies(); From 0d900d3cc99d82f2efee06e667f621d784518881 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 1 Oct 2025 03:29:17 +0400 Subject: [PATCH 97/98] fix: fix remnant from merge conflict --- src/writer/proposal.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/writer/proposal.ts b/src/writer/proposal.ts index 1eb111fb..db3a7b2b 100644 --- a/src/writer/proposal.ts +++ b/src/writer/proposal.ts @@ -9,6 +9,7 @@ import { containsFlaggedLinks, flaggedAddresses } from '../helpers/moderation'; import { isMalicious } from '../helpers/monitoring'; import db from '../helpers/mysql'; import { getLimits, getSpaceType } from '../helpers/options'; +import { validateSpaceSettings } from '../helpers/spaceValidation'; import { captureError, getQuorum, jsonParse, validateChoices } from '../helpers/utils'; const scoreAPIUrl = process.env.SCORE_API_URL || 'https://score.snapshot.org'; From c7b2d03b4ec2026152bc3e61c767a47468791711 Mon Sep 17 00:00:00 2001 From: wa0x6e <495709+wa0x6e@users.noreply.github.com> Date: Wed, 1 Oct 2025 16:32:31 +0400 Subject: [PATCH 98/98] refactor: use camelCase for variable name --- src/helpers/entityValue.ts | 24 ++++++++++-------------- src/helpers/proposalsScoresValue.ts | 16 ++++++++-------- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/src/helpers/entityValue.ts b/src/helpers/entityValue.ts index e1f113e8..a81209e4 100644 --- a/src/helpers/entityValue.ts +++ b/src/helpers/entityValue.ts @@ -44,20 +44,16 @@ export function getVoteValue(vp_value_by_strategy: number[], vp_by_strategy: num * @returns The total value of the given proposal's votes, in the currency unit specified by the proposal's vp_value_by_strategy values */ export function getProposalValue( - scores_by_strategy: number[][], - vp_value_by_strategy: number[] + scoresByStrategy: number[][], + vpValueByStrategy: number[] ): number { - if ( - !scores_by_strategy.length || - !scores_by_strategy[0]?.length || - !vp_value_by_strategy.length - ) { + if (!scoresByStrategy.length || !scoresByStrategy[0]?.length || !vpValueByStrategy.length) { return 0; } // Validate that all voteScores arrays have the same length as vp_value_by_strategy - for (const voteScores of scores_by_strategy) { - if (voteScores.length !== vp_value_by_strategy.length) { + for (const voteScores of scoresByStrategy) { + if (voteScores.length !== vpValueByStrategy.length) { throw new Error( 'Array size mismatch: voteScores length does not match vp_value_by_strategy length' ); @@ -65,8 +61,8 @@ export function getProposalValue( } let totalValue = 0; - for (let strategyIndex = 0; strategyIndex < vp_value_by_strategy.length; strategyIndex++) { - const strategyTotal = scores_by_strategy.reduce((sum, voteScores) => { + for (let strategyIndex = 0; strategyIndex < vpValueByStrategy.length; strategyIndex++) { + const strategyTotal = scoresByStrategy.reduce((sum, voteScores) => { const score = voteScores[strategyIndex]; if (typeof score !== 'number') { throw new Error(`Invalid score value: expected number, got ${typeof score}`); @@ -74,13 +70,13 @@ export function getProposalValue( return sum + score; }, 0); - if (typeof vp_value_by_strategy[strategyIndex] !== 'number') { + if (typeof vpValueByStrategy[strategyIndex] !== 'number') { throw new Error( - `Invalid vp_value: expected number, got ${typeof vp_value_by_strategy[strategyIndex]}` + `Invalid vp_value: expected number, got ${typeof vpValueByStrategy[strategyIndex]}` ); } - totalValue += strategyTotal * vp_value_by_strategy[strategyIndex]; + totalValue += strategyTotal * vpValueByStrategy[strategyIndex]; } return totalValue; diff --git a/src/helpers/proposalsScoresValue.ts b/src/helpers/proposalsScoresValue.ts index c65ef99b..c98f3aa3 100644 --- a/src/helpers/proposalsScoresValue.ts +++ b/src/helpers/proposalsScoresValue.ts @@ -6,8 +6,8 @@ import { CB } from '../constants'; type Proposal = { id: string; - vp_value_by_strategy: number[]; - scores_by_strategy: number[][]; + vpValueByStrategy: number[]; + scoresByStrategy: number[][]; }; const REFRESH_INTERVAL = 10 * 1000; @@ -24,8 +24,8 @@ async function getProposals(): Promise { const proposals = await db.queryAsync(query, [CB.PENDING_CLOSE, 'final', BATCH_SIZE]); return proposals.map((p: any) => { - p.vp_value_by_strategy = JSON.parse(p.vp_value_by_strategy); - p.scores_by_strategy = JSON.parse(p.scores_by_strategy); + p.scoresByStrategy = JSON.parse(p.vp_value_by_strategy); + p.vpValueByStrategy = JSON.parse(p.scores_by_strategy); return p; }); } @@ -38,11 +38,11 @@ async function refreshScoresTotal(proposals: Proposal[]) { query.push('UPDATE proposals SET scores_total_value = ?, cb = ? WHERE id = ? LIMIT 1'); try { - const scores_total_value = getProposalValue( - proposal.scores_by_strategy, - proposal.vp_value_by_strategy + const scoresTotalValue = getProposalValue( + proposal.scoresByStrategy, + proposal.vpValueByStrategy ); - params.push(scores_total_value, CB.FINAL, proposal.id); + params.push(scoresTotalValue, CB.FINAL, proposal.id); } catch (e) { capture(e); params.push(0, CB.INELIGIBLE, proposal.id);