From f97e238b24c305c4c3831bf4890b1949d4f803e9 Mon Sep 17 00:00:00 2001 From: Wan Qi Chen <495709+wa0x6e@users.noreply.github.com> Date: Mon, 4 Nov 2024 14:21:31 +0900 Subject: [PATCH 01/11] feat: connect to envelop database --- .env.example | 1 + src/helpers/mysql.ts | 18 ++++++++++++++++-- test/.env.test | 1 + test/schema_envelop.sql | 11 +++++++++++ test/setupDb.ts | 15 +++++++++------ 5 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 test/schema_envelop.sql diff --git a/.env.example b/.env.example index 7db4f7fb..14ecb3aa 100644 --- a/.env.example +++ b/.env.example @@ -3,6 +3,7 @@ NETWORK=testnet MAINTENANCE= HUB_DATABASE_URL=mysql://... SEQ_DATABASE_URL=mysql://... +ENVELOP_DATABASE_URL=mysql:// RELAYER_PK=0x123... DEFAULT_NETWORK=1 SHUTTER_URL=https://... diff --git a/src/helpers/mysql.ts b/src/helpers/mysql.ts index a4f2a1c5..5c0f81f2 100644 --- a/src/helpers/mysql.ts +++ b/src/helpers/mysql.ts @@ -33,7 +33,21 @@ sequencerConfig.connectTimeout = 60e3; sequencerConfig.acquireTimeout = 60e3; sequencerConfig.timeout = 60e3; sequencerConfig.charset = 'utf8mb4'; -bluebird.promisifyAll([Pool, Connection]); const sequencerDB = mysql.createPool(sequencerConfig); -export { hubDB as default, sequencerDB }; +// @ts-ignore +const envelopConfig = parse(process.env.ENVELOP_DATABASE_URL); +envelopConfig.connectionLimit = connectionLimit; +envelopConfig.multipleStatements = true; +envelopConfig.database = envelopConfig.path[0]; +envelopConfig.host = envelopConfig.hosts[0].name; +envelopConfig.port = envelopConfig.hosts[0].port; +envelopConfig.connectTimeout = 60e3; +envelopConfig.acquireTimeout = 60e3; +envelopConfig.timeout = 60e3; +envelopConfig.charset = 'utf8mb4'; +const envelopDB = mysql.createPool(envelopConfig); + +bluebird.promisifyAll([Pool, Connection]); + +export { hubDB as default, sequencerDB, envelopDB }; diff --git a/test/.env.test b/test/.env.test index e8bb7e75..776f296b 100644 --- a/test/.env.test +++ b/test/.env.test @@ -1,5 +1,6 @@ HUB_DATABASE_URL=mysql://root:root@127.0.0.1:3306/snapshot_sequencer_test SEQ_DATABASE_URL=mysql://root:root@127.0.0.1:3306/snapshot_sequencer_test +ENVELOP_DATABASE_URL=mysql://root:root@127.0.0.1:3306/snapshot_sequencer_test NETWORK=mainnet RELAYER_PK=01686849e86499c1860ea0afc97f29c11018cbac049abf843df875c60054076e NODE_ENV=test diff --git a/test/schema_envelop.sql b/test/schema_envelop.sql new file mode 100644 index 00000000..8e67b7ab --- /dev/null +++ b/test/schema_envelop.sql @@ -0,0 +1,11 @@ +CREATE TABLE subscribers ( + email VARCHAR(256) NOT NULL, + address VARCHAR(256) NOT NULL, + subscriptions JSON DEFAULT NULL, + created BIGINT NOT NULL, + verified BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (email, address), + UNIQUE KEY idx_address_email (address, email), + INDEX created (created), + INDEX verified (verified) +); diff --git a/test/setupDb.ts b/test/setupDb.ts index b2aff5e1..bbd37184 100755 --- a/test/setupDb.ts +++ b/test/setupDb.ts @@ -1,9 +1,9 @@ -import mysql from 'mysql'; -import Pool from 'mysql/lib/Pool'; -import Connection from 'mysql/lib/Connection'; +import fs from 'fs'; import bluebird from 'bluebird'; import parse from 'connection-string'; -import fs from 'fs'; +import mysql from 'mysql'; +import Connection from 'mysql/lib/Connection'; +import Pool from 'mysql/lib/Pool'; // @ts-ignore const config = parse(process.env.HUB_DATABASE_URL); @@ -19,6 +19,8 @@ if (!dbName.endsWith('_test')) { process.exit(1); } +const schemaFiles = ['./test/schema.sql', './test/schema_envelop.sql']; + async function run() { const splitToken = ');'; @@ -30,8 +32,9 @@ async function run() { console.info(`- Creating new database: ${dbName}`); await db.queryAsync(`CREATE DATABASE ${dbName}`); - const schema = fs - .readFileSync('./test/schema.sql', 'utf8') + const schema = schemaFiles + .map(file => fs.readFileSync(file, 'utf8')) + .join(' ') .replaceAll('CREATE TABLE ', `CREATE TABLE ${dbName}.`) .split(splitToken) .filter(s => s.trim().length > 0); From fb49cc27669f98ea731b95bb5017f81778eee64c Mon Sep 17 00:00:00 2001 From: Wan Qi Chen <495709+wa0x6e@users.noreply.github.com> Date: Tue, 12 Nov 2024 22:02:24 +0900 Subject: [PATCH 02/11] feat: add delete-subscription writer --- src/writer/delete-subscription.ts | 22 ++++++ src/writer/index.ts | 2 + .../writer/delete-subscription.test.ts | 71 +++++++++++++++++++ 3 files changed, 95 insertions(+) create mode 100644 src/writer/delete-subscription.ts create mode 100644 test/integration/writer/delete-subscription.test.ts diff --git a/src/writer/delete-subscription.ts b/src/writer/delete-subscription.ts new file mode 100644 index 00000000..76e89d51 --- /dev/null +++ b/src/writer/delete-subscription.ts @@ -0,0 +1,22 @@ +import { envelopDB } from '../helpers/mysql'; + +type Message = { address: string }; + +export async function verify(message: Message): Promise { + const result = await envelopDB.queryAsync( + `SELECT * FROM subscribers WHERE address = ? AND verified > 0 LIMIT 1`, + [message.address] + ); + + if (!result[0]) { + return Promise.reject('user not subscribed'); + } + + return result[0]; +} + +export async function action(message: Message): Promise { + await envelopDB.queryAsync(`DELETE FROM subscribers WHERE address = ? AND verified > 0 LIMIT 1`, [ + message.address + ]); +} diff --git a/src/writer/index.ts b/src/writer/index.ts index 96be0b62..05ce9712 100644 --- a/src/writer/index.ts +++ b/src/writer/index.ts @@ -1,6 +1,7 @@ import * as alias from './alias'; import * as deleteProposal from './delete-proposal'; import * as deleteSpace from './delete-space'; +import * as deleteSubscription from './delete-subscription'; import * as flagProposal from './flag-proposal'; import * as follow from './follow'; import * as profile from './profile'; @@ -25,6 +26,7 @@ export default { unfollow, subscribe, unsubscribe, + 'delete-subscription': deleteSubscription, alias, profile, statement diff --git a/test/integration/writer/delete-subscription.test.ts b/test/integration/writer/delete-subscription.test.ts new file mode 100644 index 00000000..822dc21b --- /dev/null +++ b/test/integration/writer/delete-subscription.test.ts @@ -0,0 +1,71 @@ +import db, { envelopDB, sequencerDB } from '../../../src/helpers/mysql'; +import { action, verify } from '../../../src/writer/delete-subscription'; + +describe('writer/delete-subscription', () => { + const TEST_PREFIX = 'test-delete-subscription'; + + afterAll(async () => { + await envelopDB.queryAsync('DELETE FROM subscribers'); + await envelopDB.endAsync(); + await db.endAsync(); + await sequencerDB.endAsync(); + }); + + describe('verify()', () => { + beforeAll(async () => { + await Promise.all([ + envelopDB.queryAsync( + 'INSERT INTO subscribers SET address = ?, email = ?, subscriptions = ?, created = ?, verified = ?', + [`${TEST_PREFIX}-0x0`, 'test@snapshot.org', '[]', 0, 0] + ), + envelopDB.queryAsync( + 'INSERT INTO subscribers SET address = ?, email = ?, subscriptions = ?, created = ?, verified = ?', + [`${TEST_PREFIX}-0x1`, 'test1@snapshot.org', '[]', 0, 1] + ) + ]); + }); + + it('rejects when the address is not subscribed', () => { + return expect(verify({ address: '0x0' })).rejects.toEqual(`user not subscribed`); + }); + + it('rejects when the address is not verified', () => { + return expect(verify({ address: `${TEST_PREFIX}-0x0` })).rejects.toEqual( + `user not subscribed` + ); + }); + + it('resolves when the address is verified', () => { + return expect(verify({ address: `${TEST_PREFIX}-0x1` })).resolves.toHaveProperty('address'); + }); + }); + + describe('action()', () => { + const address = `${TEST_PREFIX}-0x3`; + + beforeAll(async () => { + await Promise.all([ + envelopDB.queryAsync( + 'INSERT INTO subscribers SET address = ?, email = ?, subscriptions = ?, created = ?, verified = ?', + [address, 'test@snapshot.org', '[]', 0, 0] + ), + envelopDB.queryAsync( + 'INSERT INTO subscribers SET address = ?, email = ?, subscriptions = ?, created = ?, verified = ?', + [address, 'test1@snapshot.org', '[]', 0, 1] + ) + ]); + }); + + it('deletes the subscription', async () => { + await action({ address: address }); + + const results = await envelopDB.queryAsync('SELECT * FROM subscribers WHERE address = ?', [ + address + ]); + + // Only delete the verified subscription + expect(results.length).toBe(1); + expect(results[0].email).toEqual('test@snapshot.org'); + }); + }); +}); From 48da32593b7a317e9620d783872c53755f229f29 Mon Sep 17 00:00:00 2001 From: Wan Qi Chen <495709+wa0x6e@users.noreply.github.com> Date: Tue, 12 Nov 2024 22:44:51 +0900 Subject: [PATCH 03/11] feat: add update-subscription writer --- src/writer/index.ts | 2 + src/writer/update-subscription.ts | 54 +++++++++++++ .../writer/update-subscription.test.ts | 81 +++++++++++++++++++ 3 files changed, 137 insertions(+) create mode 100644 src/writer/update-subscription.ts create mode 100644 test/integration/writer/update-subscription.test.ts diff --git a/src/writer/index.ts b/src/writer/index.ts index 05ce9712..b1ee52ad 100644 --- a/src/writer/index.ts +++ b/src/writer/index.ts @@ -12,6 +12,7 @@ import * as subscribe from './subscribe'; import * as unfollow from './unfollow'; import * as unsubscribe from './unsubscribe'; import * as updateProposal from './update-proposal'; +import * as updateSubscription from './update-subscription'; import * as vote from './vote'; export default { @@ -26,6 +27,7 @@ export default { unfollow, subscribe, unsubscribe, + 'update-subscription': updateSubscription, 'delete-subscription': deleteSubscription, alias, profile, diff --git a/src/writer/update-subscription.ts b/src/writer/update-subscription.ts new file mode 100644 index 00000000..98fc6bc7 --- /dev/null +++ b/src/writer/update-subscription.ts @@ -0,0 +1,54 @@ +import { envelopDB } from '../helpers/mysql'; +import { jsonParse } from '../helpers/utils'; + +type Message = { msg: string; address: string }; +type Payload = { + type: string; + value: string; + metadata: { subscriptions: string[] }; +}; + +const VALID_SUBSCRIPTIONS = ['summary', 'newProposal', 'closedProposal']; + +function extractPayload(message: Message): Payload { + const payload = jsonParse(message.msg).payload; + + return { + ...payload, + metadata: jsonParse(payload.metadata) + }; +} + +export async function verify(message: Message): Promise { + const result = await envelopDB.queryAsync(`SELECT * FROM subscribers WHERE address = ? LIMIT 1`, [ + message.address + ]); + + if (!result[0]) { + return Promise.reject('user not subscribed'); + } + + if (!result[0].verified) { + return Promise.reject('email not verified'); + } + + const { + metadata: { subscriptions } + } = extractPayload(message); + if ((subscriptions || []).some(s => !VALID_SUBSCRIPTIONS.includes(s))) { + return Promise.reject('invalid subscription value'); + } + + return result[0]; +} + +export async function action(message: Message): Promise { + const { + metadata: { subscriptions } + } = extractPayload(message); + + await envelopDB.queryAsync( + `UPDATE subscribers SET subscriptions = ? WHERE address = ? AND verified > 0 LIMIT 1`, + [JSON.stringify(subscriptions), message.address] + ); +} diff --git a/test/integration/writer/update-subscription.test.ts b/test/integration/writer/update-subscription.test.ts new file mode 100644 index 00000000..c880f811 --- /dev/null +++ b/test/integration/writer/update-subscription.test.ts @@ -0,0 +1,81 @@ +import db, { envelopDB, sequencerDB } from '../../../src/helpers/mysql'; +import { action, verify } from '../../../src/writer/update-subscription'; + +describe('writer/update-subscription', () => { + const TEST_PREFIX = 'test-update-subscription'; + + afterAll(async () => { + await envelopDB.queryAsync('DELETE FROM subscribers'); + await envelopDB.endAsync(); + await db.endAsync(); + await sequencerDB.endAsync(); + }); + + describe('verify()', () => { + const msg = JSON.stringify({ payload: { subscriptions: ['closedProposal'] } }); + const msgWithInvalidSubscriptions = JSON.stringify({ + payload: { subscriptions: ['test'] } + }); + + beforeAll(async () => { + await Promise.all([ + envelopDB.queryAsync( + 'INSERT INTO subscribers SET address = ?, email = ?, subscriptions = ?, created = ?, verified = ?', + [`${TEST_PREFIX}-0x0`, 'test@snapshot.org', '[]', 0, 0] + ), + envelopDB.queryAsync( + 'INSERT INTO subscribers SET address = ?, email = ?, subscriptions = ?, created = ?, verified = ?', + [`${TEST_PREFIX}-0x1`, 'test1@snapshot.org', '[]', 0, 1] + ) + ]); + }); + + it('rejects when the address is not subscribed', () => { + return expect(verify({ address: '0x0', msg })).rejects.toEqual(`user not subscribed`); + }); + + it('rejects when the address is not verified', () => { + return expect(verify({ address: `${TEST_PREFIX}-0x0`, msg })).rejects.toEqual( + `email not verified` + ); + }); + + it('rejects when subscription values are not valid', () => { + return expect( + verify({ address: `${TEST_PREFIX}-0x1`, msg: msgWithInvalidSubscriptions }) + ).rejects.toEqual(`invalid subscription value`); + }); + + it('resolves when all args are valid', () => { + return expect(verify({ address: `${TEST_PREFIX}-0x1`, msg })).resolves.toHaveProperty( + 'address' + ); + }); + }); + + describe('action()', () => { + const address = `${TEST_PREFIX}-0x3`; + const subscriptions = ['newProposal', 'closedProposal']; + + beforeAll(async () => { + await envelopDB.queryAsync( + 'INSERT INTO subscribers SET address = ?, email = ?, subscriptions = ?, created = ?, verified = ?', + [address, 'test@snapshot.org', '["summary"]', 0, 1] + ); + }); + + it('updates the subscription', async () => { + await action({ + address: address, + msg: JSON.stringify({ payload: { subscriptions } }) + }); + + const result = await envelopDB.queryAsync( + `SELECT subscriptions FROM subscribers WHERE address = ? LIMIT 1`, + [address] + ); + + expect(JSON.parse(result[0].subscriptions)).toEqual(subscriptions); + }); + }); +}); From 81aa88920557843138997a0ae49fc877dc74f2bd Mon Sep 17 00:00:00 2001 From: Wan Qi Chen <495709+wa0x6e@users.noreply.github.com> Date: Wed, 13 Nov 2024 19:51:10 +0900 Subject: [PATCH 04/11] feat: add subscription writer --- src/writer/index.ts | 2 + src/writer/subscription.ts | 39 +++++++++++++ test/integration/writer/subscription.test.ts | 59 ++++++++++++++++++++ 3 files changed, 100 insertions(+) create mode 100644 src/writer/subscription.ts create mode 100644 test/integration/writer/subscription.test.ts diff --git a/src/writer/index.ts b/src/writer/index.ts index b1ee52ad..c6b05c35 100644 --- a/src/writer/index.ts +++ b/src/writer/index.ts @@ -9,6 +9,7 @@ import * as proposal from './proposal'; import * as settings from './settings'; import * as statement from './statement'; import * as subscribe from './subscribe'; +import * as subscription from './subscription'; import * as unfollow from './unfollow'; import * as unsubscribe from './unsubscribe'; import * as updateProposal from './update-proposal'; @@ -27,6 +28,7 @@ export default { unfollow, subscribe, unsubscribe, + subscription, 'update-subscription': updateSubscription, 'delete-subscription': deleteSubscription, alias, diff --git a/src/writer/subscription.ts b/src/writer/subscription.ts new file mode 100644 index 00000000..6251c771 --- /dev/null +++ b/src/writer/subscription.ts @@ -0,0 +1,39 @@ +import { envelopDB } from '../helpers/mysql'; +import { jsonParse } from '../helpers/utils'; + +type Message = { msg: string; address: string }; + +const VALID_TYPES = ['email']; + +function extractPayload(message: Message) { + return jsonParse(message.msg).payload; +} + +export async function verify(message: Message): Promise { + const payload = extractPayload(message); + + if (!VALID_TYPES.includes(payload.type)) { + return Promise.reject('invalid type'); + } + + const result = await envelopDB.queryAsync( + `SELECT * FROM subscribers WHERE address = ? AND email = ? LIMIT 1`, + [message.address, payload.value] + ); + + if (result[0]) { + return Promise.reject('user already subscribed'); + } + + return true; +} + +export async function action(message: Message): Promise { + const payload = extractPayload(message); + + await envelopDB.queryAsync(`INSERT INTO subscribers (email, address, created) VALUES(?, ?, ?)`, [ + payload.value, + message.address, + (Date.now() / 1e3).toFixed() + ]); +} diff --git a/test/integration/writer/subscription.test.ts b/test/integration/writer/subscription.test.ts new file mode 100644 index 00000000..9ab24234 --- /dev/null +++ b/test/integration/writer/subscription.test.ts @@ -0,0 +1,59 @@ +import db, { envelopDB, sequencerDB } from '../../../src/helpers/mysql'; +import { action, verify } from '../../../src/writer/subscription'; + +describe('writer/subscription', () => { + const TEST_PREFIX = 'test-subscription'; + const msg = JSON.stringify({ payload: { value: 'test@snapshot.org', type: 'email' } }); + + afterAll(async () => { + await envelopDB.queryAsync('DELETE FROM subscribers'); + await envelopDB.endAsync(); + await db.endAsync(); + await sequencerDB.endAsync(); + }); + + describe('verify()', () => { + const address = `${TEST_PREFIX}-0x0`; + const invalidMsg = JSON.stringify({ + payload: { type: 'hello', value: 'test@snapshot.org' } + }); + + beforeAll(async () => { + await envelopDB.queryAsync( + 'INSERT INTO subscribers SET address = ?, email = ?, subscriptions = ?, created = ?, verified = ?', + [address, 'test@snapshot.org', '[]', 0, 0] + ); + }); + + it('rejects when the address is already subscribed', () => { + return expect(verify({ address: address, msg })).rejects.toEqual(`user already subscribed`); + }); + + it('rejects when the subscription type is not valid', () => { + return expect(verify({ address: address, msg: invalidMsg })).rejects.toEqual('invalid type'); + }); + + it('resolves when all args are valid', () => { + return expect(verify({ address: `${TEST_PREFIX}-0x1`, msg })).resolves.toBe(true); + }); + }); + + describe('action()', () => { + const address = `${TEST_PREFIX}-0x1`; + + it('creates a subscription', async () => { + await action({ + address: address, + msg + }); + + const result = await envelopDB.queryAsync( + `SELECT * FROM subscribers WHERE address = ? LIMIT 1`, + [address] + ); + + expect(result[0].email).toEqual('test@snapshot.org'); + expect(result[0].verified).toEqual(0); + }); + }); +}); From 48c73f0681801109679e440db0dc6b295e1eea07 Mon Sep 17 00:00:00 2001 From: Wan Qi Chen <495709+wa0x6e@users.noreply.github.com> Date: Wed, 13 Nov 2024 19:56:45 +0900 Subject: [PATCH 05/11] feat: enable subscriptions via alias --- src/ingestor.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/ingestor.ts b/src/ingestor.ts index 5a503251..fccb97be 100644 --- a/src/ingestor.ts +++ b/src/ingestor.ts @@ -106,7 +106,17 @@ export default async function ingestor(req) { } // Check if signing address is an alias - const aliasTypes = ['follow', 'unfollow', 'subscribe', 'unsubscribe', 'profile', 'statement']; + const aliasTypes = [ + 'follow', + 'unfollow', + 'subscribe', + 'unsubscribe', + 'profile', + 'statement', + 'subscription', + 'update-subscription', + 'delete-subscription' + ]; const aliasOptionTypes = ['vote', 'vote-array', 'vote-string', 'proposal', 'delete-proposal']; if (body.address !== message.from) { if (!aliasTypes.includes(type) && !aliasOptionTypes.includes(type)) From 6993a1a5fb7b71ef497d6922fb97a4d772810968 Mon Sep 17 00:00:00 2001 From: Wan Qi Chen <495709+wa0x6e@users.noreply.github.com> Date: Fri, 15 Nov 2024 09:31:54 +0900 Subject: [PATCH 06/11] fix: add subscription support to ingestor --- src/ingestor.ts | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/ingestor.ts b/src/ingestor.ts index fccb97be..dd93738f 100644 --- a/src/ingestor.ts +++ b/src/ingestor.ts @@ -31,6 +31,10 @@ const NETWORK_METADATA = { } }; +function shouldPinIpfs(type: string, message: any) { + return !(type === 'subscription' && message.type === 'email' && message.value !== ''); +} + export default async function ingestor(req) { if (flaggedIps.includes(sha256(getIp(req)))) { return Promise.reject('unauthorized'); @@ -92,7 +96,7 @@ export default async function ingestor(req) { } let aliased = false; - if (!['settings', 'alias', 'profile'].includes(type)) { + if (!['settings', 'alias', 'profile', 'subscription', 'delete-subscription'].includes(type)) { if (!message.space) return Promise.reject('unknown space'); try { @@ -216,6 +220,18 @@ export default async function ingestor(req) { type = 'vote'; } + if (type === 'subscription' || type === 'delete-subscription') { + if (message.type === 'email' && message.value === '') { + type = 'update-subscription'; + } + + payload = { + type: message.type, + value: message.value, + metadata: message.metadata + }; + } + let legacyBody: any = { address: message.from, msg: JSON.stringify({ @@ -255,7 +271,7 @@ export default async function ingestor(req) { ...restBody }; [pinned, receipt] = await Promise.all([ - pin(ipfsBody, process.env.PINEAPPLE_URL), + shouldPinIpfs(type, message) ? pin(ipfsBody, process.env.PINEAPPLE_URL) : { cid: '' }, issueReceipt(formattedSignature) ]); } catch (e) { From 50aea60f81b6ce6e51d584dd06ad8fea6359bd0e Mon Sep 17 00:00:00 2001 From: Wan Qi Chen <495709+wa0x6e@users.noreply.github.com> Date: Sat, 23 Nov 2024 21:16:25 +0800 Subject: [PATCH 07/11] refactor: rename subscription to email-subscription --- src/ingestor.ts | 24 +++--- ...iption.ts => delete-email-subscription.ts} | 8 +- src/writer/email-subscription.ts | 85 +++++++++++++++++++ src/writer/index.ts | 10 +-- src/writer/statement.ts | 1 + src/writer/subscription.ts | 39 --------- src/writer/update-subscription.ts | 54 ------------ ...t.ts => delete-email-subscription.test.ts} | 4 +- ...ion.test.ts => email-subscription.test.ts} | 12 +-- ...t.ts => update-email-subscription.test.ts} | 14 ++- 10 files changed, 120 insertions(+), 131 deletions(-) rename src/writer/{delete-subscription.ts => delete-email-subscription.ts} (55%) create mode 100644 src/writer/email-subscription.ts delete mode 100644 src/writer/subscription.ts delete mode 100644 src/writer/update-subscription.ts rename test/integration/writer/{delete-subscription.test.ts => delete-email-subscription.test.ts} (92%) rename test/integration/writer/{subscription.test.ts => email-subscription.test.ts} (83%) rename test/integration/writer/{update-subscription.test.ts => update-email-subscription.test.ts} (86%) diff --git a/src/ingestor.ts b/src/ingestor.ts index dd93738f..c1aad9cd 100644 --- a/src/ingestor.ts +++ b/src/ingestor.ts @@ -32,7 +32,7 @@ const NETWORK_METADATA = { }; function shouldPinIpfs(type: string, message: any) { - return !(type === 'subscription' && message.type === 'email' && message.value !== ''); + return !(type === 'email-subscription' && message.email); } export default async function ingestor(req) { @@ -96,7 +96,11 @@ export default async function ingestor(req) { } let aliased = false; - if (!['settings', 'alias', 'profile', 'subscription', 'delete-subscription'].includes(type)) { + if ( + !['settings', 'alias', 'profile', 'email-subscription', 'delete-email-subscription'].includes( + type + ) + ) { if (!message.space) return Promise.reject('unknown space'); try { @@ -117,9 +121,8 @@ export default async function ingestor(req) { 'unsubscribe', 'profile', 'statement', - 'subscription', - 'update-subscription', - 'delete-subscription' + 'email-subscription', + 'delete-email-subscription' ]; const aliasOptionTypes = ['vote', 'vote-array', 'vote-string', 'proposal', 'delete-proposal']; if (body.address !== message.from) { @@ -220,15 +223,10 @@ export default async function ingestor(req) { type = 'vote'; } - if (type === 'subscription' || type === 'delete-subscription') { - if (message.type === 'email' && message.value === '') { - type = 'update-subscription'; - } - + if (type === 'email-subscription') { payload = { - type: message.type, - value: message.value, - metadata: message.metadata + email: message.email, + subscriptions: message.subscriptions }; } diff --git a/src/writer/delete-subscription.ts b/src/writer/delete-email-subscription.ts similarity index 55% rename from src/writer/delete-subscription.ts rename to src/writer/delete-email-subscription.ts index 76e89d51..f509fa66 100644 --- a/src/writer/delete-subscription.ts +++ b/src/writer/delete-email-subscription.ts @@ -2,9 +2,9 @@ import { envelopDB } from '../helpers/mysql'; type Message = { address: string }; -export async function verify(message: Message): Promise { +export async function verify(message: Message): Promise { const result = await envelopDB.queryAsync( - `SELECT * FROM subscribers WHERE address = ? AND verified > 0 LIMIT 1`, + 'SELECT * FROM subscribers WHERE address = ? AND verified > 0 LIMIT 1', [message.address] ); @@ -12,11 +12,11 @@ export async function verify(message: Message): Promise { return Promise.reject('user not subscribed'); } - return result[0]; + return true; } export async function action(message: Message): Promise { - await envelopDB.queryAsync(`DELETE FROM subscribers WHERE address = ? AND verified > 0 LIMIT 1`, [ + await envelopDB.queryAsync('DELETE FROM subscribers WHERE address = ? AND verified > 0 LIMIT 1', [ message.address ]); } diff --git a/src/writer/email-subscription.ts b/src/writer/email-subscription.ts new file mode 100644 index 00000000..f6fa13d5 --- /dev/null +++ b/src/writer/email-subscription.ts @@ -0,0 +1,85 @@ +import snapshot from '@snapshot-labs/snapshot.js'; +import log from '../helpers/log'; +import { envelopDB } from '../helpers/mysql'; +import { jsonParse } from '../helpers/utils'; + +type Message = { msg: string; address: string }; +type Payload = { + email?: string; + subscriptions?: string[]; +}; + +function extractPayload(message: Message): Payload { + return jsonParse(message.msg).payload; +} + +export async function verify(message: Message): Promise { + const payload = extractPayload(message); + + const schemaIsValid = snapshot.utils.validateSchema(snapshot.schemas.emailSubscription, payload); + if (schemaIsValid !== true) { + log.warn(`[writer] Wrong email subscription format ${JSON.stringify(schemaIsValid)}`); + return Promise.reject('wrong email subscription format'); + } + + if (payload.email?.length) { + return verifySubscriptionCreation(message, payload); + } else { + return verifySubscriptionUpdate(message, payload); + } +} + +export async function action(message: Message): Promise { + const payload = extractPayload(message); + + if (payload.email?.length) { + await createAction(message, payload); + } else { + await updateAction(message, payload); + } +} + +async function verifySubscriptionCreation(message: Message, payload: Payload): Promise { + const result = await envelopDB.queryAsync( + `SELECT * FROM subscribers WHERE address = ? AND email = ? LIMIT 1`, + [message.address, payload.email] + ); + + if (result[0]) { + return Promise.reject('email already subscribed'); + } + + return true; +} + +async function verifySubscriptionUpdate(message: Message, payload: Payload): Promise { + const result = await envelopDB.queryAsync( + `SELECT * FROM subscribers WHERE address = ? ORDER BY verified DESC LIMIT 1`, + [message.address, payload.email] + ); + + if (!result[0]) { + return Promise.reject('email not subscribed'); + } + + if (!result[0].verified) { + return Promise.reject('email not verified'); + } + + return true; +} + +async function createAction(message: Message, payload: Payload) { + await envelopDB.queryAsync(`INSERT INTO subscribers (email, address, created) VALUES(?, ?, ?)`, [ + payload.email, + message.address, + (Date.now() / 1e3).toFixed() + ]); +} + +async function updateAction(message: Message, payload: Payload) { + await envelopDB.queryAsync( + `UPDATE subscribers SET subscriptions = ? WHERE address = ? AND verified > 0 LIMIT 1`, + [JSON.stringify(payload.subscriptions), message.address] + ); +} diff --git a/src/writer/index.ts b/src/writer/index.ts index c6b05c35..4a4a2e8f 100644 --- a/src/writer/index.ts +++ b/src/writer/index.ts @@ -1,7 +1,8 @@ import * as alias from './alias'; +import * as deleteEmailSubscription from './delete-email-subscription'; import * as deleteProposal from './delete-proposal'; import * as deleteSpace from './delete-space'; -import * as deleteSubscription from './delete-subscription'; +import * as emailSubscription from './email-subscription'; import * as flagProposal from './flag-proposal'; import * as follow from './follow'; import * as profile from './profile'; @@ -9,11 +10,9 @@ import * as proposal from './proposal'; import * as settings from './settings'; import * as statement from './statement'; import * as subscribe from './subscribe'; -import * as subscription from './subscription'; import * as unfollow from './unfollow'; import * as unsubscribe from './unsubscribe'; import * as updateProposal from './update-proposal'; -import * as updateSubscription from './update-subscription'; import * as vote from './vote'; export default { @@ -28,9 +27,8 @@ export default { unfollow, subscribe, unsubscribe, - subscription, - 'update-subscription': updateSubscription, - 'delete-subscription': deleteSubscription, + 'email-subscription': emailSubscription, + 'delete-email-subscription': deleteEmailSubscription, alias, profile, statement diff --git a/src/writer/statement.ts b/src/writer/statement.ts index 82337d0e..75a0868b 100644 --- a/src/writer/statement.ts +++ b/src/writer/statement.ts @@ -5,6 +5,7 @@ import { DEFAULT_NETWORK_ID, jsonParse, NETWORK_IDS } from '../helpers/utils'; export async function verify(body): Promise { const msg = jsonParse(body.msg, {}); + const schemaIsValid = snapshot.utils.validateSchema(snapshot.schemas.statement, msg.payload); if (schemaIsValid !== true) { log.warn(`[writer] Wrong statement format ${JSON.stringify(schemaIsValid)}`); diff --git a/src/writer/subscription.ts b/src/writer/subscription.ts deleted file mode 100644 index 6251c771..00000000 --- a/src/writer/subscription.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { envelopDB } from '../helpers/mysql'; -import { jsonParse } from '../helpers/utils'; - -type Message = { msg: string; address: string }; - -const VALID_TYPES = ['email']; - -function extractPayload(message: Message) { - return jsonParse(message.msg).payload; -} - -export async function verify(message: Message): Promise { - const payload = extractPayload(message); - - if (!VALID_TYPES.includes(payload.type)) { - return Promise.reject('invalid type'); - } - - const result = await envelopDB.queryAsync( - `SELECT * FROM subscribers WHERE address = ? AND email = ? LIMIT 1`, - [message.address, payload.value] - ); - - if (result[0]) { - return Promise.reject('user already subscribed'); - } - - return true; -} - -export async function action(message: Message): Promise { - const payload = extractPayload(message); - - await envelopDB.queryAsync(`INSERT INTO subscribers (email, address, created) VALUES(?, ?, ?)`, [ - payload.value, - message.address, - (Date.now() / 1e3).toFixed() - ]); -} diff --git a/src/writer/update-subscription.ts b/src/writer/update-subscription.ts deleted file mode 100644 index 98fc6bc7..00000000 --- a/src/writer/update-subscription.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { envelopDB } from '../helpers/mysql'; -import { jsonParse } from '../helpers/utils'; - -type Message = { msg: string; address: string }; -type Payload = { - type: string; - value: string; - metadata: { subscriptions: string[] }; -}; - -const VALID_SUBSCRIPTIONS = ['summary', 'newProposal', 'closedProposal']; - -function extractPayload(message: Message): Payload { - const payload = jsonParse(message.msg).payload; - - return { - ...payload, - metadata: jsonParse(payload.metadata) - }; -} - -export async function verify(message: Message): Promise { - const result = await envelopDB.queryAsync(`SELECT * FROM subscribers WHERE address = ? LIMIT 1`, [ - message.address - ]); - - if (!result[0]) { - return Promise.reject('user not subscribed'); - } - - if (!result[0].verified) { - return Promise.reject('email not verified'); - } - - const { - metadata: { subscriptions } - } = extractPayload(message); - if ((subscriptions || []).some(s => !VALID_SUBSCRIPTIONS.includes(s))) { - return Promise.reject('invalid subscription value'); - } - - return result[0]; -} - -export async function action(message: Message): Promise { - const { - metadata: { subscriptions } - } = extractPayload(message); - - await envelopDB.queryAsync( - `UPDATE subscribers SET subscriptions = ? WHERE address = ? AND verified > 0 LIMIT 1`, - [JSON.stringify(subscriptions), message.address] - ); -} diff --git a/test/integration/writer/delete-subscription.test.ts b/test/integration/writer/delete-email-subscription.test.ts similarity index 92% rename from test/integration/writer/delete-subscription.test.ts rename to test/integration/writer/delete-email-subscription.test.ts index 822dc21b..91f3bda1 100644 --- a/test/integration/writer/delete-subscription.test.ts +++ b/test/integration/writer/delete-email-subscription.test.ts @@ -1,5 +1,5 @@ import db, { envelopDB, sequencerDB } from '../../../src/helpers/mysql'; -import { action, verify } from '../../../src/writer/delete-subscription'; +import { action, verify } from '../../../src/writer/delete-email-subscription'; describe('writer/delete-subscription', () => { const TEST_PREFIX = 'test-delete-subscription'; @@ -36,7 +36,7 @@ describe('writer/delete-subscription', () => { }); it('resolves when the address is verified', () => { - return expect(verify({ address: `${TEST_PREFIX}-0x1` })).resolves.toHaveProperty('address'); + expect(verify({ address: `${TEST_PREFIX}-0x1` })).resolves; }); }); diff --git a/test/integration/writer/subscription.test.ts b/test/integration/writer/email-subscription.test.ts similarity index 83% rename from test/integration/writer/subscription.test.ts rename to test/integration/writer/email-subscription.test.ts index 9ab24234..66340cdc 100644 --- a/test/integration/writer/subscription.test.ts +++ b/test/integration/writer/email-subscription.test.ts @@ -1,9 +1,9 @@ import db, { envelopDB, sequencerDB } from '../../../src/helpers/mysql'; -import { action, verify } from '../../../src/writer/subscription'; +import { action, verify } from '../../../src/writer/email-subscription'; describe('writer/subscription', () => { const TEST_PREFIX = 'test-subscription'; - const msg = JSON.stringify({ payload: { value: 'test@snapshot.org', type: 'email' } }); + const msg = JSON.stringify({ payload: { email: 'test@snapshot.org', subscriptions: [] } }); afterAll(async () => { await envelopDB.queryAsync('DELETE FROM subscribers'); @@ -15,7 +15,7 @@ describe('writer/subscription', () => { describe('verify()', () => { const address = `${TEST_PREFIX}-0x0`; const invalidMsg = JSON.stringify({ - payload: { type: 'hello', value: 'test@snapshot.org' } + payload: { email: 'not an email' } }); beforeAll(async () => { @@ -26,11 +26,13 @@ describe('writer/subscription', () => { }); it('rejects when the address is already subscribed', () => { - return expect(verify({ address: address, msg })).rejects.toEqual(`user already subscribed`); + return expect(verify({ address: address, msg })).rejects.toEqual('email already subscribed'); }); it('rejects when the subscription type is not valid', () => { - return expect(verify({ address: address, msg: invalidMsg })).rejects.toEqual('invalid type'); + return expect(verify({ address: address, msg: invalidMsg })).rejects.toEqual( + 'wrong email subscription format' + ); }); it('resolves when all args are valid', () => { diff --git a/test/integration/writer/update-subscription.test.ts b/test/integration/writer/update-email-subscription.test.ts similarity index 86% rename from test/integration/writer/update-subscription.test.ts rename to test/integration/writer/update-email-subscription.test.ts index c880f811..26f1da6c 100644 --- a/test/integration/writer/update-subscription.test.ts +++ b/test/integration/writer/update-email-subscription.test.ts @@ -1,5 +1,5 @@ import db, { envelopDB, sequencerDB } from '../../../src/helpers/mysql'; -import { action, verify } from '../../../src/writer/update-subscription'; +import { action, verify } from '../../../src/writer/email-subscription'; describe('writer/update-subscription', () => { const TEST_PREFIX = 'test-update-subscription'; @@ -31,7 +31,7 @@ describe('writer/update-subscription', () => { }); it('rejects when the address is not subscribed', () => { - return expect(verify({ address: '0x0', msg })).rejects.toEqual(`user not subscribed`); + return expect(verify({ address: '0x0', msg })).rejects.toEqual(`email not subscribed`); }); it('rejects when the address is not verified', () => { @@ -43,13 +43,11 @@ describe('writer/update-subscription', () => { it('rejects when subscription values are not valid', () => { return expect( verify({ address: `${TEST_PREFIX}-0x1`, msg: msgWithInvalidSubscriptions }) - ).rejects.toEqual(`invalid subscription value`); + ).rejects.toEqual(`wrong email subscription format`); }); it('resolves when all args are valid', () => { - return expect(verify({ address: `${TEST_PREFIX}-0x1`, msg })).resolves.toHaveProperty( - 'address' - ); + expect(verify({ address: `${TEST_PREFIX}-0x1`, msg })).resolves; }); }); @@ -66,12 +64,12 @@ describe('writer/update-subscription', () => { it('updates the subscription', async () => { await action({ - address: address, + address, msg: JSON.stringify({ payload: { subscriptions } }) }); const result = await envelopDB.queryAsync( - `SELECT subscriptions FROM subscribers WHERE address = ? LIMIT 1`, + 'SELECT subscriptions FROM subscribers WHERE address = ? AND verified > 0 LIMIT 1', [address] ); From b4a3d50ae64eaed8b6f511a911a329a56ea0ff64 Mon Sep 17 00:00:00 2001 From: Wan Qi Chen <495709+wa0x6e@users.noreply.github.com> Date: Mon, 9 Dec 2024 17:13:22 +0800 Subject: [PATCH 08/11] fix: make envelopDB connection optional --- src/helpers/mysql.ts | 27 ++++++++++++++----------- src/writer/delete-email-subscription.ts | 8 ++++++++ src/writer/email-subscription.ts | 8 ++++++++ 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/src/helpers/mysql.ts b/src/helpers/mysql.ts index 5c0f81f2..78b3acef 100644 --- a/src/helpers/mysql.ts +++ b/src/helpers/mysql.ts @@ -35,18 +35,21 @@ sequencerConfig.timeout = 60e3; sequencerConfig.charset = 'utf8mb4'; const sequencerDB = mysql.createPool(sequencerConfig); -// @ts-ignore -const envelopConfig = parse(process.env.ENVELOP_DATABASE_URL); -envelopConfig.connectionLimit = connectionLimit; -envelopConfig.multipleStatements = true; -envelopConfig.database = envelopConfig.path[0]; -envelopConfig.host = envelopConfig.hosts[0].name; -envelopConfig.port = envelopConfig.hosts[0].port; -envelopConfig.connectTimeout = 60e3; -envelopConfig.acquireTimeout = 60e3; -envelopConfig.timeout = 60e3; -envelopConfig.charset = 'utf8mb4'; -const envelopDB = mysql.createPool(envelopConfig); +let envelopDB; +if (process.env.ENVELOP_DATABASE_URL) { + // @ts-ignore + const envelopConfig = parse(process.env.ENVELOP_DATABASE_URL); + envelopConfig.connectionLimit = connectionLimit; + envelopConfig.multipleStatements = true; + envelopConfig.database = envelopConfig.path[0]; + envelopConfig.host = envelopConfig.hosts[0].name; + envelopConfig.port = envelopConfig.hosts[0].port; + envelopConfig.connectTimeout = 60e3; + envelopConfig.acquireTimeout = 60e3; + envelopConfig.timeout = 60e3; + envelopConfig.charset = 'utf8mb4'; + envelopDB = mysql.createPool(envelopConfig); +} bluebird.promisifyAll([Pool, Connection]); diff --git a/src/writer/delete-email-subscription.ts b/src/writer/delete-email-subscription.ts index f509fa66..983d7ca1 100644 --- a/src/writer/delete-email-subscription.ts +++ b/src/writer/delete-email-subscription.ts @@ -3,6 +3,10 @@ import { envelopDB } from '../helpers/mysql'; type Message = { address: string }; export async function verify(message: Message): Promise { + if (!envelopDB) { + return Promise.reject('not supported'); + } + const result = await envelopDB.queryAsync( 'SELECT * FROM subscribers WHERE address = ? AND verified > 0 LIMIT 1', [message.address] @@ -16,6 +20,10 @@ export async function verify(message: Message): Promise { } export async function action(message: Message): Promise { + if (!envelopDB) { + return Promise.reject('not supported'); + } + await envelopDB.queryAsync('DELETE FROM subscribers WHERE address = ? AND verified > 0 LIMIT 1', [ message.address ]); diff --git a/src/writer/email-subscription.ts b/src/writer/email-subscription.ts index f6fa13d5..251ef3f9 100644 --- a/src/writer/email-subscription.ts +++ b/src/writer/email-subscription.ts @@ -14,6 +14,10 @@ function extractPayload(message: Message): Payload { } export async function verify(message: Message): Promise { + if (!envelopDB) { + return Promise.reject('not supported'); + } + const payload = extractPayload(message); const schemaIsValid = snapshot.utils.validateSchema(snapshot.schemas.emailSubscription, payload); @@ -30,6 +34,10 @@ export async function verify(message: Message): Promise { } export async function action(message: Message): Promise { + if (!envelopDB) { + return Promise.reject('not supported'); + } + const payload = extractPayload(message); if (payload.email?.length) { From 1d54188834d7b852cd7d4ae49605cfa5e90715cf Mon Sep 17 00:00:00 2001 From: Wan <495709+wa0x6e@users.noreply.github.com> Date: Mon, 9 Dec 2024 18:13:56 +0900 Subject: [PATCH 09/11] Update src/writer/statement.ts Co-authored-by: Chaitanya --- src/writer/statement.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/writer/statement.ts b/src/writer/statement.ts index 75a0868b..82337d0e 100644 --- a/src/writer/statement.ts +++ b/src/writer/statement.ts @@ -5,7 +5,6 @@ import { DEFAULT_NETWORK_ID, jsonParse, NETWORK_IDS } from '../helpers/utils'; export async function verify(body): Promise { const msg = jsonParse(body.msg, {}); - const schemaIsValid = snapshot.utils.validateSchema(snapshot.schemas.statement, msg.payload); if (schemaIsValid !== true) { log.warn(`[writer] Wrong statement format ${JSON.stringify(schemaIsValid)}`); From 3409e853ba01702ebcd1f14e6dc614f894e6ed7e Mon Sep 17 00:00:00 2001 From: Wan Qi Chen <495709+wa0x6e@users.noreply.github.com> Date: Mon, 9 Dec 2024 17:16:49 +0800 Subject: [PATCH 10/11] fix: subscription deletion should delete regardless of verified status --- src/writer/delete-email-subscription.ts | 2 +- test/integration/writer/delete-email-subscription.test.ts | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/writer/delete-email-subscription.ts b/src/writer/delete-email-subscription.ts index 983d7ca1..bd65172a 100644 --- a/src/writer/delete-email-subscription.ts +++ b/src/writer/delete-email-subscription.ts @@ -24,7 +24,7 @@ export async function action(message: Message): Promise { return Promise.reject('not supported'); } - await envelopDB.queryAsync('DELETE FROM subscribers WHERE address = ? AND verified > 0 LIMIT 1', [ + await envelopDB.queryAsync('DELETE FROM subscribers WHERE address = ? LIMIT 1', [ message.address ]); } diff --git a/test/integration/writer/delete-email-subscription.test.ts b/test/integration/writer/delete-email-subscription.test.ts index 91f3bda1..1d96c24d 100644 --- a/test/integration/writer/delete-email-subscription.test.ts +++ b/test/integration/writer/delete-email-subscription.test.ts @@ -56,16 +56,14 @@ describe('writer/delete-subscription', () => { ]); }); - it('deletes the subscription', async () => { + it('deletes all the subscriptions associated to the address', async () => { await action({ address: address }); const results = await envelopDB.queryAsync('SELECT * FROM subscribers WHERE address = ?', [ address ]); - // Only delete the verified subscription - expect(results.length).toBe(1); - expect(results[0].email).toEqual('test@snapshot.org'); + expect(results.length).toBe(0); }); }); }); From 27f15bca18a972730f2228aa7b388fef12a0ef70 Mon Sep 17 00:00:00 2001 From: Wan Qi Chen <495709+wa0x6e@users.noreply.github.com> Date: Mon, 9 Dec 2024 18:33:16 +0800 Subject: [PATCH 11/11] fix: exclude all emails-* actions from IPFS pinning --- src/ingestor.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ingestor.ts b/src/ingestor.ts index c1aad9cd..9d7e5f87 100644 --- a/src/ingestor.ts +++ b/src/ingestor.ts @@ -31,8 +31,8 @@ const NETWORK_METADATA = { } }; -function shouldPinIpfs(type: string, message: any) { - return !(type === 'email-subscription' && message.email); +function shouldPinIpfs(type: string) { + return type !== 'email-subscription' && type !== 'delete-email-subscription'; } export default async function ingestor(req) { @@ -269,7 +269,7 @@ export default async function ingestor(req) { ...restBody }; [pinned, receipt] = await Promise.all([ - shouldPinIpfs(type, message) ? pin(ipfsBody, process.env.PINEAPPLE_URL) : { cid: '' }, + shouldPinIpfs(type) ? pin(ipfsBody, process.env.PINEAPPLE_URL) : { cid: '' }, issueReceipt(formattedSignature) ]); } catch (e) {