From 1c9f8115da5d2cc43236019fab7c13748fdb0780 Mon Sep 17 00:00:00 2001
From: Wan Qi Chen <495709+wa0x6e@users.noreply.github.com>
Date: Fri, 27 Oct 2023 10:48:09 +0700
Subject: [PATCH 01/12] refactor: extract redis config to its own file

---
 .env.example             |  1 +
 src/helpers/rateLimit.ts | 24 ++++--------------------
 src/helpers/redis.ts     | 19 +++++++++++++++++++
 test/.env.test           |  2 ++
 4 files changed, 26 insertions(+), 20 deletions(-)
 create mode 100644 src/helpers/redis.ts

diff --git a/.env.example b/.env.example
index bdea9c3e..1a6d4f05 100644
--- a/.env.example
+++ b/.env.example
@@ -11,6 +11,7 @@ PINEAPPLE_URL=https://.../upload # optional
 SCORE_API_URL=https://score.snapshot.org
 # If you need unlimted access to score-api, use `https://score.snapshot.org?apiKey=...`
 RATE_LIMIT_DATABASE_URL= # optional
+RATE_LIMIT_KEYS_PREFIX=snapshot-sequencer: #optional
 BROVIDER_URL=https://rpc.snapshot.org # optional
 # Secret for laser
 AUTH_SECRET=1dfd84a695705665668c260222ded178d1f1d62d251d7bee8148428dac6d0487 # optional
diff --git a/src/helpers/rateLimit.ts b/src/helpers/rateLimit.ts
index b7044922..a64049e7 100644
--- a/src/helpers/rateLimit.ts
+++ b/src/helpers/rateLimit.ts
@@ -1,23 +1,7 @@
 import rateLimit from 'express-rate-limit';
 import RedisStore from 'rate-limit-redis';
-import { createClient } from 'redis';
+import redisClient from './redis';
 import { getIp, sendError, sha256 } from './utils';
-import log from './log';
-
-let client;
-
-(async () => {
-  if (!process.env.RATE_LIMIT_DATABASE_URL) return;
-
-  log.info('[redis-rl] Connecting to Redis');
-  client = createClient({ url: process.env.RATE_LIMIT_DATABASE_URL });
-  client.on('connect', () => log.info('[redis-rl] Redis connect'));
-  client.on('ready', () => log.info('[redis-rl] Redis ready'));
-  client.on('reconnecting', err => log.info('[redis-rl] Redis reconnecting', err));
-  client.on('error', err => log.info('[redis-rl] Redis error', err));
-  client.on('end', err => log.info('[redis-rl] Redis end', err));
-  await client.connect();
-})();
 
 const hashedIp = (req): string => sha256(getIp(req)).slice(0, 7);
 
@@ -34,10 +18,10 @@ export default rateLimit({
       429
     );
   },
-  store: client
+  store: redisClient
     ? new RedisStore({
-        sendCommand: (...args: string[]) => client.sendCommand(args),
-        prefix: 'snapshot-sequencer:'
+        sendCommand: (...args: string[]) => redisClient.sendCommand(args),
+        prefix: process.env.RATE_LIMIT_KEYS_PREFIX || 'snapshot-sequencer:'
       })
     : undefined
 });
diff --git a/src/helpers/redis.ts b/src/helpers/redis.ts
new file mode 100644
index 00000000..375eadc2
--- /dev/null
+++ b/src/helpers/redis.ts
@@ -0,0 +1,19 @@
+import { createClient } from 'redis';
+import log from './log';
+
+let client;
+
+(async () => {
+  if (!process.env.RATE_LIMIT_DATABASE_URL) return;
+
+  log.info('[redis] Connecting to Redis');
+  client = createClient({ url: process.env.RATE_LIMIT_DATABASE_URL });
+  client.on('connect', () => log.info('[redis] Redis connect'));
+  client.on('ready', () => log.info('[redis] Redis ready'));
+  client.on('reconnecting', err => log.info('[redis] Redis reconnecting', err));
+  client.on('error', err => log.info('[redis] Redis error', err));
+  client.on('end', err => log.info('[redis] Redis end', err));
+  await client.connect();
+})();
+
+export default client;
diff --git a/test/.env.test b/test/.env.test
index e1015222..dcc0e92c 100644
--- a/test/.env.test
+++ b/test/.env.test
@@ -3,3 +3,5 @@ SEQ_DATABASE_URL=mysql://root:root@127.0.0.1:3306/snapshot_sequencer_test
 NETWORK=main
 RELAYER_PK=01686849e86499c1860ea0afc97f29c11018cbac049abf843df875c60054076e
 NODE_ENV=test
+RATE_LIMIT_DATABASE_URL=redis://localhost:6379
+RATE_LIMIT_KEYS_PREFIX=snapshot-sequencer:

From 5e63a14171715fc750cab68a38eb7f8a66047624 Mon Sep 17 00:00:00 2001
From: Wan Qi Chen <495709+wa0x6e@users.noreply.github.com>
Date: Fri, 27 Oct 2023 10:48:18 +0700
Subject: [PATCH 02/12] chore: enable redis in test suite

---
 .github/workflows/test.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 2862f72f..599b8e2a 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -9,3 +9,4 @@ jobs:
     with:
       mysql_database_name: snapshot_sequencer_test
       mysql_schema_path: 'test/schema.sql'
+      redis: true

From b7fb7a22da1bcd5db2dc540d63c41b14506ef27a Mon Sep 17 00:00:00 2001
From: Wan Qi Chen <495709+wa0x6e@users.noreply.github.com>
Date: Fri, 27 Oct 2023 10:51:03 +0700
Subject: [PATCH 03/12] chore: use different prefix to tests

---
 test/.env.test | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/.env.test b/test/.env.test
index dcc0e92c..c767193d 100644
--- a/test/.env.test
+++ b/test/.env.test
@@ -4,4 +4,4 @@ NETWORK=main
 RELAYER_PK=01686849e86499c1860ea0afc97f29c11018cbac049abf843df875c60054076e
 NODE_ENV=test
 RATE_LIMIT_DATABASE_URL=redis://localhost:6379
-RATE_LIMIT_KEYS_PREFIX=snapshot-sequencer:
+RATE_LIMIT_KEYS_PREFIX=snapshot-sequencer-test:

From 8ac9c1369798c9b37c9feff065249df61e64fa48 Mon Sep 17 00:00:00 2001
From: Wan Qi Chen <495709+wa0x6e@users.noreply.github.com>
Date: Mon, 30 Oct 2023 22:17:47 +0700
Subject: [PATCH 04/12] fix: add a more aggressive rate limit for high errored
 users

---
 src/helpers/rateLimit.ts | 19 ++++++++++++++++---
 src/index.ts             |  4 ++--
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/src/helpers/rateLimit.ts b/src/helpers/rateLimit.ts
index b7044922..6d560615 100644
--- a/src/helpers/rateLimit.ts
+++ b/src/helpers/rateLimit.ts
@@ -21,9 +21,7 @@ let client;
 
 const hashedIp = (req): string => sha256(getIp(req)).slice(0, 7);
 
-export default rateLimit({
-  windowMs: 60 * 1e3,
-  max: 100,
+const rateLimitConfig = {
   keyGenerator: req => hashedIp(req),
   standardHeaders: true,
   legacyHeaders: false,
@@ -40,4 +38,19 @@ export default rateLimit({
         prefix: 'snapshot-sequencer:'
       })
     : undefined
+};
+
+const regularRateLimit = rateLimit({
+  windowMs: 60 * 1e3,
+  max: 100,
+  ...rateLimitConfig
+});
+
+const spamRateLimit = rateLimit({
+  windowMs: 15 * 1e3,
+  max: 10,
+  skipSuccessfulRequests: true,
+  ...rateLimitConfig
 });
+
+export { regularRateLimit, spamRateLimit };
diff --git a/src/index.ts b/src/index.ts
index 63f0e312..4aef611e 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -3,7 +3,7 @@ import cors from 'cors';
 import { initLogger, fallbackLogger } from '@snapshot-labs/snapshot-sentry';
 import express from 'express';
 import api from './api';
-import rateLimit from './helpers/rateLimit';
+import { regularRateLimit, spamRateLimit } from './helpers/rateLimit';
 import shutter from './helpers/shutter';
 import log from './helpers/log';
 import refreshModeration from './helpers/moderation';
@@ -19,7 +19,7 @@ app.disable('x-powered-by');
 app.use(express.json({ limit: '20mb' }));
 app.use(express.urlencoded({ limit: '20mb', extended: false }));
 app.use(cors({ maxAge: 86400 }));
-app.use(rateLimit);
+app.use(regularRateLimit, spamRateLimit);
 app.set('trust proxy', 1);
 app.use('/', api);
 app.use('/shutter', shutter);

From 1cba651189a7cfcbbbeee461ab1f495f69caadc2 Mon Sep 17 00:00:00 2001
From: Wan Qi Chen <495709+wa0x6e@users.noreply.github.com>
Date: Tue, 31 Oct 2023 15:52:19 +0700
Subject: [PATCH 05/12] chore: upgrade rate-express-limit

---
 yarn.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index e5cdd2bc..b5c4254c 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2760,9 +2760,9 @@ express-prom-bundle@^6.6.0:
     url-value-parser "^2.0.0"
 
 express-rate-limit@^6.9.0:
-  version "6.9.0"
-  resolved "https://registry.yarnpkg.com/express-rate-limit/-/express-rate-limit-6.9.0.tgz#afecb23936d9cd1d133a3c20056708b9955cad0f"
-  integrity sha512-AnISR3V8qy4gpKM62/TzYdoFO9NV84fBx0POXzTryHU/qGUJBWuVGd+JhbvtVmKBv37t8/afmqdnv16xWoQxag==
+  version "6.11.2"
+  resolved "https://registry.yarnpkg.com/express-rate-limit/-/express-rate-limit-6.11.2.tgz#6c42035603d3b52e4e2fb59f6ebaa89e628ef980"
+  integrity sha512-a7uwwfNTh1U60ssiIkuLFWHt4hAC5yxlLGU2VP0X4YNlyEDZAqF4tK3GD3NSitVBrCQmQ0++0uOyFOgC2y4DDw==
 
 express@^4.18.1:
   version "4.18.1"

From e81463fa67721f12f01c5bb0da71a39476d725a9 Mon Sep 17 00:00:00 2001
From: Wan Qi Chen <495709+wa0x6e@users.noreply.github.com>
Date: Tue, 31 Oct 2023 15:52:45 +0700
Subject: [PATCH 06/12] fix: use separate key for different rate-limiter

---
 src/helpers/rateLimit.ts | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/helpers/rateLimit.ts b/src/helpers/rateLimit.ts
index 6d560615..23214ff7 100644
--- a/src/helpers/rateLimit.ts
+++ b/src/helpers/rateLimit.ts
@@ -22,7 +22,6 @@ let client;
 const hashedIp = (req): string => sha256(getIp(req)).slice(0, 7);
 
 const rateLimitConfig = {
-  keyGenerator: req => hashedIp(req),
   standardHeaders: true,
   legacyHeaders: false,
   handler: (req, res) => {
@@ -41,14 +40,16 @@ const rateLimitConfig = {
 };
 
 const regularRateLimit = rateLimit({
+  keyGenerator: req => `rl:${hashedIp(req)}`,
   windowMs: 60 * 1e3,
   max: 100,
   ...rateLimitConfig
 });
 
 const spamRateLimit = rateLimit({
+  keyGenerator: req => `rl-spam:${hashedIp(req)}`,
   windowMs: 15 * 1e3,
-  max: 10,
+  max: 15,
   skipSuccessfulRequests: true,
   ...rateLimitConfig
 });

From ff30217e3ba727f8a6b719c7391ec145f2bc9ab6 Mon Sep 17 00:00:00 2001
From: Wan Qi Chen <495709+wa0x6e@users.noreply.github.com>
Date: Tue, 31 Oct 2023 17:10:01 +0700
Subject: [PATCH 07/12] chore: add tests

---
 test/e2e/api.test.ts | 59 ++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 54 insertions(+), 5 deletions(-)

diff --git a/test/e2e/api.test.ts b/test/e2e/api.test.ts
index d5014d0b..48ed386a 100644
--- a/test/e2e/api.test.ts
+++ b/test/e2e/api.test.ts
@@ -1,16 +1,34 @@
 import fetch from 'node-fetch';
 import proposalInput from '../fixtures/ingestor-payload/proposal.json';
+import redis from '../../src/helpers/redis';
 
 const HOST = `http://localhost:${process.env.PORT || 3003}`;
 
+function successRequest() {
+  return fetch(`${HOST}`);
+}
+
+function failRequest() {
+  return fetch(HOST, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify(proposalInput)
+  });
+}
+
 describe('POST /', () => {
+  beforeEach(async () => {
+    await redis.del('snapshot-sequencer-test:rl:3e48ef9');
+    await redis.del('snapshot-sequencer-test:rl-spam:3e48ef9');
+  });
+
+  afterAll(async () => {
+    await redis.quit();
+  });
+
   describe('on invalid client input', () => {
     it('returns a 400 error', async () => {
-      const response = await fetch(HOST, {
-        method: 'POST',
-        headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify(proposalInput)
-      });
+      const response = await failRequest();
       const body = await response.json();
 
       expect(response.status).toBe(400);
@@ -18,4 +36,35 @@ describe('POST /', () => {
       expect(body.error_description).toBe('wrong timestamp');
     });
   });
+
+  describe('rate limit', () => {
+    describe('on a mix of success and failed requests', () => {
+      it('should return a 429 errors only after 100 requests / min', async () => {
+        for (let i = 1; i <= 100; i++) {
+          // 10% of failing requests
+          const response = await (Math.random() < 0.1 ? failRequest() : successRequest());
+          expect(response.status).not.toEqual(429);
+        }
+
+        const response = await fetch(HOST, {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify(proposalInput)
+        });
+        expect(response.status).toBe(429);
+      });
+    });
+
+    describe('on multiple failed requests', () => {
+      it('should return a 429 errors after 15 requests / 15s', async () => {
+        for (let i = 1; i <= 15; i++) {
+          const response = await failRequest();
+          expect(response.status).toBe(400);
+        }
+
+        const response = await fetch(`${HOST}/scores/proposal-id`);
+        expect(response.status).toBe(429);
+      });
+    });
+  });
 });

From cfcbad884c82e2a905636981256ce69c407b378e Mon Sep 17 00:00:00 2001
From: Wan Qi Chen <495709+wa0x6e@users.noreply.github.com>
Date: Tue, 31 Oct 2023 17:20:16 +0700
Subject: [PATCH 08/12] refactor: rename rateLimit to a more appropriate name

---
 src/helpers/rateLimit.ts | 6 +++---
 src/index.ts             | 4 ++--
 test/e2e/api.test.ts     | 5 +++--
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/helpers/rateLimit.ts b/src/helpers/rateLimit.ts
index 39a5777c..ce5daec6 100644
--- a/src/helpers/rateLimit.ts
+++ b/src/helpers/rateLimit.ts
@@ -30,12 +30,12 @@ const regularRateLimit = rateLimit({
   ...rateLimitConfig
 });
 
-const spamRateLimit = rateLimit({
-  keyGenerator: req => `rl-spam:${hashedIp(req)}`,
+const highErroredRateLimit = rateLimit({
+  keyGenerator: req => `rl-s:${hashedIp(req)}`,
   windowMs: 15 * 1e3,
   max: 15,
   skipSuccessfulRequests: true,
   ...rateLimitConfig
 });
 
-export { regularRateLimit, spamRateLimit };
+export { regularRateLimit, highErroredRateLimit };
diff --git a/src/index.ts b/src/index.ts
index 4aef611e..cb6a57cb 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -3,7 +3,7 @@ import cors from 'cors';
 import { initLogger, fallbackLogger } from '@snapshot-labs/snapshot-sentry';
 import express from 'express';
 import api from './api';
-import { regularRateLimit, spamRateLimit } from './helpers/rateLimit';
+import { regularRateLimit, highErroredRateLimit } from './helpers/rateLimit';
 import shutter from './helpers/shutter';
 import log from './helpers/log';
 import refreshModeration from './helpers/moderation';
@@ -19,7 +19,7 @@ app.disable('x-powered-by');
 app.use(express.json({ limit: '20mb' }));
 app.use(express.urlencoded({ limit: '20mb', extended: false }));
 app.use(cors({ maxAge: 86400 }));
-app.use(regularRateLimit, spamRateLimit);
+app.use(regularRateLimit, highErroredRateLimit);
 app.set('trust proxy', 1);
 app.use('/', api);
 app.use('/shutter', shutter);
diff --git a/test/e2e/api.test.ts b/test/e2e/api.test.ts
index 48ed386a..b6e8d7b7 100644
--- a/test/e2e/api.test.ts
+++ b/test/e2e/api.test.ts
@@ -18,8 +18,9 @@ function failRequest() {
 
 describe('POST /', () => {
   beforeEach(async () => {
-    await redis.del('snapshot-sequencer-test:rl:3e48ef9');
-    await redis.del('snapshot-sequencer-test:rl-spam:3e48ef9');
+    const keyPrefix = process.env.RATE_LIMIT_KEYS_PREFIX || 'snapshot-sequencer:';
+    await redis.del(`${keyPrefix}rl:3e48ef9`);
+    await redis.del(`${keyPrefix}rl-s:3e48ef9`);
   });
 
   afterAll(async () => {

From 04190bdaf308390103f9d6b75cdada3676ebb42e Mon Sep 17 00:00:00 2001
From: Wan Qi Chen <495709+wa0x6e@users.noreply.github.com>
Date: Tue, 31 Oct 2023 17:23:38 +0700
Subject: [PATCH 09/12] chore: fix tests

---
 test/e2e/api.test.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/e2e/api.test.ts b/test/e2e/api.test.ts
index b6e8d7b7..f247467f 100644
--- a/test/e2e/api.test.ts
+++ b/test/e2e/api.test.ts
@@ -42,8 +42,8 @@ describe('POST /', () => {
     describe('on a mix of success and failed requests', () => {
       it('should return a 429 errors only after 100 requests / min', async () => {
         for (let i = 1; i <= 100; i++) {
-          // 10% of failing requests
-          const response = await (Math.random() < 0.1 ? failRequest() : successRequest());
+          // 2% of failing requests
+          const response = await (Math.random() < 0.02 ? failRequest() : successRequest());
           expect(response.status).not.toEqual(429);
         }
 

From e6ca0bbf073d6731edf7ead2cab54f35459ec5f4 Mon Sep 17 00:00:00 2001
From: Wan Qi Chen <495709+wa0x6e@users.noreply.github.com>
Date: Tue, 14 Nov 2023 15:16:38 +0800
Subject: [PATCH 10/12] chore: sync package.json

---
 package.json | 2 +-
 yarn.lock    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/package.json b/package.json
index b174d98f..30a95a28 100644
--- a/package.json
+++ b/package.json
@@ -37,7 +37,7 @@
     "cors": "^2.8.5",
     "dotenv": "^16.0.2",
     "express": "^4.18.1",
-    "express-rate-limit": "^6.9.0",
+    "express-rate-limit": "^6.11.2",
     "lodash": "^4.17.21",
     "mysql": "^2.18.1",
     "node-fetch": "^2.7.0",
diff --git a/yarn.lock b/yarn.lock
index 19f49d3c..8c0eb46d 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2759,7 +2759,7 @@ express-prom-bundle@^6.6.0:
     on-finished "^2.3.0"
     url-value-parser "^2.0.0"
 
-express-rate-limit@^6.9.0:
+express-rate-limit@^6.11.2:
   version "6.11.2"
   resolved "https://registry.yarnpkg.com/express-rate-limit/-/express-rate-limit-6.11.2.tgz#6c42035603d3b52e4e2fb59f6ebaa89e628ef980"
   integrity sha512-a7uwwfNTh1U60ssiIkuLFWHt4hAC5yxlLGU2VP0X4YNlyEDZAqF4tK3GD3NSitVBrCQmQ0++0uOyFOgC2y4DDw==

From 25697ae9fe9b0ab50cf943311d06a3c4236aa473 Mon Sep 17 00:00:00 2001
From: Wan Qi Chen <495709+wa0x6e@users.noreply.github.com>
Date: Tue, 14 Nov 2023 15:52:45 +0800
Subject: [PATCH 11/12] chore: fix tests

---
 test/e2e/api.test.ts | 269 ++++++++++++++++++++++---------------------
 1 file changed, 139 insertions(+), 130 deletions(-)

diff --git a/test/e2e/api.test.ts b/test/e2e/api.test.ts
index cc0749bb..e8f74f7e 100644
--- a/test/e2e/api.test.ts
+++ b/test/e2e/api.test.ts
@@ -32,177 +32,186 @@ function failRequest() {
   });
 }
 
-describe('POST /', () => {
+describe('api', () => {
   beforeEach(async () => {
     const keyPrefix = process.env.RATE_LIMIT_KEYS_PREFIX || 'snapshot-sequencer:';
-    await redis.del(`${keyPrefix}rl:3e48ef9`);
-    await redis.del(`${keyPrefix}rl-s:3e48ef9`);
+    const keys = await redis.keys(`${keyPrefix}*`);
+
+    await Promise.all(keys.map(k => redis.del(k)));
   });
 
   afterAll(async () => {
     await redis.quit();
   });
 
-  describe('on invalid client input', () => {
-    it('returns a 400 error', async () => {
-      const response = await failRequest();
-      const body = await response.json();
+  describe('POST /', () => {
+    describe('on invalid client input', () => {
+      it('returns a 400 error', async () => {
+        const response = await failRequest();
+        const body = await response.json();
 
-      expect(response.status).toBe(400);
-      expect(body.error).toBe('client_error');
-      expect(body.error_description).toBe('wrong timestamp');
+        expect(response.status).toBe(400);
+        expect(body.error).toBe('client_error');
+        expect(body.error_description).toBe('wrong timestamp');
+      });
     });
-  });
 
-  describe('rate limit', () => {
-    describe('on a mix of success and failed requests', () => {
-      it('should return a 429 errors only after 100 requests / min', async () => {
-        for (let i = 1; i <= 100; i++) {
-          // 2% of failing requests
-          const response = await (Math.random() < 0.02 ? failRequest() : successRequest());
-          expect(response.status).not.toEqual(429);
-        }
-
-        const response = await fetch(HOST, {
-          method: 'POST',
-          headers: { 'Content-Type': 'application/json' },
-          body: JSON.stringify(proposalInput)
+    describe('rate limit', () => {
+      describe('on a mix of success and failed requests', () => {
+        it('should return a 429 errors only after 100 requests / min', async () => {
+          for (let i = 1; i <= 100; i++) {
+            // 2% of failing requests
+            const response = await (Math.random() < 0.02 ? failRequest() : successRequest());
+            expect(response.status).not.toEqual(429);
+          }
+
+          const response = await fetch(HOST, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify(proposalInput)
+          });
+          expect(response.status).toBe(429);
         });
-        expect(response.status).toBe(429);
       });
-    });
 
-    describe('on multiple failed requests', () => {
-      it('should return a 429 errors after 15 requests / 15s', async () => {
-        for (let i = 1; i <= 15; i++) {
-          const response = await failRequest();
-          expect(response.status).toBe(400);
-        }
+      describe('on multiple failed requests', () => {
+        it('should return a 429 errors after 15 requests / 15s', async () => {
+          for (let i = 1; i <= 15; i++) {
+            const response = await failRequest();
+            expect(response.status).toBe(400);
+          }
 
-        const response = await fetch(`${HOST}/scores/proposal-id`);
-        expect(response.status).toBe(429);
+          const response = await fetch(`${HOST}/scores/proposal-id`);
+          expect(response.status).toBe(429);
+        });
       });
     });
   });
-});
 
-describe('POST /flag', () => {
-  afterAll(() => {
-    db.endAsync();
-  });
+  describe('POST /flag', () => {
+    afterAll(() => {
+      db.endAsync();
+    });
 
-  beforeEach(async () => {
-    await db.queryAsync(
-      `
+    beforeEach(async () => {
+      await db.queryAsync(
+        `
       DELETE FROM snapshot_sequencer_test.spaces WHERE id LIKE ?;
       TRUNCATE TABLE snapshot_sequencer_test.proposals
     `,
-      [`${SPACE_PREFIX}%`]
-    );
-
-    await Promise.all(
-      spacesFixtures
-        .map(space => ({
-          ...space,
-          id: `${SPACE_PREFIX}${space.id}`,
-          settings: JSON.stringify(space.settings)
-        }))
-        .map(async space => {
-          db.queryAsync('INSERT INTO snapshot_sequencer_test.spaces SET ?', space);
-        })
-    );
-
-    await Promise.all(
-      proposalsFixtures
-        .map(proposal => ({
-          ...proposal,
-          strategies: JSON.stringify(proposal.strategies),
-          validation: JSON.stringify(proposal.validation),
-          plugins: JSON.stringify(proposal.plugins),
-          choices: JSON.stringify(proposal.choices),
-          scores: JSON.stringify(proposal.scores),
-          scores_by_strategy: JSON.stringify(proposal.scores_by_strategy)
-        }))
-        .map(async proposal => {
-          db.queryAsync('INSERT INTO snapshot_sequencer_test.proposals SET ?', proposal);
-        })
-    );
-  });
-
-  it('returns a 401 error when not authorized', async () => {
-    const response = await fetch(`${HOST}/flag`, {
-      method: 'POST',
-      headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify({})
+        [`${SPACE_PREFIX}%`]
+      );
+
+      await Promise.all(
+        spacesFixtures
+          .map(space => ({
+            ...space,
+            id: `${SPACE_PREFIX}${space.id}`,
+            settings: JSON.stringify(space.settings)
+          }))
+          .map(async space => {
+            db.queryAsync('INSERT INTO snapshot_sequencer_test.spaces SET ?', space);
+          })
+      );
+
+      await Promise.all(
+        proposalsFixtures
+          .map(proposal => ({
+            ...proposal,
+            strategies: JSON.stringify(proposal.strategies),
+            validation: JSON.stringify(proposal.validation),
+            plugins: JSON.stringify(proposal.plugins),
+            choices: JSON.stringify(proposal.choices),
+            scores: JSON.stringify(proposal.scores),
+            scores_by_strategy: JSON.stringify(proposal.scores_by_strategy)
+          }))
+          .map(async proposal => {
+            db.queryAsync('INSERT INTO snapshot_sequencer_test.proposals SET ?', proposal);
+          })
+      );
     });
 
-    expect(response.status).toBe(401);
-  });
-
-  describe('when flagging a space', () => {
-    it('does nothing when the space does not exist', async () => {
-      const beforeFlaggedSpacesCount = await getFlaggedSpacesCount();
-      const response = await flagSpace('test-not-exist.eth');
-      const body = await response.json();
+    it('returns a 401 error when not authorized', async () => {
+      const response = await fetch(`${HOST}/flag`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({})
+      });
 
-      expect(response.status).toBe(200);
-      expect(body.success).toBe(false);
-      expect(await getFlaggedSpacesCount()).toBe(beforeFlaggedSpacesCount);
+      expect(response.status).toBe(401);
     });
 
-    it('return true when the space is already flagged', async () => {
-      await db.queryAsync('UPDATE spaces SET flagged = 1 WHERE id = ?', `${SPACE_PREFIX}test.eth`);
+    describe('when flagging a space', () => {
+      it('does nothing when the space does not exist', async () => {
+        const beforeFlaggedSpacesCount = await getFlaggedSpacesCount();
+        const response = await flagSpace('test-not-exist.eth');
+        const body = await response.json();
 
-      const response = await flagSpace('test.eth');
-      const body = await response.json();
+        expect(response.status).toBe(200);
+        expect(body.success).toBe(false);
+        expect(await getFlaggedSpacesCount()).toBe(beforeFlaggedSpacesCount);
+      });
 
-      expect(response.status).toBe(200);
-      expect(body.success).toBe(true);
-      expect(await getFlaggedSpacesCount()).toBe(1);
-    });
+      it('return true when the space is already flagged', async () => {
+        await db.queryAsync(
+          'UPDATE spaces SET flagged = 1 WHERE id = ?',
+          `${SPACE_PREFIX}test.eth`
+        );
 
-    it('flags the space when it exists', async () => {
-      const response = await flagSpace('test.eth');
-      const body = await response.json();
+        const response = await flagSpace('test.eth');
+        const body = await response.json();
 
-      expect(body).toEqual({ success: true });
-      expect(
-        (await db.queryAsync('SELECT id FROM spaces WHERE flagged = 1')).map(r => r.id)
-      ).toEqual([`${SPACE_PREFIX}test.eth`]);
+        expect(response.status).toBe(200);
+        expect(body.success).toBe(true);
+        expect(await getFlaggedSpacesCount()).toBe(1);
+      });
+
+      it('flags the space when it exists', async () => {
+        const response = await flagSpace('test.eth');
+        const body = await response.json();
+
+        expect(body).toEqual({ success: true });
+        expect(
+          (await db.queryAsync('SELECT id FROM spaces WHERE flagged = 1')).map(r => r.id)
+        ).toEqual([`${SPACE_PREFIX}test.eth`]);
+      });
     });
-  });
 
-  describe('when un-flagging a space', () => {
-    it('does nothing when the space does not exist', async () => {
-      const beforeFlaggedSpacesCount = await getFlaggedSpacesCount();
+    describe('when un-flagging a space', () => {
+      it('does nothing when the space does not exist', async () => {
+        const beforeFlaggedSpacesCount = await getFlaggedSpacesCount();
 
-      const response = await flagSpace('test-not-exist.eth', 'unflag');
-      const body = await response.json();
+        const response = await flagSpace('test-not-exist.eth', 'unflag');
+        const body = await response.json();
 
-      expect(response.status).toBe(200);
-      expect(body.success).toBe(false);
-      expect(await getFlaggedSpacesCount()).toBe(beforeFlaggedSpacesCount);
-    });
+        expect(response.status).toBe(200);
+        expect(body.success).toBe(false);
+        expect(await getFlaggedSpacesCount()).toBe(beforeFlaggedSpacesCount);
+      });
 
-    it('returns true when the space is not flagged', async () => {
-      const beforeFlaggedSpacesCount = await getFlaggedSpacesCount();
-      const response = await flagSpace('test.eth', 'unflag');
-      const body = await response.json();
+      it('returns true when the space is not flagged', async () => {
+        const beforeFlaggedSpacesCount = await getFlaggedSpacesCount();
+        const response = await flagSpace('test.eth', 'unflag');
+        const body = await response.json();
 
-      expect(response.status).toBe(200);
-      expect(body.success).toBe(true);
-      expect(await getFlaggedSpacesCount()).toBe(beforeFlaggedSpacesCount);
-    });
+        expect(response.status).toBe(200);
+        expect(body.success).toBe(true);
+        expect(await getFlaggedSpacesCount()).toBe(beforeFlaggedSpacesCount);
+      });
 
-    it('un-flags the space when it is flagged', async () => {
-      await db.queryAsync('UPDATE spaces SET flagged = 1 WHERE id = ?', `${SPACE_PREFIX}test.eth`);
+      it('un-flags the space when it is flagged', async () => {
+        await db.queryAsync(
+          'UPDATE spaces SET flagged = 1 WHERE id = ?',
+          `${SPACE_PREFIX}test.eth`
+        );
 
-      const response = await flagSpace('test.eth', 'unflag');
-      const body = await response.json();
+        const response = await flagSpace('test.eth', 'unflag');
+        const body = await response.json();
 
-      expect(response.status).toBe(200);
-      expect(body.success).toBe(true);
-      expect(await getFlaggedSpacesCount()).toBe(0);
+        expect(response.status).toBe(200);
+        expect(body.success).toBe(true);
+        expect(await getFlaggedSpacesCount()).toBe(0);
+      });
     });
   });
 });

From e6703242eda982370d6a8f10d30225f3e416a7b5 Mon Sep 17 00:00:00 2001
From: Wan Qi Chen <495709+wa0x6e@users.noreply.github.com>
Date: Tue, 28 Nov 2023 15:40:17 +0400
Subject: [PATCH 12/12] fix: fix leaking redisStore key expiration

---
 src/helpers/rateLimit.ts | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/src/helpers/rateLimit.ts b/src/helpers/rateLimit.ts
index ce5daec6..73930196 100644
--- a/src/helpers/rateLimit.ts
+++ b/src/helpers/rateLimit.ts
@@ -5,6 +5,15 @@ import { getIp, sendError, sha256 } from './utils';
 
 const hashedIp = (req): string => sha256(getIp(req)).slice(0, 7);
 
+function getStore() {
+  return redisClient
+    ? new RedisStore({
+        sendCommand: (...args: string[]) => redisClient.sendCommand(args),
+        prefix: process.env.RATE_LIMIT_KEYS_PREFIX || 'snapshot-sequencer:'
+      })
+    : undefined;
+}
+
 const rateLimitConfig = {
   standardHeaders: true,
   legacyHeaders: false,
@@ -14,27 +23,22 @@ const rateLimitConfig = {
       'too many requests, refer to https://docs.snapshot.org/tools/api/api-keys#limits',
       429
     );
-  },
-  store: redisClient
-    ? new RedisStore({
-        sendCommand: (...args: string[]) => redisClient.sendCommand(args),
-        prefix: process.env.RATE_LIMIT_KEYS_PREFIX || 'snapshot-sequencer:'
-      })
-    : undefined
+  }
 };
 
-const regularRateLimit = rateLimit({
-  keyGenerator: req => `rl:${hashedIp(req)}`,
-  windowMs: 60 * 1e3,
-  max: 100,
-  ...rateLimitConfig
-});
-
 const highErroredRateLimit = rateLimit({
   keyGenerator: req => `rl-s:${hashedIp(req)}`,
   windowMs: 15 * 1e3,
   max: 15,
   skipSuccessfulRequests: true,
+  store: getStore(),
+  ...rateLimitConfig
+});
+const regularRateLimit = rateLimit({
+  keyGenerator: req => `rl:${hashedIp(req)}`,
+  windowMs: 60 * 1e3,
+  max: 100,
+  store: getStore(),
   ...rateLimitConfig
 });