fix(models): Handle RangeError in sizeof for large objects

packages/models/src/util.js

@@ -350,7 +350,86 @@ export function buildLabels(classMap, events) {
 // sizeof returns a naive byte count for an object when serialized.
 // I was using an external library for this (object-sizeof), but getting results off by a factor of
 // ~2. This is awfully wasteful, slow and inaccurate but it works for now. -DB
-export const sizeof = (obj) => JSON.stringify(obj).length;
+export const sizeof = (obj) => {
+  try {
+    return JSON.stringify(obj).length;
+  } catch (e) {
+    // In case of large objects (e.g. ~500MB), JSON.stringify might fail with RangeError: Invalid string length.
+    // In that case, we fall back to a recursive calculation.
+    if (e instanceof RangeError) {
+      const seen = new WeakSet();
+
+      const recursiveSize = (value) => {
+        try {
+          const s = JSON.stringify(value);
+          // undefined returns undefined. The loops shouldn't pass undefined, but if they do, handling it is safe.
+          if (s === undefined) return 0;
+          return s.length;
+        } catch (err) {
+          if (!(err instanceof RangeError)) throw err;
+        }
+
+        if (value === null) return 4;
+        if (typeof value !== 'object') {
+          // Fallback for huge strings that failed stringify
+          if (typeof value === 'string') {
+            return value.length + 2; // +2 for quotes
+          }
+          return String(value).length;
+        }
+
+        if (seen.has(value)) {
+          throw new TypeError('Converting circular structure to JSON');
+        }
+        seen.add(value);
+
+        if (Array.isArray(value)) {
+          let size = 2; // []
+          if (value.length > 0) {
+            size += value.length - 1; // commas
+            for (let i = 0; i < value.length; i += 1) {
+              const item = value[i];
+              // In arrays, undefined, functions, and symbols are converted to null
+              if (item === undefined || typeof item === 'function' || typeof item === 'symbol') {
+                size += 4; // "null"
+              } else {
+                size += recursiveSize(item);
+              }
+            }
+          }
+          return size;
+        }
+
+        // Generic Object
+        let size = 2; // {}
+        const keys = Object.keys(value);
+        let addedProps = 0;
+        for (let i = 0; i < keys.length; i += 1) {
+          const k = keys[i];
+          const v = value[k];
+
+          // In objects, properties with undefined, function, or symbol values are omitted
+          if (v === undefined || typeof v === 'function' || typeof v === 'symbol') {
+            continue;
+          }
+
+          if (addedProps > 0) {
+            size += 1; // comma
+          }
+
+          size += JSON.stringify(k).length; // "key"
+          size += 1; // :
+          size += recursiveSize(v);
+          addedProps += 1;
+        }
+        return size;
+      };
+
+      return recursiveSize(obj);
+    }
+    throw e;
+  }
+};
 
 // Returns a unique 'hash' (or really, a key) tied to the event's core identity: SQL, HTTP, or a
 // specific method on a specific class. This is _really_ naive. The idea is that this better finds

packages/models/tests/unit/sizeof.spec.js

@@ -0,0 +1,110 @@
+import { sizeof } from '../../src/util';
+
+describe('sizeof', () => {
+  it('calculates size of simple objects', () => {
+    const obj = { a: 1 };
+    // {"a":1} -> 7 chars
+    expect(sizeof(obj)).toEqual(7);
+  });
+
+  it('calculates size of arrays', () => {
+    const arr = [1, 2];
+    // [1,2] -> 5 chars
+    expect(sizeof(arr)).toEqual(5);
+  });
+
+  it('handles nested objects', () => {
+    const obj = { a: { b: 1 } };
+    // {"a":{"b":1}} -> 13 chars
+    expect(sizeof(obj)).toEqual(13);
+  });
+
+  describe('fallback behavior', () => {
+    let stringifySpy;
+    const originalStringify = JSON.stringify;
+
+    beforeEach(() => {
+      // Mock JSON.stringify to throw RangeError for objects with a specific flag
+      stringifySpy = jest.spyOn(JSON, 'stringify').mockImplementation((val) => {
+        if (val && typeof val === 'object' && val !== null && val.forceFail) {
+          throw new RangeError('Invalid string length: forced by test');
+        }
+        return originalStringify(val);
+      });
+    });
+
+    afterEach(() => {
+      stringifySpy.mockRestore();
+    });
+
+    it('falls back to recursive calculation on RangeError', () => {
+      const hugeObj = {
+        forceFail: true,
+        a: 1,
+        b: [2, 3],
+      };
+      // We manually calculate expected size or temporarily bypass the mock
+      // {"forceFail":true,"a":1,"b":[2,3]} length is 34
+      expect(sizeof(hugeObj)).toEqual(34);
+      expect(stringifySpy).toHaveBeenCalled();
+    });
+
+    it('correctly calculates size of strings with escaped characters', () => {
+      const obj = {
+        forceFail: true,
+        str: 'hello "world"\n',
+      };
+
+      const expected = originalStringify(obj).length;
+
+      expect(sizeof(obj)).toEqual(expected);
+    });
+
+    it('handles arrays with undefined, null, functions', () => {
+      const obj = {
+        forceFail: true,
+        arr: [undefined, null, () => {}, 123],
+      };
+
+      const expected = originalStringify(obj).length;
+      expect(sizeof(obj)).toEqual(expected);
+    });
+
+    it('handles objects with undefined, null, functions', () => {
+      const obj = {
+        forceFail: true,
+        a: undefined,
+        b: null,
+        c: () => {},
+        d: 123,
+      };
+
+      const expected = originalStringify(obj).length;
+      expect(sizeof(obj)).toEqual(expected);
+    });
+
+    it('detects circular references and throws TypeError', () => {
+      const obj = {
+        forceFail: true,
+      };
+      obj.self = obj;
+
+      expect(() => sizeof(obj)).toThrow(TypeError);
+      expect(() => sizeof(obj)).toThrow('Converting circular structure to JSON');
+    });
+
+    it('handles deep nesting', () => {
+      const obj = {
+        forceFail: true,
+        level1: {
+          level2: {
+            level3: [1, 2, 3],
+          },
+        },
+      };
+
+      const expected = originalStringify(obj).length;
+      expect(sizeof(obj)).toEqual(expected);
+    });
+  });
+});

packages/validate/src/build.js

@@ -2,7 +2,7 @@ const FileSystem = require('fs');
 const Semver = require('semver');
 const YAML = require('yaml');
 
-const lines = FileSystem.readFileSync(`${__dirname}/macro.yml`, 'utf8').split('\n');
+const lines = FileSystem.readFileSync(`${__dirname}/macro.yml`, 'utf8').split(/\r?\n/);
 
 const versions = ['1.13.1', '1.13.0', '1.12.0', '1.11.0', '1.10.0', '1.9.0', '1.8.0', '1.7.0', '1.6.0', '1.5.1', '1.5.0', '1.4.1', '1.4.0', '1.3.0', '1.2.0'];