more refactoring to plugins and unit tests

wip


wip, compiles


wip processing sample site


wip

Author: hatton

package.json

@@ -50,7 +50,7 @@
   "//chalk@4": "also ESM related problem",
   "//notion-client@4": "also ESM related problem",
   "dependencies": {
-    "@notionhq/client": "1",
+    "@notionhq/client": "2.2.3",
     "chalk": "^4.1.2",
     "commander": "^9.2.0",
     "cosmiconfig": "^8.0.0",
@@ -62,7 +62,6 @@
     "notion-client": "^4",
     "notion-to-md": "^2.5.5",
     "path": "^0.12.7",
-    "postinstall-postinstall": "^2.1.0",
     "sanitize-filename": "^1.6.3"
   },
   "devDependencies": {
@@ -80,7 +79,6 @@
     "eslint-plugin-prettier": "^3.4.0",
     "jest": "^28.1.3",
     "lint-staged": "^10.5.4",
-    "patch-package": "^6.5.1",
     "prettier": "^2.2.1",
     "semantic-release": "^19.0.2",
     "ts-jest": "^28.0.7",

patches/@notionhq+client+1.0.4.patch

@@ -102,7 +102,7 @@ function notionEmbedsToMDX(input: string): {
   return { body, imports: [...imports].join("\n") };
 }
 
-export const imgur: IPlugin = {
+export const imgurGifTweak: IPlugin = {
   name: "imgur",
   regexMarkdownModifications: [
     {

src/DocusaurusTweaks.ts

@@ -36,7 +36,7 @@ export class HierarchicalNamedLayoutStrategy extends LayoutStrategy {
       .replaceAll("'", "")
       .replaceAll("?", "-");
 
-    const context = ("/" + page.context + "/").replaceAll("//", "/");
+    const context = ("/" + page.layoutContext + "/").replaceAll("//", "/");
     const path =
       this.rootDirectory + context + sanitizedName + extensionWithDot;
 

src/HierarchicalNamedLayoutStrategy.ts

@@ -4,18 +4,11 @@ import {
   GetPageResponse,
   ListBlockChildrenResponse,
 } from "@notionhq/client/build/src/api-endpoints";
-import { RateLimiter } from "limiter";
 import { Client } from "@notionhq/client";
 import { logDebug } from "./log";
-import { parseLinkId } from "./links";
+import { parseLinkId } from "./plugins/internalLinks";
 import { info } from "console";
-
-const notionLimiter = new RateLimiter({
-  tokensPerInterval: 3,
-  interval: "second",
-});
-
-let notionClient: Client;
+import { ListBlockChildrenResponseResults } from "notion-to-md/build/types";
 
 // Notion has 2 kinds of pages: a normal one which is just content, and what I'm calling a "database page", which has whatever properties you put on it.
 // docu-notion supports the later via links from outline pages. That is, you put the database pages in a database, then separately, in the outline, you
@@ -25,55 +18,50 @@ export enum PageType {
   DatabasePage,
   Simple,
 }
-export function initNotionClient(notionToken: string): Client {
-  notionClient = new Client({
-    auth: notionToken,
-  });
-  return notionClient;
-}
 
 export class NotionPage {
-  private metadata: GetPageResponse;
-  public readonly pageId: string;
-  public readonly order: number;
-  public context: string; // where we found it in the hierarchy of the outline
+  public metadata: GetPageResponse;
+  public pageId: string;
+  public order: number;
+  public layoutContext: string; // where we found it in the hierarchy of the outline
   public foundDirectlyInOutline: boolean; // the page was found as a descendent of /outline instead of being linked to
 
-  private constructor(
-    context: string,
-    pageId: string,
-    order: number,
-    metadata: GetPageResponse,
-    foundDirectlyInOutline: boolean
-  ) {
-    this.context = context;
-    this.pageId = pageId;
-    this.order = order;
-    this.metadata = metadata;
-    this.foundDirectlyInOutline = foundDirectlyInOutline;
+  // public constructor(
+  //   layoutContext: string,
+  //   pageId: string,
+  //   order: number,
+  //   metadata: GetPageResponse,
+  //   foundDirectlyInOutline: boolean
+  // ) {
+  //   this.layoutContext = layoutContext;
+  //   this.pageId = pageId;
+  //   this.order = order;
+  //   this.metadata = metadata;
+  //   this.foundDirectlyInOutline = foundDirectlyInOutline;
+
+  //   // review: this is expensive to learn as it takes another api call... I
+  //   // think? We can tell if it's a database because it has a "Name" instead of a
+  //   // "tile" and "parent": "type": "database_id". But do we need to differentiate
+  //   //this.type = PageType.Unknown;
+  // }
+  public constructor(args: {
+    layoutContext: string;
+    pageId: string;
+    order: number;
+    metadata: GetPageResponse;
+    foundDirectlyInOutline: boolean;
+  }) {
+    this.layoutContext = args.layoutContext;
+    this.pageId = args.pageId;
+    this.order = args.order;
+    this.metadata = args.metadata;
+    this.foundDirectlyInOutline = args.foundDirectlyInOutline;
 
     // review: this is expensive to learn as it takes another api call... I
     // think? We can tell if it's a database because it has a "Name" instead of a
     // "tile" and "parent": "type": "database_id". But do we need to differentiate
     //this.type = PageType.Unknown;
   }
-  public static async fromPageId(
-    context: string,
-    pageId: string,
-    order: number,
-    foundDirectlyInOutline: boolean
-  ): Promise<NotionPage> {
-    const metadata = await getPageMetadata(pageId);
-
-    //logDebug("notion metadata", JSON.stringify(metadata));
-    return new NotionPage(
-      context,
-      pageId,
-      order,
-      metadata,
-      foundDirectlyInOutline
-    );
-  }
 
   public matchesLinkId(id: string): boolean {
     const { baseLinkId } = parseLinkId(id);
@@ -172,14 +160,6 @@ export class NotionPage {
     return this.getSelectProperty("Status");
   }
 
-  private async getChildren(): Promise<ListBlockChildrenResponse> {
-    const children = await notionClient.blocks.children.list({
-      block_id: this.pageId,
-      page_size: 100, // max hundred links in a page
-    });
-    return children;
-  }
-
   public getPlainTextProperty(
     property: string,
     defaultIfEmpty: string
@@ -244,68 +224,28 @@ export class NotionPage {
     return p.select?.name || undefined;
   }
 
-  public async getBlockChildren(): Promise<ListBlockChildrenResponse> {
-    // we can only get so many responses per call, so we set this to
-    // the first response we get, then keep adding to its array of blocks
-    // with each subsequent response
-    let overallResult: ListBlockChildrenResponse | undefined = undefined;
-    let start_cursor = undefined;
-
-    do {
-      await rateLimit();
-
-      const response: ListBlockChildrenResponse =
-        await notionClient.blocks.children.list({
-          start_cursor: start_cursor,
-          block_id: this.pageId,
-        });
-      if (!overallResult) {
-        overallResult = response;
-      } else {
-        overallResult.results.push(...response.results);
-      }
-
-      start_cursor = response?.next_cursor;
-    } while (start_cursor != null);
-    return overallResult;
-  }
-
-  public async getContentInfo(): Promise<{
+  public async getContentInfo(
+    children: ListBlockChildrenResponseResults
+  ): Promise<{
     childPageIdsAndOrder: { id: string; order: number }[];
     linksPageIdsAndOrder: { id: string; order: number }[];
     hasParagraphs: boolean;
   }> {
-    const children = await this.getChildren();
-    for (let i = 0; i < children.results.length; i++) {
-      (children.results[i] as any).order = i;
+    for (let i = 0; i < children.length; i++) {
+      (children[i] as any).order = i;
     }
     return {
-      childPageIdsAndOrder: children.results
+      childPageIdsAndOrder: children
         .filter((b: any) => b.type === "child_page")
         .map((b: any) => ({ id: b.id, order: b.order })),
-      linksPageIdsAndOrder: children.results
+      linksPageIdsAndOrder: children
         .filter((b: any) => b.type === "link_to_page")
         .map((b: any) => ({ id: b.link_to_page.page_id, order: b.order })),
-      hasParagraphs: children.results.some(
+      hasParagraphs: children.some(
         b =>
           (b as any).type === "paragraph" &&
           (b as any).paragraph.rich_text.length > 0
       ),
     };
   }
 }
-
-async function getPageMetadata(id: string): Promise<GetPageResponse> {
-  await rateLimit();
-
-  return await notionClient.pages.retrieve({
-    page_id: id,
-  });
-}
-
-async function rateLimit() {
-  if (notionLimiter.getTokensRemaining() < 1) {
-    logDebug("", "*** delaying for rate limit");
-  }
-  await notionLimiter.removeTokens(1);
-}