From 74735260bb06591de29d848fc9c5e4c40e5f770f Mon Sep 17 00:00:00 2001
From: Nulo <git@nulo.in>
Date: Mon, 27 Jan 2025 10:15:53 -0300
Subject: [PATCH] fix dataset fetch for index.md generation

---
 sepa/index-gen/b2.ts | 26 ++++++++++++++++++--------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/sepa/index-gen/b2.ts b/sepa/index-gen/b2.ts
index 79c15a1..415b46e 100644
--- a/sepa/index-gen/b2.ts
+++ b/sepa/index-gen/b2.ts
@@ -36,18 +36,28 @@ function getS3Client() {
     },
   });
 }
-
 export async function listDirectory(directoryName: string) {
   const { B2_BUCKET_NAME } = getVariables();
   const s3 = getS3Client();
-  const command = new ListObjectsV2Command({
-    Bucket: B2_BUCKET_NAME,
-    Prefix: directoryName ? `${directoryName}/` : "",
-    Delimiter: "/",
-  });
+  let allContents: string[] = [];
+  let continuationToken: string | undefined;
 
-  const response = await s3.send(command);
-  return response.Contents?.map((item) => item.Key ?? "") ?? [];
+  do {
+    const command = new ListObjectsV2Command({
+      Bucket: B2_BUCKET_NAME,
+      Prefix: directoryName ? `${directoryName}/` : "",
+      Delimiter: "/",
+      ContinuationToken: continuationToken,
+    });
+
+    const response = await s3.send(command);
+    allContents = allContents.concat(
+      response.Contents?.map((item) => item.Key ?? "") ?? []
+    );
+    continuationToken = response.NextContinuationToken;
+  } while (continuationToken);
+
+  return allContents;
 }
 
 export async function getFileContent(fileName: string) {