Initial commit

2021-11-25 13:27:33 +01:00 · 2021-11-25 13:27:33 +01:00 · ea6e7a8ece
commit ea6e7a8ece
11 changed files with 520 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@ -0,0 +1,2 @@
+/dist
+/node_modules
--- a/21
+++ b/21
@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Ribose
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
--- a/README.adoc
+++ b/README.adoc
@ -0,0 +1,21 @@
+Aspirationally, it is a set of low-level helpers
+to simplify working with Git LFS via Isomorphic Git.
+
+As of 0.1.0, support is limited to:
+
+- `readPointer({ dir, gitdir, content })`
+
+where `dir`, `gitdir` behavior mimics that of Isomorphic Git,
+and content is a `Buffer`.
+
+
+- `downloadBlobFromPointer({ http, headers, url }, lfsPointer)`
+
+where `http` is an `HttpClient` as supported by Isomorphic Git,
+URL is repository URL
+and pointer is an object returned by `readPointer()`.
+
+- `populateCache(workDir, ref?)`
+
+where `workDir` is a path to working directory,
+and `ref` should probably be left at the default `"HEAD"`.
--- a/package.json
+++ b/package.json
@ -0,0 +1,32 @@
+{
+  "name": "@riboseinc/isogit-lfs",
+  "version": "0.1.0",
+  "description": "LFS helpers for Isomorphic Git (Node only)",
+  "main": "index.js",
+  "repository": "git@github.com:riboseinc/isogit-lfs.git",
+  "scripts": {
+    "build": "rm -r dist; mkdir dist; tsc --outDir dist; cp package.json README.adoc dist/"
+  },
+  "files": [
+    "README.adoc",
+    "*.js",
+    "*.js.map",
+    "*.d.ts",
+    "*/**/*.js",
+    "*/**/*.js.map",
+    "*/**/*.d.ts"
+  ],
+  "author": {
+    "name": "Ribose Inc.",
+    "email": "open.source@ribose.com"
+  },
+  "peerDependencies": {
+    "isomorphic-git": "^1.7.8"
+  },
+  "devDependencies": {
+    "@types/node": "^16.11.7",
+    "isomorphic-git": "^1.7.8",
+    "typescript": "^4.4.2"
+  },
+  "license": "MIT"
+}
--- a/src/download.ts
+++ b/src/download.ts
@ -0,0 +1,74 @@
+import git from 'isomorphic-git';
+import { HttpClient } from 'isomorphic-git/http/node';
+
+import { bodyToBuffer } from './util';
+import { Pointer } from './pointers';
+
+
+interface DownloadBlobRequset {
+  http: HttpClient;
+  headers?: Record<string, any>;
+
+  /** Repository URL. */
+  url: string;
+}
+
+interface LFSInfoResponse {
+  objects: {
+    actions: {
+      download: {
+        href: string;
+      };
+    };
+  }[];
+}
+
+function isValidLFSInfoResponseData(val: Record<string, any>): val is LFSInfoResponse {
+  return val.objects?.[0]?.actions?.download?.href?.trim !== undefined;
+}
+
+
+/** Downloads a blob corresponding to given LFS pointer. */
+export default async function downloadBlobFromPointer(
+  { http: { request }, headers = {}, url }: DownloadBlobRequset,
+  { info, objectPath }: Pointer
+): Promise<Buffer> {
+  // Request LFS metadata
+
+  const lfsInfoRequestData = {
+    operation: 'download',
+    transfers: ['basic'],
+    objects: [info],
+  };
+
+  const { body: lfsInfoBody } = await request({
+    url: `${url}/info/lfs/objects/batch`,
+    method: 'POST',
+    headers: {
+      'User-Agent': `git/isomorphic-git@${git.version()}`,
+      ...headers,
+      'Accept': 'application/vnd.git-lfs+json',
+      'Content-Type': 'application/vnd.git-lfs+json',
+    },
+    body: [Buffer.from(JSON.stringify(lfsInfoRequestData))],
+  });
+
+  const lfsInfoResponseData = JSON.parse((await bodyToBuffer(lfsInfoBody)).toString());
+
+  if (isValidLFSInfoResponseData(lfsInfoResponseData)) {
+
+    // Request the actual blob
+    const lfsObjectDownloadURL = lfsInfoResponseData.objects[0].actions.download.href;
+
+    const { body: lfsObjectBody } = await request({
+      url: lfsObjectDownloadURL,
+      method: 'GET',
+      headers,
+    });
+
+    return await bodyToBuffer(lfsObjectBody);
+
+  } else {
+    throw new Error("LFS response didn’t return an expected structure");
+  }
+}
--- a/src/index.ts
+++ b/src/index.ts
@ -0,0 +1,3 @@
+export { default as downloadBlobFromPointer } from './download';
+export { default as populateCache } from './populateCache';
+export { readPointer } from './pointers';
--- a/src/pointers.ts
+++ b/src/pointers.ts
@ -0,0 +1,52 @@
+import path from 'path';
+
+
+interface PointerInfo {
+  oid: string;
+  size: string;
+}
+
+export interface Pointer {
+  info: PointerInfo;
+
+  /** Path to blob in LFS cache. */
+  objectPath: string;
+}
+
+function isValidPointerInfo(val: Record<string, any>): val is PointerInfo {
+  return val.oid.trim !== undefined && val.oid.size !== undefined;
+}
+
+interface PointerRequest {
+  dir: string;
+  gitdir?: string;
+  content: Buffer;
+}
+
+
+export function readPointer({ dir, gitdir = path.join(dir, '.git'), content }: PointerRequest): Pointer {
+  const info = content.toString().trim().split('\n').reduce((accum, line) => {
+    const [k, v] = line.split(' ', 2);
+    if (k === 'oid') {
+      accum[k] = v.split(':', 2)[1];
+    } else if (k === 'size') {
+      accum[k] = v;
+    }
+    return accum;
+  }, {} as Record<string, any>);
+
+  if (isValidPointerInfo(info)) {
+    const objectPath = path.join(
+      gitdir,
+      'lfs',
+      'objects',
+      info.oid.substr(0, 2),
+      info.oid.substr(2, 2),
+      info.oid);
+
+    return { info, objectPath };
+
+  } else {
+    throw new Error("LFS pointer is incomplete or cannot be read");
+  }
+}
--- a/src/populateCache.ts
+++ b/src/populateCache.ts
@ -0,0 +1,79 @@
+import path from 'path';
+import fs from 'fs/promises';
+
+import git from 'isomorphic-git';
+import http from 'isomorphic-git/http/node';
+
+import { isWriteable, pointsToLFS } from './util';
+import downloadBlobFromPointer from './download';
+import { readPointer } from "./pointers";
+
+
+const SYMLINK_MODE = 40960;
+
+
+/**
+ * Populates LFS cache for each repository object that is an LFS pointer.
+ *
+ * Does not touch working directory.
+ *
+ * NOTE: If LFS cache path, as extracted from the pointer,
+ * is not writeable at the time of download start,
+ * the object will be silently skipped;
+ * if LFS cache path is not writeable at the time download completes,
+ * an error will be thrown.
+ */
+export default async function populateCache(workDir: string, ref: string = 'HEAD') {
+  const remoteURL = await git.getConfig({
+    fs,
+    dir: workDir,
+    path: 'remote.origin.url',
+  });
+  if (remoteURL) {
+    await git.walk({
+      fs,
+      dir: workDir,
+      trees: [git.TREE({ ref })],
+      map: async (filepath, entries) => {
+        if (entries === null || entries[0] === null) {
+          return null;
+        }
+
+        const [entry] = entries;
+        const entryType = await entry.type();
+
+        if (entryType === 'tree') {
+          // Walk children
+          return true;
+
+        } else if (entryType === 'blob' && (await entry.mode()) !== SYMLINK_MODE) {
+          const content = await entry.content();
+
+          if (content) {
+            const buff = Buffer.from(content.buffer);
+
+            if (pointsToLFS(buff)) {
+              const pointer = readPointer({ dir: workDir, content: buff });
+
+              // Don’t even start the download if LFS cache path is not accessible.
+              if (await isWriteable(pointer.objectPath) === false)
+                return;
+
+              const content = await downloadBlobFromPointer(
+                { http, url: remoteURL },
+                pointer);
+
+              // Write LFS cache for this object, if cache path is still accessible.
+              if (await isWriteable(pointer.objectPath) === false)
+                return;
+
+              await fs.mkdir(path.dirname(pointer.objectPath), { recursive: true });
+              await fs.writeFile(pointer.objectPath, content);
+            }
+          }
+        }
+        return;
+      }
+    });
+  }
+}
--- a/src/util.ts
+++ b/src/util.ts
@ -0,0 +1,47 @@
+import fs from 'fs/promises';
+import { constants as fsConstants } from 'fs';
+
+export const LFS_POINTER_PREAMBLE = 'version https://git-lfs.github.com/spec/v1\n';
+
+
+export function pointsToLFS(content: Buffer): boolean {
+  return (
+    content[0] === 118 // 'v'
+    && content.subarray(0, 100).indexOf(LFS_POINTER_PREAMBLE) === 0);
+}
+
+
+/**
+ * Returns true if given path is available for writing,
+ * regardless of whether or not it is occupied.
+ */
+export async function isWriteable(filepath: string): Promise<boolean> {
+  try {
+    await fs.access(filepath, fsConstants.W_OK);
+    return true;
+  } catch (e) {
+    if ((e as { code: string }).code !== 'ENOENT') {
+      return false;
+    } else {
+      return true;
+    }
+  }
+}
+
+
+export async function bodyToBuffer(body: Uint8Array[]): Promise<Buffer> {
+  const buffers = [];
+  let offset = 0;
+  let size = 0;
+  for await (const chunk of body) {
+    buffers.push(chunk);
+    size += chunk.byteLength;
+  }
+
+  const result = new Uint8Array(size);
+  for (const buffer of buffers) {
+    result.set(buffer, offset);
+    offset += buffer.byteLength;
+  }
+  return Buffer.from(result.buffer);
+}
--- a/tsconfig.json
+++ b/tsconfig.json
@ -0,0 +1,22 @@
+{
+    "compilerOptions": {
+      "target": "es2017",
+      "module": "esnext",
+      "moduleResolution": "node",
+
+      "strict": true,
+      "noUnusedLocals": true,
+      "noFallthroughCasesInSwitch": true,
+      "noImplicitReturns": true,
+
+      "sourceMap": true,
+      "inlineSources": true,
+
+      "allowSyntheticDefaultImports": true,
+      "experimentalDecorators": true,
+
+      "newLine": "lf",
+
+      "declaration": true
+    }
+}
--- a/yarn.lock
+++ b/yarn.lock
@ -0,0 +1,167 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"@types/node@^16.11.7":
+  version "16.11.10"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-16.11.10.tgz#2e3ad0a680d96367103d3e670d41c2fed3da61ae"
+  integrity sha512-3aRnHa1KlOEEhJ6+CvyHKK5vE9BcLGjtUpwvqYLRvYNQKMfabu3BwfJaA/SLW8dxe28LsNDjtHwePTuzn3gmOA==
+
+async-lock@^1.1.0:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/async-lock/-/async-lock-1.3.0.tgz#0fba111bea8b9693020857eba4f9adca173df3e5"
+  integrity sha512-8A7SkiisnEgME2zEedtDYPxUPzdv3x//E7n5IFktPAtMYSEAV7eNJF0rMwrVyUFj6d/8rgajLantbjcNRQYXIg==
+
+clean-git-ref@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/clean-git-ref/-/clean-git-ref-2.0.1.tgz#dcc0ca093b90e527e67adb5a5e55b1af6816dcd9"
+  integrity sha512-bLSptAy2P0s6hU4PzuIMKmMJJSE6gLXGH1cntDu7bWJUksvuM+7ReOK61mozULErYvP6a15rnYl0zFDef+pyPw==
+
+crc-32@^1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/crc-32/-/crc-32-1.2.0.tgz#cb2db6e29b88508e32d9dd0ec1693e7b41a18208"
+  integrity sha512-1uBwHxF+Y/4yF5G48fwnKq6QsIXheor3ZLPT80yGBV1oEUwpPojlEhQbWKVw1VwcTQyMGHK1/XMmTjmlsmTTGA==
+  dependencies:
+    exit-on-epipe "~1.0.1"
+    printj "~1.1.0"
+
+decompress-response@^4.2.0:
+  version "4.2.1"
+  resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-4.2.1.tgz#414023cc7a302da25ce2ec82d0d5238ccafd8986"
+  integrity sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==
+  dependencies:
+    mimic-response "^2.0.0"
+
+diff3@0.0.3:
+  version "0.0.3"
+  resolved "https://registry.yarnpkg.com/diff3/-/diff3-0.0.3.tgz#d4e5c3a4cdf4e5fe1211ab42e693fcb4321580fc"
+  integrity sha1-1OXDpM305f4SEatC5pP8tDIVgPw=
+
+exit-on-epipe@~1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/exit-on-epipe/-/exit-on-epipe-1.0.1.tgz#0bdd92e87d5285d267daa8171d0eb06159689692"
+  integrity sha512-h2z5mrROTxce56S+pnvAV890uu7ls7f1kEvVGJbw1OlFH3/mlJ5bkXu0KRyW94v37zzHPiUd55iLn3DA7TjWpw==
+
+ignore@^5.1.4:
+  version "5.1.9"
+  resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.1.9.tgz#9ec1a5cbe8e1446ec60d4420060d43aa6e7382fb"
+  integrity sha512-2zeMQpbKz5dhZ9IwL0gbxSW5w0NK/MSAMtNuhgIHEPmaU3vPdKPL0UdvUCXs5SS4JAwsBxysK5sFMW8ocFiVjQ==
+
+inherits@^2.0.1, inherits@^2.0.3:
+  version "2.0.4"
+  resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
+  integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==
+
+isomorphic-git@^1.7.8:
+  version "1.10.1"
+  resolved "https://registry.yarnpkg.com/isomorphic-git/-/isomorphic-git-1.10.1.tgz#2f3a3d2d41baf6a88e046a21e0527bc13a21d663"
+  integrity sha512-abbPpKkykIVDJ92rtYoD4AOuT5/7PABHR2fDBrsm7H0r2ZT+MGpPL/FynrEJM6nTcFSieaIDxnHNGhfHO/v+bA==
+  dependencies:
+    async-lock "^1.1.0"
+    clean-git-ref "^2.0.1"
+    crc-32 "^1.2.0"
+    diff3 "0.0.3"
+    ignore "^5.1.4"
+    minimisted "^2.0.0"
+    pako "^1.0.10"
+    pify "^4.0.1"
+    readable-stream "^3.4.0"
+    sha.js "^2.4.9"
+    simple-get "^3.0.2"
+
+mimic-response@^2.0.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-2.1.0.tgz#d13763d35f613d09ec37ebb30bac0469c0ee8f43"
+  integrity sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==
+
+minimist@^1.2.5:
+  version "1.2.5"
+  resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602"
+  integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==
+
+minimisted@^2.0.0:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/minimisted/-/minimisted-2.0.1.tgz#d059fb905beecf0774bc3b308468699709805cb1"
+  integrity sha512-1oPjfuLQa2caorJUM8HV8lGgWCc0qqAO1MNv/k05G4qslmsndV/5WdNZrqCiyqiz3wohia2Ij2B7w2Dr7/IyrA==
+  dependencies:
+    minimist "^1.2.5"
+
+once@^1.3.1:
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1"
+  integrity sha1-WDsap3WWHUsROsF9nFC6753Xa9E=
+  dependencies:
+    wrappy "1"
+
+pako@^1.0.10:
+  version "1.0.11"
+  resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf"
+  integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==
+
+pify@^4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/pify/-/pify-4.0.1.tgz#4b2cd25c50d598735c50292224fd8c6df41e3231"
+  integrity sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==
+
+printj@~1.1.0:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/printj/-/printj-1.1.2.tgz#d90deb2975a8b9f600fb3a1c94e3f4c53c78a222"
+  integrity sha512-zA2SmoLaxZyArQTOPj5LXecR+RagfPSU5Kw1qP+jkWeNlrq+eJZyY2oS68SU1Z/7/myXM4lo9716laOFAVStCQ==
+
+readable-stream@^3.4.0:
+  version "3.6.0"
+  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198"
+  integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==
+  dependencies:
+    inherits "^2.0.3"
+    string_decoder "^1.1.1"
+    util-deprecate "^1.0.1"
+
+safe-buffer@^5.0.1, safe-buffer@~5.2.0:
+  version "5.2.1"
+  resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
+  integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
+
+sha.js@^2.4.9:
+  version "2.4.11"
+  resolved "https://registry.yarnpkg.com/sha.js/-/sha.js-2.4.11.tgz#37a5cf0b81ecbc6943de109ba2960d1b26584ae7"
+  integrity sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==
+  dependencies:
+    inherits "^2.0.1"
+    safe-buffer "^5.0.1"
+
+simple-concat@^1.0.0:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/simple-concat/-/simple-concat-1.0.1.tgz#f46976082ba35c2263f1c8ab5edfe26c41c9552f"
+  integrity sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==
+
+simple-get@^3.0.2:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/simple-get/-/simple-get-3.1.0.tgz#b45be062435e50d159540b576202ceec40b9c6b3"
+  integrity sha512-bCR6cP+aTdScaQCnQKbPKtJOKDp/hj9EDLJo3Nw4y1QksqaovlW/bnptB6/c1e+qmNIDHRK+oXFDdEqBT8WzUA==
+  dependencies:
+    decompress-response "^4.2.0"
+    once "^1.3.1"
+    simple-concat "^1.0.0"
+
+string_decoder@^1.1.1:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e"
+  integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==
+  dependencies:
+    safe-buffer "~5.2.0"
+
+typescript@^4.4.2:
+  version "4.5.2"
+  resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.5.2.tgz#8ac1fba9f52256fdb06fb89e4122fa6a346c2998"
+  integrity sha512-5BlMof9H1yGt0P8/WF+wPNw6GfctgGjXp5hkblpyT+8rkASSmkUKMXrxR0Xg8ThVCi/JnHQiKXeBaEwCeQwMFw==
+
+util-deprecate@^1.0.1:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
+  integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=
+
+wrappy@1:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
+  integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=