Compare commits

...

5 commits

10 changed files with 448 additions and 1880 deletions

View file

@ -1,10 +0,0 @@
{
"presets": [[
"@babel/preset-env",
{
"targets": {
"electron": "10"
}
}
]]
}

View file

@ -1,29 +1,24 @@
{
"name": "@nulo/isogit-lfs",
"version": "0.1.6",
"description": "LFS helpers for Isomorphic Git (Node only)",
"main": "index.js",
"version": "0.2.0",
"description": "LFS helpers for Isomorphic Git",
"main": "src/index.ts",
"repository": "git@github.com:riboseinc/isogit-lfs.git",
"scripts": {},
"files": [
"README.adoc",
"*.js",
"*.js.map",
"*.d.ts",
"*/**/*.js",
"*/**/*.js.map",
"*/**/*.d.ts"
],
"author": {
"name": "Ribose Inc.",
"email": "open.source@ribose.com"
},
"dependencies": {
"@aws-crypto/sha256-universal": "^2.0.2",
"buffer": "^6.0.3",
"path-browserify": "^1.0.1"
},
"peerDependencies": {
"isomorphic-git": "^1.7.8",
"@aws-crypto/sha256-universal": "^2.0.0"
"isomorphic-git": "^1.7.8"
},
"devDependencies": {
"@aws-crypto/sha256-universal": "^2.0.0",
"@types/path-browserify": "^1.0.0",
"isomorphic-git": "^1.7.8",
"typescript": "^4.4.2"
},

279
pnpm-lock.yaml Normal file
View file

@ -0,0 +1,279 @@
lockfileVersion: 5.4
specifiers:
'@aws-crypto/sha256-universal': ^2.0.2
'@types/path-browserify': ^1.0.0
buffer: ^6.0.3
isomorphic-git: ^1.7.8
path-browserify: ^1.0.1
typescript: ^4.4.2
dependencies:
'@aws-crypto/sha256-universal': 2.0.2
buffer: 6.0.3
path-browserify: 1.0.1
devDependencies:
'@types/path-browserify': 1.0.0
isomorphic-git: 1.21.0
typescript: 4.9.3
packages:
/@aws-crypto/ie11-detection/2.0.2:
resolution: {integrity: sha512-5XDMQY98gMAf/WRTic5G++jfmS/VLM0rwpiOpaainKi4L0nqWMSB1SzsrEG5rjFZGYN6ZAefO+/Yta2dFM0kMw==}
dependencies:
tslib: 1.14.1
dev: false
/@aws-crypto/sha256-browser/2.0.2:
resolution: {integrity: sha512-V7nEV6nKYHqiWVksjQ/BnIppDHrvALDrLoL9lsxvhn/iVo77L7zGLjR+/+nFFvqg/EUz/AJr7YnVGimf1e9X7Q==}
dependencies:
'@aws-crypto/ie11-detection': 2.0.2
'@aws-crypto/sha256-js': 2.0.2
'@aws-crypto/supports-web-crypto': 2.0.2
'@aws-crypto/util': 2.0.2
'@aws-sdk/types': 3.224.0
'@aws-sdk/util-locate-window': 3.208.0
'@aws-sdk/util-utf8-browser': 3.188.0
tslib: 1.14.1
dev: false
/@aws-crypto/sha256-js/2.0.2:
resolution: {integrity: sha512-iXLdKH19qPmIC73fVCrHWCSYjN/sxaAvZ3jNNyw6FclmHyjLKg0f69WlC9KTnyElxCR5MO9SKaG00VwlJwyAkQ==}
dependencies:
'@aws-crypto/util': 2.0.2
'@aws-sdk/types': 3.224.0
tslib: 1.14.1
dev: false
/@aws-crypto/sha256-universal/2.0.2:
resolution: {integrity: sha512-BhTq7ZpAKU8IXCp+SMWj0lhckuxUZztgT/dUpiPVdW1yRlaK+OnxXB/dKV5Jw3zpf8S6+4C42LV8uMFMjBbUuw==}
dependencies:
'@aws-crypto/sha256-browser': 2.0.2
'@aws-sdk/hash-node': 3.224.0
'@aws-sdk/types': 3.224.0
tslib: 1.14.1
dev: false
/@aws-crypto/supports-web-crypto/2.0.2:
resolution: {integrity: sha512-6mbSsLHwZ99CTOOswvCRP3C+VCWnzBf+1SnbWxzzJ9lR0mA0JnY2JEAhp8rqmTE0GPFy88rrM27ffgp62oErMQ==}
dependencies:
tslib: 1.14.1
dev: false
/@aws-crypto/util/2.0.2:
resolution: {integrity: sha512-Lgu5v/0e/BcrZ5m/IWqzPUf3UYFTy/PpeED+uc9SWUR1iZQL8XXbGQg10UfllwwBryO3hFF5dizK+78aoXC1eA==}
dependencies:
'@aws-sdk/types': 3.224.0
'@aws-sdk/util-utf8-browser': 3.188.0
tslib: 1.14.1
dev: false
/@aws-sdk/hash-node/3.224.0:
resolution: {integrity: sha512-y7TXMDOSy5E2VZPvmsvRfyXkcQWcjTLFTd85yc70AAeFZiffff1nvZifQSzD78bW6ELJsWHXA2O8yxdBURyoBg==}
engines: {node: '>=14.0.0'}
dependencies:
'@aws-sdk/types': 3.224.0
'@aws-sdk/util-buffer-from': 3.208.0
tslib: 2.4.1
dev: false
/@aws-sdk/is-array-buffer/3.201.0:
resolution: {integrity: sha512-UPez5qLh3dNgt0DYnPD/q0mVJY84rA17QE26hVNOW3fAji8W2wrwrxdacWOxyXvlxWsVRcKmr+lay1MDqpAMfg==}
engines: {node: '>=14.0.0'}
dependencies:
tslib: 2.4.1
dev: false
/@aws-sdk/types/3.224.0:
resolution: {integrity: sha512-7te9gRondKPjEebyiPYn59Kr5LZOL48HXC05TzFIN/JXwWPJbQpROBPeKd53V1aRdr3vSQhDY01a+vDOBBrEUQ==}
engines: {node: '>=14.0.0'}
dev: false
/@aws-sdk/util-buffer-from/3.208.0:
resolution: {integrity: sha512-7L0XUixNEFcLUGPeBF35enCvB9Xl+K6SQsmbrPk1P3mlV9mguWSDQqbOBwY1Ir0OVbD6H/ZOQU7hI/9RtRI0Zw==}
engines: {node: '>=14.0.0'}
dependencies:
'@aws-sdk/is-array-buffer': 3.201.0
tslib: 2.4.1
dev: false
/@aws-sdk/util-locate-window/3.208.0:
resolution: {integrity: sha512-iua1A2+P7JJEDHVgvXrRJSvsnzG7stYSGQnBVphIUlemwl6nN5D+QrgbjECtrbxRz8asYFHSzhdhECqN+tFiBg==}
engines: {node: '>=14.0.0'}
dependencies:
tslib: 2.4.1
dev: false
/@aws-sdk/util-utf8-browser/3.188.0:
resolution: {integrity: sha512-jt627x0+jE+Ydr9NwkFstg3cUvgWh56qdaqAMDsqgRlKD21md/6G226z/Qxl7lb1VEW2LlmCx43ai/37Qwcj2Q==}
dependencies:
tslib: 2.4.1
dev: false
/@types/path-browserify/1.0.0:
resolution: {integrity: sha512-XMCcyhSvxcch8b7rZAtFAaierBYdeHXVvg2iYnxOV0MCQHmPuRRmGZPFDRzPayxcGiiSL1Te9UIO+f3cuj0tfw==}
dev: true
/async-lock/1.4.0:
resolution: {integrity: sha512-coglx5yIWuetakm3/1dsX9hxCNox22h7+V80RQOu2XUUMidtArxKoZoOtHUPuR84SycKTXzgGzAUR5hJxujyJQ==}
dev: true
/base64-js/1.5.1:
resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==}
dev: false
/buffer/6.0.3:
resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==}
dependencies:
base64-js: 1.5.1
ieee754: 1.2.1
dev: false
/clean-git-ref/2.0.1:
resolution: {integrity: sha512-bLSptAy2P0s6hU4PzuIMKmMJJSE6gLXGH1cntDu7bWJUksvuM+7ReOK61mozULErYvP6a15rnYl0zFDef+pyPw==}
dev: true
/crc-32/1.2.2:
resolution: {integrity: sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==}
engines: {node: '>=0.8'}
hasBin: true
dev: true
/decompress-response/6.0.0:
resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==}
engines: {node: '>=10'}
dependencies:
mimic-response: 3.1.0
dev: true
/diff3/0.0.3:
resolution: {integrity: sha512-iSq8ngPOt0K53A6eVr4d5Kn6GNrM2nQZtC740pzIriHtn4pOQ2lyzEXQMBeVcWERN0ye7fhBsk9PbLLQOnUx/g==}
dev: true
/ieee754/1.2.1:
resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}
dev: false
/ignore/5.2.1:
resolution: {integrity: sha512-d2qQLzTJ9WxQftPAuEQpSPmKqzxePjzVbpAVv62AQ64NTL+wR4JkrVqR/LqFsFEUsHDAiId52mJteHDFuDkElA==}
engines: {node: '>= 4'}
dev: true
/inherits/2.0.4:
resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==}
dev: true
/isomorphic-git/1.21.0:
resolution: {integrity: sha512-ZqCAUM63CYepA3fB8H7NVyPSiOkgzIbQ7T+QPrm9xtYgQypN9JUJ5uLMjB5iTfomdJf3mdm6aSxjZwnT6ubvEA==}
engines: {node: '>=12'}
hasBin: true
dependencies:
async-lock: 1.4.0
clean-git-ref: 2.0.1
crc-32: 1.2.2
diff3: 0.0.3
ignore: 5.2.1
minimisted: 2.0.1
pako: 1.0.11
pify: 4.0.1
readable-stream: 3.6.0
sha.js: 2.4.11
simple-get: 4.0.1
dev: true
/mimic-response/3.1.0:
resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==}
engines: {node: '>=10'}
dev: true
/minimist/1.2.7:
resolution: {integrity: sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g==}
dev: true
/minimisted/2.0.1:
resolution: {integrity: sha512-1oPjfuLQa2caorJUM8HV8lGgWCc0qqAO1MNv/k05G4qslmsndV/5WdNZrqCiyqiz3wohia2Ij2B7w2Dr7/IyrA==}
dependencies:
minimist: 1.2.7
dev: true
/once/1.4.0:
resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==}
dependencies:
wrappy: 1.0.2
dev: true
/pako/1.0.11:
resolution: {integrity: sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==}
dev: true
/path-browserify/1.0.1:
resolution: {integrity: sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==}
dev: false
/pify/4.0.1:
resolution: {integrity: sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==}
engines: {node: '>=6'}
dev: true
/readable-stream/3.6.0:
resolution: {integrity: sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==}
engines: {node: '>= 6'}
dependencies:
inherits: 2.0.4
string_decoder: 1.3.0
util-deprecate: 1.0.2
dev: true
/safe-buffer/5.2.1:
resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==}
dev: true
/sha.js/2.4.11:
resolution: {integrity: sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==}
hasBin: true
dependencies:
inherits: 2.0.4
safe-buffer: 5.2.1
dev: true
/simple-concat/1.0.1:
resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==}
dev: true
/simple-get/4.0.1:
resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==}
dependencies:
decompress-response: 6.0.0
once: 1.4.0
simple-concat: 1.0.1
dev: true
/string_decoder/1.3.0:
resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==}
dependencies:
safe-buffer: 5.2.1
dev: true
/tslib/1.14.1:
resolution: {integrity: sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==}
dev: false
/tslib/2.4.1:
resolution: {integrity: sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==}
dev: false
/typescript/4.9.3:
resolution: {integrity: sha512-CIfGzTelbKNEnLpLdGFgdyKhG23CKdKgQPOBc+OUNrkJ2vr+KSzsSV5kq5iWhEQbok+quxgGzrAtGWCyU7tHnA==}
engines: {node: '>=4.2.0'}
hasBin: true
dev: true
/util-deprecate/1.0.2:
resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==}
dev: true
/wrappy/1.0.2:
resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==}
dev: true

View file

@ -1,10 +1,10 @@
import path from 'path';
import fsp from 'fs/promises';
import { bodyToBuffer, getAuthHeader, isWriteable } from './util';
import { Pointer } from './pointers';
import { HTTPRequest } from './types';
import path from "path-browserify";
import { Buffer } from "buffer";
import { bodyToBuffer, getAuthHeader, isWriteable } from "./util";
import { Pointer } from "./pointers";
import { HTTPRequest } from "./types";
import { PromiseFsClient } from "isomorphic-git";
interface LFSInfoResponse {
objects: {
@ -17,54 +17,53 @@ interface LFSInfoResponse {
}[];
}
function isValidLFSInfoResponseData(val: Record<string, any>): val is LFSInfoResponse {
function isValidLFSInfoResponseData(
val: Record<string, any>
): val is LFSInfoResponse {
return val.objects?.[0]?.actions?.download?.href?.trim !== undefined;
}
/**
* Downloads, caches and returns a blob corresponding to given LFS pointer.
* Uses already cached object, if size matches.
*/
export default async function downloadBlobFromPointer(
{ promises: fs }: PromiseFsClient,
{ http: { request }, headers = {}, url, auth }: HTTPRequest,
{ info, objectPath }: Pointer,
{ info, objectPath }: Pointer
): Promise<Buffer> {
try {
const cached = await fsp.readFile(objectPath);
const cached = await fs.readFile(objectPath);
if (cached.byteLength === info.size) {
return cached;
}
} catch (e) {
// Silence file not found errors (implies cache miss)
if ((e as any).code !== 'ENOENT') {
if ((e as any).code !== "ENOENT") {
throw e;
}
}
const authHeaders: Record<string, string> = auth
? getAuthHeader(auth)
: {};
const authHeaders: Record<string, string> = auth ? getAuthHeader(auth) : {};
// Request LFS transfer
const lfsInfoRequestData = {
operation: 'download',
transfers: ['basic'],
operation: "download",
transfers: ["basic"],
objects: [info],
};
const { body: lfsInfoBody } = await request({
url: `${url}/info/lfs/objects/batch`,
method: 'POST',
method: "POST",
headers: {
// Github LFS doesnt seem to accept this UA, but works fine without any
// 'User-Agent': `git/isomorphic-git@${git.version()}`,
...headers,
...authHeaders,
'Accept': 'application/vnd.git-lfs+json',
'Content-Type': 'application/vnd.git-lfs+json',
Accept: "application/vnd.git-lfs+json",
"Content-Type": "application/vnd.git-lfs+json",
},
body: [Buffer.from(JSON.stringify(lfsInfoRequestData))],
});
@ -74,11 +73,12 @@ export default async function downloadBlobFromPointer(
try {
lfsInfoResponseData = JSON.parse(lfsInfoResponseRaw);
} catch (e) {
throw new Error(`Unexpected structure received from LFS server: unable to parse JSON ${lfsInfoResponseRaw}`);
throw new Error(
`Unexpected structure received from LFS server: unable to parse JSON ${lfsInfoResponseRaw}`
);
}
if (isValidLFSInfoResponseData(lfsInfoResponseData)) {
// Request the actual blob
const downloadAction = lfsInfoResponseData.objects[0].actions.download;
@ -93,21 +93,22 @@ export default async function downloadBlobFromPointer(
const { body: lfsObjectBody } = await request({
url: lfsObjectDownloadURL,
method: 'GET',
method: "GET",
headers: dlHeaders,
});
const blob = await bodyToBuffer(lfsObjectBody);
// Write LFS cache for this object, if cache path is accessible.
if (await isWriteable(objectPath)) {
await fsp.mkdir(path.dirname(objectPath), { recursive: true });
await fsp.writeFile(objectPath, blob);
if (await isWriteable({ promises: fs }, objectPath)) {
await fs.mkdir(path.dirname(objectPath), { recursive: true });
await fs.writeFile(objectPath, blob);
}
return blob;
} else {
throw new Error("Unexpected JSON structure received for LFS download request");
throw new Error(
"Unexpected JSON structure received for LFS download request"
);
}
}

View file

@ -1,7 +1,7 @@
import path from 'path';
import { Sha256 } from '@aws-crypto/sha256-universal';
import { SPEC_URL, toHex } from './util';
import path from "path-browserify";
import { Sha256 } from "@aws-crypto/sha256-universal";
import { Buffer } from "buffer";
import { SPEC_URL, toHex } from "./util";
export interface PointerInfo {
/** OID (currently, SHA256 hash) of actual blob contents. */
@ -19,20 +19,23 @@ export interface Pointer {
}
function isValidPointerInfo(val: Record<string, any>): val is PointerInfo {
return val.oid.trim !== undefined && typeof val.size === 'number';
return val.oid.trim !== undefined && typeof val.size === "number";
}
export function readPointerInfo(content: Buffer): PointerInfo {
const info = content.toString().trim().split('\n').reduce((accum, line) => {
const [k, v] = line.split(' ', 2);
if (k === 'oid') {
accum[k] = v.split(':', 2)[1];
} else if (k === 'size') {
accum[k] = parseInt(v, 10);
}
return accum;
}, {} as Record<string, any>);
const info = content
.toString()
.trim()
.split("\n")
.reduce((accum, line) => {
const [k, v] = line.split(" ", 2);
if (k === "oid") {
accum[k] = v.split(":", 2)[1];
} else if (k === "size") {
accum[k] = parseInt(v, 10);
}
return accum;
}, {} as Record<string, any>);
if (isValidPointerInfo(info)) {
return info;
@ -41,27 +44,30 @@ export function readPointerInfo(content: Buffer): PointerInfo {
}
}
interface PointerRequest {
dir: string;
gitdir?: string;
content: Buffer;
}
export function readPointer({ dir, gitdir = path.join(dir, '.git'), content }: PointerRequest): Pointer {
export function readPointer({
dir,
gitdir = path.join(dir, ".git"),
content,
}: PointerRequest): Pointer {
const info = readPointerInfo(content);
const objectPath = path.join(
gitdir,
'lfs',
'objects',
"lfs",
"objects",
info.oid.substr(0, 2),
info.oid.substr(2, 2),
info.oid);
info.oid
);
return { info, objectPath };
}
/** Formats given PointerInfo for writing in Git tree. */
export function formatPointerInfo(info: PointerInfo): Buffer {
const lines = [
@ -69,14 +75,12 @@ export function formatPointerInfo(info: PointerInfo): Buffer {
`oid sha256:${info.oid}`,
`size ${info.size}`,
];
return Buffer.from(lines.join('\n'));
return Buffer.from(lines.join("\n"));
}
export async function buildPointerInfo(content: Buffer): Promise<PointerInfo> {
const size = Buffer.byteLength(content);
const hash = new Sha256();
hash.update(content);
const oid = toHex(await hash.digest());
const size = content.byteLength;
const hash = await crypto.subtle.digest("SHA-256", content);
const oid = toHex(hash);
return { oid, size };
}

View file

@ -1,18 +1,15 @@
import fs from 'fs';
import { Buffer } from "buffer";
import git from 'isomorphic-git';
import http, { GitProgressEvent } from 'isomorphic-git/http/node';
import git, { PromiseFsClient } from "isomorphic-git";
import http, { GitProgressEvent } from "isomorphic-git/http/node";
import { isVacantAndWriteable, pointsToLFS } from './util';
import downloadBlobFromPointer from './download';
import { isVacantAndWriteable, pointsToLFS } from "./util";
import downloadBlobFromPointer from "./download";
import { readPointer } from "./pointers";
const SYMLINK_MODE = 40960;
type ProgressHandler = (progress: GitProgressEvent) => void
type ProgressHandler = (progress: GitProgressEvent) => void;
/**
* Populates LFS cache for each repository object that is an LFS pointer.
@ -22,23 +19,23 @@ type ProgressHandler = (progress: GitProgressEvent) => void
* NOTE: If LFS cache path, as extracted from the pointer,
* is not writeable at the time of download start,
* the object will be silently skipped.
*
*
* NOTE: This function skips objects silently in case of errors.
*
*
* NOTE: onProgress currently doesnt report loaded/total values accurately.
*/
export default async function populateCache(
fs: PromiseFsClient,
workDir: string,
remoteURL: string,
ref: string = 'HEAD',
onProgress?: ProgressHandler,
ref: string = "HEAD",
onProgress?: ProgressHandler
) {
await git.walk({
fs,
dir: workDir,
trees: [git.TREE({ ref })],
map: async function lfsDownloadingWalker(filepath, entries) {
if (entries === null || entries[0] === null) {
return null;
}
@ -48,33 +45,41 @@ export default async function populateCache(
const [entry] = entries;
const entryType = await entry.type();
if (entryType === 'tree') {
if (entryType === "tree") {
// Walk children
return true;
} else if (entryType === 'blob' && (await entry.mode()) !== SYMLINK_MODE) {
} else if (
entryType === "blob" &&
(await entry.mode()) !== SYMLINK_MODE
) {
const content = await entry.content();
if (content) {
const buff = Buffer.from(content.buffer);
if (pointsToLFS(buff)) {
const pointer = readPointer({ dir: workDir, content: buff });
// Dont even start the download if LFS cache path is not accessible,
// or if it already exists
if (await isVacantAndWriteable(pointer.objectPath) === false)
if ((await isVacantAndWriteable(pointer.objectPath)) === false)
return;
onProgress?.({ phase: `downloading: ${filepath}`, loaded: 5, total: 10 });
await downloadBlobFromPointer({ http, url: remoteURL }, pointer);
onProgress?.({
phase: `downloading: ${filepath}`,
loaded: 5,
total: 10,
});
await downloadBlobFromPointer(
fs,
{ http, url: remoteURL },
pointer
);
}
}
}
return;
}
},
});
}

View file

@ -1,7 +1,8 @@
import { HTTPRequest } from './types';
import { buildPointerInfo, PointerInfo } from './pointers';
import { bodyToBuffer, getAuthHeader } from './util';
import { Buffer } from "buffer";
import { HTTPRequest } from "./types";
import { buildPointerInfo, PointerInfo } from "./pointers";
import { bodyToBuffer, getAuthHeader } from "./util";
interface LFSInfoResponse {
objects: {
@ -18,15 +19,13 @@ interface LFSInfoResponse {
}[];
}
function isValidLFSInfoResponseData(val: Record<string, any>): val is LFSInfoResponse {
function isValidLFSInfoResponseData(
val: Record<string, any>
): val is LFSInfoResponse {
const obj = val.objects?.[0];
return obj && (
!obj.actions ||
obj.actions.upload.href.trim !== undefined
);
return obj && (!obj.actions || obj.actions.upload.href.trim !== undefined);
}
/**
* Given a blob, uploads the blob to LFS server and returns a PointerInfo,
* which the caller can then combine with object path into a Pointer
@ -34,33 +33,30 @@ function isValidLFSInfoResponseData(val: Record<string, any>): val is LFSInfoRes
*/
export default async function uploadBlob(
{ http: { request }, headers = {}, url, auth }: HTTPRequest,
content: Buffer,
content: Buffer
): Promise<PointerInfo> {
const info = await buildPointerInfo(content);
const authHeaders: Record<string, string> = auth
? getAuthHeader(auth)
: {};
const authHeaders: Record<string, string> = auth ? getAuthHeader(auth) : {};
// Request LFS transfer
const lfsInfoRequestData = {
operation: 'upload',
transfers: ['basic'],
operation: "upload",
transfers: ["basic"],
objects: [info],
};
const { body: lfsInfoBody } = await request({
url: `${url}/info/lfs/objects/batch`,
method: 'POST',
method: "POST",
headers: {
// Github LFS doesnt seem to accept this UA
// 'User-Agent': `git/isomorphic-git@${git.version()}`,
...headers,
...authHeaders,
'Accept': 'application/vnd.git-lfs+json',
'Content-Type': 'application/vnd.git-lfs+json',
Accept: "application/vnd.git-lfs+json",
"Content-Type": "application/vnd.git-lfs+json",
},
body: [Buffer.from(JSON.stringify(lfsInfoRequestData))],
});
@ -70,11 +66,12 @@ export default async function uploadBlob(
try {
lfsInfoResponseData = JSON.parse(lfsInfoResponseRaw);
} catch (e) {
throw new Error(`Unexpected structure received from LFS server: unable to parse JSON ${lfsInfoResponseRaw}`);
throw new Error(
`Unexpected structure received from LFS server: unable to parse JSON ${lfsInfoResponseRaw}`
);
}
if (isValidLFSInfoResponseData(lfsInfoResponseData)) {
// Upload the actual blob
const actions = lfsInfoResponseData.objects[0].actions;
@ -83,7 +80,6 @@ export default async function uploadBlob(
// Presume LFS already has the blob. Dont fail loudly.
return info;
} else {
const uploadAction = actions.upload;
const lfsObjectUploadURL = uploadAction.href;
const lfsObjectUploadHeaders = uploadAction.header ?? {};
@ -96,7 +92,7 @@ export default async function uploadBlob(
const resp = await request({
url: lfsObjectUploadURL,
method: 'PUT',
method: "PUT",
headers: dlHeaders,
body: [content],
});
@ -109,12 +105,12 @@ export default async function uploadBlob(
if (verifyAction) {
const verificationResp = await request({
url: verifyAction.href,
method: 'POST',
method: "POST",
headers: {
// Isomorphic Gits UA header is considered invalid
// and missing UA header causes an error in this case;
// cURL is considered valid, so…
'User-Agent': `curl/7.54`,
"User-Agent": `curl/7.54`,
// TODO: Generalize UA header handling
// - Leave UA header twiddling to callers?
// - Figure out which LFS implementation wants which UA header?
@ -126,17 +122,22 @@ export default async function uploadBlob(
if (verificationResp.statusCode === 200) {
return info;
} else {
throw new Error(`Upload might have been unsuccessful, verification action yielded HTTP ${verificationResp.statusCode}`);
throw new Error(
`Upload might have been unsuccessful, verification action yielded HTTP ${verificationResp.statusCode}`
);
}
} else {
return info;
}
} else {
throw new Error(`Upload might have been unsuccessful, upload action yielded HTTP ${resp.statusCode}`);
throw new Error(
`Upload might have been unsuccessful, upload action yielded HTTP ${resp.statusCode}`
);
}
}
} else {
throw new Error("Unexpected JSON structure received for LFS upload request");
throw new Error(
"Unexpected JSON structure received for LFS upload request"
);
}
}

View file

@ -1,66 +1,69 @@
import fs from 'fs/promises';
import { constants as fsConstants } from 'fs';
import { BasicAuth } from './types';
import { Buffer } from "buffer";
import { BasicAuth } from "./types";
import { PromiseFsClient } from "isomorphic-git";
export const SPEC_URL = 'https://git-lfs.github.com/spec/v1';
export const SPEC_URL = "https://git-lfs.github.com/spec/v1";
export const LFS_POINTER_PREAMBLE = `version ${SPEC_URL}\n`;
/** Returns true if given blob represents an LFS pointer. */
export function pointsToLFS(content: Buffer): boolean {
return (
content[0] === 118 // 'v'
&& content.subarray(0, 100).indexOf(LFS_POINTER_PREAMBLE) === 0);
content[0] === 118 && // 'v'
// TODO: This is inefficient, it should only search the first line or first few bytes.
content.indexOf(LFS_POINTER_PREAMBLE) === 0
);
}
/**
* Returns properly encoded HTTP Basic auth header,
* given basic auth credentials.
*/
export function getAuthHeader(auth: BasicAuth): Record<string, string> {
return {
'Authorization':
`Basic ${Buffer.from(`${auth.username}:${auth.password}`).toString('base64')}`,
Authorization: `Basic ${Buffer.from(
`${auth.username}:${auth.password}`
).toString("base64")}`,
};
}
/**
* Returns true if given path is available for writing,
* regardless of whether or not it is occupied.
*/
export async function isWriteable(filepath: string): Promise<boolean> {
export async function isWriteable(
{ promises: fs }: PromiseFsClient,
filepath: string
): Promise<boolean> {
try {
await fs.access(filepath, fsConstants.W_OK);
// TODO: there's no API for this in PromiseFsClient world
// await fs.access(filepath, fsConstants.W_OK);
return true;
} catch (e) {
if ((e as { code: string }).code === 'ENOENT') {
if ((e as { code: string }).code === "ENOENT") {
return true;
}
return false;
}
}
/**
* Returns true if given path is available for writing
* and not occupied.
*/
export async function isVacantAndWriteable(filepath: string): Promise<boolean> {
try {
await fs.access(filepath, fsConstants.W_OK);
// TODO: there's no API for this in PromiseFsClient world
return true;
// await fs.access(filepath, fsConstants.W_OK);
} catch (e) {
if ((e as { code: string }).code === 'ENOENT') {
if ((e as { code: string }).code === "ENOENT") {
return true;
}
}
return false;
}
export async function bodyToBuffer(body: Uint8Array[]): Promise<Buffer> {
const buffers = [];
let offset = 0;
@ -78,13 +81,12 @@ export async function bodyToBuffer(body: Uint8Array[]): Promise<Buffer> {
return Buffer.from(result.buffer);
}
// Borrowed from Isomorphic Git core, it is not importable.
export function toHex(buffer: ArrayBuffer): string {
let hex = ''
let hex = "";
for (const byte of new Uint8Array(buffer)) {
if (byte < 16) hex += '0'
hex += byte.toString(16)
if (byte < 16) hex += "0";
hex += byte.toString(16);
}
return hex
return hex;
}

View file

@ -1,22 +1,22 @@
{
"compilerOptions": {
"target": "es2017",
"module": "esnext",
"moduleResolution": "node",
"compilerOptions": {
"target": "es2018",
"module": "esnext",
"moduleResolution": "node",
"strict": true,
"noUnusedLocals": true,
"noFallthroughCasesInSwitch": true,
"noImplicitReturns": true,
"strict": true,
"noUnusedLocals": true,
"noFallthroughCasesInSwitch": true,
"noImplicitReturns": true,
"sourceMap": true,
"inlineSources": true,
"sourceMap": true,
"inlineSources": true,
"allowSyntheticDefaultImports": true,
"experimentalDecorators": true,
"allowSyntheticDefaultImports": true,
"experimentalDecorators": true,
"newLine": "lf",
"newLine": "lf",
"declaration": true
}
"declaration": true
}
}

1709
yarn.lock

File diff suppressed because it is too large Load diff