Nulo/isogit-lfs
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Activity

Make it compatible with browser

Browse source
This commit is contained in:
Cat /dev/Nulo 2022-12-07 20:39:09 -03:00
parent 305f7a3435
commit e072728c12
7 changed files with 163 additions and 147 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
package.json
View file

@ -10,12 +10,15 @@
"email": "open.source@ribose.com"
},
"dependencies": {
"@aws-crypto/sha256-universal": "^2.0.2"
"@aws-crypto/sha256-universal": "^2.0.2",
"buffer": "^6.0.3",
"path-browserify": "^1.0.1"
},
"peerDependencies": {
"isomorphic-git": "^1.7.8"
},
"devDependencies": {
"@types/path-browserify": "^1.0.0",
"isomorphic-git": "^1.7.8",
"typescript": "^4.4.2"
},

57
src/download.ts
View file

@ -1,10 +1,10 @@
import path from 'path';
import fsp from 'fs/promises';
import { bodyToBuffer, getAuthHeader, isWriteable } from './util';
import { Pointer } from './pointers';
import { HTTPRequest } from './types';
import path from "path-browserify";
import { Buffer } from "buffer";
import { bodyToBuffer, getAuthHeader, isWriteable } from "./util";
import { Pointer } from "./pointers";
import { HTTPRequest } from "./types";
import { PromiseFsClient } from "isomorphic-git";
interface LFSInfoResponse {
objects: {
@ -17,54 +17,53 @@ interface LFSInfoResponse {
}[];
}
function isValidLFSInfoResponseData(val: Record<string, any>): val is LFSInfoResponse {
function isValidLFSInfoResponseData(
val: Record<string, any>
): val is LFSInfoResponse {
return val.objects?.[0]?.actions?.download?.href?.trim !== undefined;
}
/**
* Downloads, caches and returns a blob corresponding to given LFS pointer.
* Uses already cached object, if size matches.
*/
export default async function downloadBlobFromPointer(
{ promises: fs }: PromiseFsClient,
{ http: { request }, headers = {}, url, auth }: HTTPRequest,
{ info, objectPath }: Pointer,
{ info, objectPath }: Pointer
): Promise<Buffer> {
try {
const cached = await fsp.readFile(objectPath);
const cached = await fs.readFile(objectPath);
if (cached.byteLength === info.size) {
return cached;
}
} catch (e) {
// Silence file not found errors (implies cache miss)
if ((e as any).code !== 'ENOENT') {
if ((e as any).code !== "ENOENT") {
throw e;
}
}
const authHeaders: Record<string, string> = auth
? getAuthHeader(auth)
: {};
const authHeaders: Record<string, string> = auth ? getAuthHeader(auth) : {};
// Request LFS transfer
const lfsInfoRequestData = {
operation: 'download',
transfers: ['basic'],
operation: "download",
transfers: ["basic"],
objects: [info],
};
const { body: lfsInfoBody } = await request({
url: `${url}/info/lfs/objects/batch`,
method: 'POST',
method: "POST",
headers: {
// Github LFS doesn’t seem to accept this UA, but works fine without any
// 'User-Agent': `git/isomorphic-git@${git.version()}`,
...headers,
...authHeaders,
'Accept': 'application/vnd.git-lfs+json',
'Content-Type': 'application/vnd.git-lfs+json',
Accept: "application/vnd.git-lfs+json",
"Content-Type": "application/vnd.git-lfs+json",
},
body: [Buffer.from(JSON.stringify(lfsInfoRequestData))],
});
@ -74,11 +73,12 @@ export default async function downloadBlobFromPointer(
try {
lfsInfoResponseData = JSON.parse(lfsInfoResponseRaw);
} catch (e) {
throw new Error(`Unexpected structure received from LFS server: unable to parse JSON ${lfsInfoResponseRaw}`);
throw new Error(
`Unexpected structure received from LFS server: unable to parse JSON ${lfsInfoResponseRaw}`
);
}
if (isValidLFSInfoResponseData(lfsInfoResponseData)) {
// Request the actual blob
const downloadAction = lfsInfoResponseData.objects[0].actions.download;
@ -93,21 +93,22 @@ export default async function downloadBlobFromPointer(
const { body: lfsObjectBody } = await request({
url: lfsObjectDownloadURL,
method: 'GET',
method: "GET",
headers: dlHeaders,
});
const blob = await bodyToBuffer(lfsObjectBody);
// Write LFS cache for this object, if cache path is accessible.
if (await isWriteable(objectPath)) {
await fsp.mkdir(path.dirname(objectPath), { recursive: true });
await fsp.writeFile(objectPath, blob);
if (await isWriteable({ promises: fs }, objectPath)) {
await fs.mkdir(path.dirname(objectPath), { recursive: true });
await fs.writeFile(objectPath, blob);
}
return blob;
} else {
throw new Error("Unexpected JSON structure received for LFS download request");
throw new Error(
"Unexpected JSON structure received for LFS download request"
);
}
}

50
src/pointers.ts
View file

@ -1,7 +1,7 @@
import path from 'path';
import { Sha256 } from '@aws-crypto/sha256-universal';
import { SPEC_URL, toHex } from './util';
import path from "path-browserify";
import { Sha256 } from "@aws-crypto/sha256-universal";
import { Buffer } from "buffer";
import { SPEC_URL, toHex } from "./util";
export interface PointerInfo {
/** OID (currently, SHA256 hash) of actual blob contents. */
@ -19,16 +19,19 @@ export interface Pointer {
}
function isValidPointerInfo(val: Record<string, any>): val is PointerInfo {
return val.oid.trim !== undefined && typeof val.size === 'number';
return val.oid.trim !== undefined && typeof val.size === "number";
}
export function readPointerInfo(content: Buffer): PointerInfo {
const info = content.toString().trim().split('\n').reduce((accum, line) => {
const [k, v] = line.split(' ', 2);
if (k === 'oid') {
accum[k] = v.split(':', 2)[1];
} else if (k === 'size') {
const info = content
.toString()
.trim()
.split("\n")
.reduce((accum, line) => {
const [k, v] = line.split(" ", 2);
if (k === "oid") {
accum[k] = v.split(":", 2)[1];
} else if (k === "size") {
accum[k] = parseInt(v, 10);
}
return accum;
@ -41,27 +44,30 @@ export function readPointerInfo(content: Buffer): PointerInfo {
}
}
interface PointerRequest {
dir: string;
gitdir?: string;
content: Buffer;
}
export function readPointer({ dir, gitdir = path.join(dir, '.git'), content }: PointerRequest): Pointer {
export function readPointer({
dir,
gitdir = path.join(dir, ".git"),
content,
}: PointerRequest): Pointer {
const info = readPointerInfo(content);
const objectPath = path.join(
gitdir,
'lfs',
'objects',
"lfs",
"objects",
info.oid.substr(0, 2),
info.oid.substr(2, 2),
info.oid);
info.oid
);
return { info, objectPath };
}
/** Formats given PointerInfo for writing in Git tree. */
export function formatPointerInfo(info: PointerInfo): Buffer {
const lines = [
@ -69,14 +75,12 @@ export function formatPointerInfo(info: PointerInfo): Buffer {
`oid sha256:${info.oid}`,
`size ${info.size}`,
];
return Buffer.from(lines.join('\n'));
return Buffer.from(lines.join("\n"));
}
export async function buildPointerInfo(content: Buffer): Promise<PointerInfo> {
const size = Buffer.byteLength(content);
const hash = new Sha256();
hash.update(content);
const oid = toHex(await hash.digest());
const size = content.byteLength;
const hash = await crypto.subtle.digest("SHA-256", content);
const oid = toHex(hash);
return { oid, size };
}

47
src/populateCache.ts
View file

@ -1,18 +1,15 @@
import fs from 'fs';
import { Buffer } from "buffer";
import git from 'isomorphic-git';
import http, { GitProgressEvent } from 'isomorphic-git/http/node';
import git, { PromiseFsClient } from "isomorphic-git";
import http, { GitProgressEvent } from "isomorphic-git/http/node";
import { isVacantAndWriteable, pointsToLFS } from './util';
import downloadBlobFromPointer from './download';
import { isVacantAndWriteable, pointsToLFS } from "./util";
import downloadBlobFromPointer from "./download";
import { readPointer } from "./pointers";
const SYMLINK_MODE = 40960;
type ProgressHandler = (progress: GitProgressEvent) => void
type ProgressHandler = (progress: GitProgressEvent) => void;
/**
* Populates LFS cache for each repository object that is an LFS pointer.
@ -28,17 +25,17 @@ type ProgressHandler = (progress: GitProgressEvent) => void
* NOTE: onProgress currently doesn’t report loaded/total values accurately.
*/
export default async function populateCache(
fs: PromiseFsClient,
workDir: string,
remoteURL: string,
ref: string = 'HEAD',
onProgress?: ProgressHandler,
ref: string = "HEAD",
onProgress?: ProgressHandler
) {
await git.walk({
fs,
dir: workDir,
trees: [git.TREE({ ref })],
map: async function lfsDownloadingWalker(filepath, entries) {
if (entries === null || entries[0] === null) {
return null;
}
@ -48,33 +45,41 @@ export default async function populateCache(
const [entry] = entries;
const entryType = await entry.type();
if (entryType === 'tree') {
if (entryType === "tree") {
// Walk children
return true;
} else if (entryType === 'blob' && (await entry.mode()) !== SYMLINK_MODE) {
} else if (
entryType === "blob" &&
(await entry.mode()) !== SYMLINK_MODE
) {
const content = await entry.content();
if (content) {
const buff = Buffer.from(content.buffer);
if (pointsToLFS(buff)) {
const pointer = readPointer({ dir: workDir, content: buff });
// Don’t even start the download if LFS cache path is not accessible,
// or if it already exists
if (await isVacantAndWriteable(pointer.objectPath) === false)
if ((await isVacantAndWriteable(pointer.objectPath)) === false)
return;
onProgress?.({ phase: `downloading: ${filepath}`, loaded: 5, total: 10 });
await downloadBlobFromPointer({ http, url: remoteURL }, pointer);
onProgress?.({
phase: `downloading: ${filepath}`,
loaded: 5,
total: 10,
});
await downloadBlobFromPointer(
fs,
{ http, url: remoteURL },
pointer
);
}
}
}
return;
}
},
});
}

59
src/upload.ts
View file

@ -1,7 +1,8 @@
import { HTTPRequest } from './types';
import { buildPointerInfo, PointerInfo } from './pointers';
import { bodyToBuffer, getAuthHeader } from './util';
import { Buffer } from "buffer";
import { HTTPRequest } from "./types";
import { buildPointerInfo, PointerInfo } from "./pointers";
import { bodyToBuffer, getAuthHeader } from "./util";
interface LFSInfoResponse {
objects: {
@ -18,15 +19,13 @@ interface LFSInfoResponse {
}[];
}
function isValidLFSInfoResponseData(val: Record<string, any>): val is LFSInfoResponse {
function isValidLFSInfoResponseData(
val: Record<string, any>
): val is LFSInfoResponse {
const obj = val.objects?.[0];
return obj && (
!obj.actions ||
obj.actions.upload.href.trim !== undefined
);
return obj && (!obj.actions || obj.actions.upload.href.trim !== undefined);
}
/**
* Given a blob, uploads the blob to LFS server and returns a PointerInfo,
* which the caller can then combine with object path into a Pointer
@ -34,33 +33,30 @@ function isValidLFSInfoResponseData(val: Record<string, any>): val is LFSInfoRes
*/
export default async function uploadBlob(
{ http: { request }, headers = {}, url, auth }: HTTPRequest,
content: Buffer,
content: Buffer
): Promise<PointerInfo> {
const info = await buildPointerInfo(content);
const authHeaders: Record<string, string> = auth
? getAuthHeader(auth)
: {};
const authHeaders: Record<string, string> = auth ? getAuthHeader(auth) : {};
// Request LFS transfer
const lfsInfoRequestData = {
operation: 'upload',
transfers: ['basic'],
operation: "upload",
transfers: ["basic"],
objects: [info],
};
const { body: lfsInfoBody } = await request({
url: `${url}/info/lfs/objects/batch`,
method: 'POST',
method: "POST",
headers: {
// Github LFS doesn’t seem to accept this UA
// 'User-Agent': `git/isomorphic-git@${git.version()}`,
...headers,
...authHeaders,
'Accept': 'application/vnd.git-lfs+json',
'Content-Type': 'application/vnd.git-lfs+json',
Accept: "application/vnd.git-lfs+json",
"Content-Type": "application/vnd.git-lfs+json",
},
body: [Buffer.from(JSON.stringify(lfsInfoRequestData))],
});
@ -70,11 +66,12 @@ export default async function uploadBlob(
try {
lfsInfoResponseData = JSON.parse(lfsInfoResponseRaw);
} catch (e) {
throw new Error(`Unexpected structure received from LFS server: unable to parse JSON ${lfsInfoResponseRaw}`);
throw new Error(
`Unexpected structure received from LFS server: unable to parse JSON ${lfsInfoResponseRaw}`
);
}
if (isValidLFSInfoResponseData(lfsInfoResponseData)) {
// Upload the actual blob
const actions = lfsInfoResponseData.objects[0].actions;
@ -83,7 +80,6 @@ export default async function uploadBlob(
// Presume LFS already has the blob. Don’t fail loudly.
return info;
} else {
const uploadAction = actions.upload;
const lfsObjectUploadURL = uploadAction.href;
const lfsObjectUploadHeaders = uploadAction.header ?? {};
@ -96,7 +92,7 @@ export default async function uploadBlob(
const resp = await request({
url: lfsObjectUploadURL,
method: 'PUT',
method: "PUT",
headers: dlHeaders,
body: [content],
});
@ -109,12 +105,12 @@ export default async function uploadBlob(
if (verifyAction) {
const verificationResp = await request({
url: verifyAction.href,
method: 'POST',
method: "POST",
headers: {
// Isomorphic Git’s UA header is considered invalid
// and missing UA header causes an error in this case;
// cURL is considered valid, so…
'User-Agent': `curl/7.54`,
"User-Agent": `curl/7.54`,
// TODO: Generalize UA header handling
// - Leave UA header twiddling to callers?
// - Figure out which LFS implementation wants which UA header?
@ -126,17 +122,22 @@ export default async function uploadBlob(
if (verificationResp.statusCode === 200) {
return info;
} else {
throw new Error(`Upload might have been unsuccessful, verification action yielded HTTP ${verificationResp.statusCode}`);
throw new Error(
`Upload might have been unsuccessful, verification action yielded HTTP ${verificationResp.statusCode}`
);
}
} else {
return info;
}
} else {
throw new Error(`Upload might have been unsuccessful, upload action yielded HTTP ${resp.statusCode}`);
throw new Error(
`Upload might have been unsuccessful, upload action yielded HTTP ${resp.statusCode}`
);
}
}
} else {
throw new Error("Unexpected JSON structure received for LFS upload request");
throw new Error(
"Unexpected JSON structure received for LFS upload request"
);
}
}

50
src/util.ts
View file

@ -1,66 +1,69 @@
import fs from 'fs/promises';
import { constants as fsConstants } from 'fs';
import { BasicAuth } from './types';
import { Buffer } from "buffer";
import { BasicAuth } from "./types";
import { PromiseFsClient } from "isomorphic-git";
export const SPEC_URL = 'https://git-lfs.github.com/spec/v1';
export const SPEC_URL = "https://git-lfs.github.com/spec/v1";
export const LFS_POINTER_PREAMBLE = `version ${SPEC_URL}\n`;
/** Returns true if given blob represents an LFS pointer. */
export function pointsToLFS(content: Buffer): boolean {
return (
content[0] === 118 // 'v'
&& content.subarray(0, 100).indexOf(LFS_POINTER_PREAMBLE) === 0);
content[0] === 118 && // 'v'
// TODO: This is inefficient, it should only search the first line or first few bytes.
content.indexOf(LFS_POINTER_PREAMBLE) === 0
);
}
/**
* Returns properly encoded HTTP Basic auth header,
* given basic auth credentials.
*/
export function getAuthHeader(auth: BasicAuth): Record<string, string> {
return {
'Authorization':
`Basic ${Buffer.from(`${auth.username}:${auth.password}`).toString('base64')}`,
Authorization: `Basic ${Buffer.from(
`${auth.username}:${auth.password}`
).toString("base64")}`,
};
}
/**
* Returns true if given path is available for writing,
* regardless of whether or not it is occupied.
*/
export async function isWriteable(filepath: string): Promise<boolean> {
export async function isWriteable(
{ promises: fs }: PromiseFsClient,
filepath: string
): Promise<boolean> {
try {
await fs.access(filepath, fsConstants.W_OK);
// TODO: there's no API for this in PromiseFsClient world
// await fs.access(filepath, fsConstants.W_OK);
return true;
} catch (e) {
if ((e as { code: string }).code === 'ENOENT') {
if ((e as { code: string }).code === "ENOENT") {
return true;
}
return false;
}
}
/**
* Returns true if given path is available for writing
* and not occupied.
*/
export async function isVacantAndWriteable(filepath: string): Promise<boolean> {
try {
await fs.access(filepath, fsConstants.W_OK);
// TODO: there's no API for this in PromiseFsClient world
return true;
// await fs.access(filepath, fsConstants.W_OK);
} catch (e) {
if ((e as { code: string }).code === 'ENOENT') {
if ((e as { code: string }).code === "ENOENT") {
return true;
}
}
return false;
}
export async function bodyToBuffer(body: Uint8Array[]): Promise<Buffer> {
const buffers = [];
let offset = 0;
@ -78,13 +81,12 @@ export async function bodyToBuffer(body: Uint8Array[]): Promise<Buffer> {
return Buffer.from(result.buffer);
}
// Borrowed from Isomorphic Git core, it is not importable.
export function toHex(buffer: ArrayBuffer): string {
let hex = ''
let hex = "";
for (const byte of new Uint8Array(buffer)) {
if (byte < 16) hex += '0'
hex += byte.toString(16)
if (byte < 16) hex += "0";
hex += byte.toString(16);
}
return hex
return hex;
}

2
tsconfig.json
View file

@ -1,6 +1,6 @@
{
"compilerOptions": {
"target": "es2017",
"target": "es2018",
"module": "esnext",
"moduleResolution": "node",