refactor uploadBlobs

This commit is contained in:
Cat /dev/Nulo 2023-01-17 11:09:58 -03:00
parent f85834bcdf
commit 0922f8d982
2 changed files with 50 additions and 91 deletions

View file

@ -1,4 +1,4 @@
export { default as downloadBlobFromPointer } from "./download";
export { default as populateCache } from "./populateCache";
export { readPointer, formatPointerInfo } from "./pointers";
export { default as uploadBlob } from "./upload";
export { default as uploadBlobs } from "./upload";

View file

@ -2,7 +2,7 @@ import { Buffer } from "buffer";
import { HTTPRequest } from "./types";
import { buildPointerInfo, PointerInfo } from "./pointers";
import { bodyToBuffer, getAuthHeader } from "./util";
import { getAuthHeader } from "./util";
interface LFSInfoResponse {
objects: {
@ -31,113 +31,72 @@ function isValidLFSInfoResponseData(
* which the caller can then combine with object path into a Pointer
* and commit in place of the original Git blob.
*/
export default async function uploadBlob(
{ http: { request }, headers = {}, url, auth }: HTTPRequest,
content: Buffer
): Promise<PointerInfo> {
const info = await buildPointerInfo(content);
export default async function uploadBlobs(
{ headers = {}, url, auth }: HTTPRequest,
contents: Buffer[]
): Promise<PointerInfo[]> {
const infos = await Promise.all(contents.map((c) => buildPointerInfo(c)));
const authHeaders: Record<string, string> = auth ? getAuthHeader(auth) : {};
// Request LFS transfer
const lfsInfoRequestData = {
operation: "upload",
transfers: ["basic"],
objects: [info],
objects: infos,
};
const { body: lfsInfoBody } = await request({
url: `${url}/info/lfs/objects/batch`,
const lfsInfoRes = await fetch(`${url}/info/lfs/objects/batch`, {
method: "POST",
headers: {
// Github LFS doesnt seem to accept this UA
// 'User-Agent': `git/isomorphic-git@${git.version()}`,
...headers,
...authHeaders,
Accept: "application/vnd.git-lfs+json",
"Content-Type": "application/vnd.git-lfs+json",
},
body: [Buffer.from(JSON.stringify(lfsInfoRequestData))],
body: JSON.stringify(lfsInfoRequestData),
});
const lfsInfoResponseRaw = (await bodyToBuffer(lfsInfoBody)).toString();
let lfsInfoResponseData: any;
try {
lfsInfoResponseData = JSON.parse(lfsInfoResponseRaw);
} catch (e) {
throw new Error(
`Unexpected structure received from LFS server: unable to parse JSON ${lfsInfoResponseRaw}`
);
}
if (isValidLFSInfoResponseData(lfsInfoResponseData)) {
// Upload the actual blob
const actions = lfsInfoResponseData.objects[0].actions;
if (!actions) {
// Presume LFS already has the blob. Dont fail loudly.
return info;
} else {
const uploadAction = actions.upload;
const lfsObjectUploadURL = uploadAction.href;
const lfsObjectUploadHeaders = uploadAction.header ?? {};
const dlHeaders = {
...headers,
...authHeaders,
...lfsObjectUploadHeaders,
};
const resp = await request({
url: lfsObjectUploadURL,
method: "PUT",
headers: dlHeaders,
body: [content],
});
if (resp.statusCode === 200) {
const verifyAction = actions.verify;
// Upload verification was requested. Do that
if (verifyAction) {
const verificationResp = await request({
url: verifyAction.href,
method: "POST",
headers: {
// Isomorphic Gits UA header is considered invalid
// and missing UA header causes an error in this case;
// cURL is considered valid, so…
"User-Agent": `curl/7.54`,
// TODO: Generalize UA header handling
// - Leave UA header twiddling to callers?
// - Figure out which LFS implementation wants which UA header?
...(verifyAction.header ?? {}),
},
body: [Buffer.from(JSON.stringify(info))],
});
if (verificationResp.statusCode === 200) {
return info;
} else {
throw new Error(
`Upload might have been unsuccessful, verification action yielded HTTP ${verificationResp.statusCode}`
);
}
} else {
return info;
}
} else {
throw new Error(
`Upload might have been unsuccessful, upload action yielded HTTP ${resp.statusCode}`
);
}
}
} else {
const lfsInfoResponseData = await lfsInfoRes.json();
if (!isValidLFSInfoResponseData(lfsInfoResponseData)) {
throw new Error(
"Unexpected JSON structure received for LFS upload request"
);
}
await Promise.all(
lfsInfoResponseData.objects.map(async (object, index) => {
// server already has file
if (!object.actions) return;
const { actions } = object;
const resp = await fetch(actions.upload.href, {
method: "PUT",
headers: {
...headers,
...authHeaders,
...(actions.upload.header ?? {}),
Accept: "application/vnd.git-lfs+json",
},
body: contents[index],
});
if (!resp.ok)
throw new Error(
`Upload might have been unsuccessful, upload action yielded HTTP ${resp.status}`
);
if (actions.verify) {
const verificationResp = await fetch(actions.verify.href, {
method: "POST",
headers: {
...(actions.verify.header ?? {}),
Accept: "application/vnd.git-lfs+json",
},
body: JSON.stringify(infos[index]),
});
throw new Error(
`Upload might have been unsuccessful, verification action yielded HTTP ${verificationResp.status}`
);
}
})
);
return infos;
}