site-analyzer/src/index.ts

import { Parser } from "htmlparser2";
import { DomHandler, Element } from "domhandler";
import { readdir, readFile } from "fs/promises";
import { join } from "path";

const { argv } = process;
const dirPath = argv[2] || ".";
interface Thing {
  type:
    | "link-http"
    | "link-absolute"
    | "link-no-href"
    | "media-http"
    | "media-absolute"
    | "media-no-src";
  description: string;
}
interface Report {
  things: Thing[];
}

interface Page {
  report: Report;
  rawHtml: string;
}

function recurseElement(page: Page, el: Element) {
  const { report, rawHtml } = page;
  if (el.name === "a") {
    if (el.attribs.href !== undefined) {
      if (isHttp(el.attribs.href)) {
        report.things.push({
          type: "link-http",
          description: getText(el),
        });
      } else if (isAbsolute(el.attribs.href)) {
        report.things.push({
          type: "link-absolute",
          description: getText(el),
        });
      }
    } else {
      report.things.push({
        type: "link-no-href",
        description: getText(el),
      });
    }
  }
  if (["audio", "video", "img", "source"].includes(el.name)) {
    if (el.attribs.src) {
      checkUrl(page, el, el.attribs.src);
    } else {
      report.things.push({
        type: "media-no-src",
        description: getHtml(rawHtml, el),
      });
    }
  }

  for (const child of el.children) {
    if (child.type === "tag") {
      recurseElement(page, child);
    }
  }
}

function checkUrl({ report, rawHtml }: Page, el: Element, url: string) {
  if (isHttp(url)) {
    report.things.push({
      type: "media-http",
      description: getHtml(rawHtml, el),
    });
  } else if (isAbsolute(url)) {
    report.things.push({
      type: "media-absolute",
      description: getHtml(rawHtml, el),
    });
  }
}

function isHttp(url: string) {
  const r = /^(https?:\/\/|\/\/)/;
  return r.test(url);
}
function isAbsolute(url: string) {
  return url.startsWith("/");
}

function getHtml(rawHtml: string, el: Element) {
  return rawHtml.slice(el.startIndex!, el.endIndex!);
}
function getText(el: Element) {
  let text = "";
  for (const child of el.children) {
    if (child.type === "text") text += child.data.trim();
    else if (child.type === "tag") text += getText(child);
  }
  return text;
}

function processFile(content: string): Promise<Report> {
  return new Promise((resolve, reject) => {
    const handler = new DomHandler(
      (error, dom) => {
        if (error) {
          reject(error);
        } else {
          let report: Report = {
            things: [],
          };
          for (const el of dom) {
            if (el.type === "tag") {
              recurseElement({ report, rawHtml: content }, el);
            }
          }
          resolve(report);
        }
      },
      { withEndIndices: true, withStartIndices: true }
    );
    const parser = new Parser(handler);
    parser.parseComplete(content);
  });
}

interface Reports {
  [key: string]: Report;
}
let reports: Reports = {};
async function recurseDirectory(reports: Reports, path: string) {
  const dir = await readdir(path, { withFileTypes: true });
  for (const file of dir) {
    const filePath = join(path, file.name);
    if (file.isDirectory()) await recurseDirectory(reports, filePath);
    else {
      if (!file.name.endsWith(".html")) continue;
      const content = await readFile(filePath, "utf-8");
      reports[filePath] = await processFile(content);
    }
  }
}
await recurseDirectory(reports, dirPath);
const totalThings = Object.values(reports)
  .map((r) => r.things)
  .flat();
const kinds = new Set(totalThings.map((t) => t.type));
console.log(
  `Finished with ${Object.keys(reports).length} files read, ${
    totalThings.length
  } things`
);
for (const kind of kinds) {
  const count = totalThings.filter((t) => t.type === kind).length;
  console.log(`==> ${kind}: ${count}`);
}
console.log("This means:");
const pathBasedCount = totalThings.filter((t) =>
  ["media-absolute", "link-absolute"].includes(t.type)
).length;
console.log(
  `==> ${pathBasedCount} problems that affect users using legacy IPFS gateways`
);
const mediaHttp = totalThings.filter((t) => t.type === "media-http").length;
console.log(
  `==> ${mediaHttp} problems that make the website not self-contained, making it miss content if HTTP is unavailable`
);
const linkHttp = totalThings.filter((t) => t.type === "link-http").length;
console.log(
  `==> ${linkHttp} links to HTTP sites, which is not a real concern unless it's a key part of the site's navigation.`
);