Upload of project source code files.

src/worker.ts

@@ -0,0 +1,47 @@
+import { workerData, parentPort } from "worker_threads";
+import fetch from "node-fetch";
+import fs from "fs-extra";
+import path from "path";
+import { fileURLToPath } from "url";
+import { randomBytes } from "crypto";
+
+const generateRandomFilename = () => randomBytes(8).toString("hex");
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const imgsDir = path.join(__dirname, "..", "..", "files");
+fs.ensureDirSync(imgsDir);
+
+const downloadFile = async (url: string) => {
+  try {
+    const response = await fetch(url);
+    if (!response.ok) throw new Error(`Failed to fetch ${url}.`);
+    const buffer = await response.buffer();
+    const fileName = path.basename(url.split("?")[0]);
+    const filePath = path.join(
+      imgsDir,
+      `${generateRandomFilename()}-${fileName}`
+    );
+    await fs.outputFile(filePath, buffer);
+    parentPort?.postMessage(`Scraped ${url} to ${filePath}.`);
+  } catch (error) {
+    if (error instanceof Error) {
+      parentPort?.postMessage(`Error scraping ${url}: ${error.message}.`);
+    } else {
+      parentPort?.postMessage(`Unknown error scraping ${url}.`);
+    }
+  }
+};
+
+const main = async () => {
+  const { urlsChunk } = workerData as {
+    urlsChunk: string[];
+  };
+
+  for (const url of urlsChunk) {
+    await downloadFile(url);
+  }
+};
+
+main();