Upload of project source code files.

dist/src/downloader.js

@@ -0,0 +1,46 @@
+import fs from "fs-extra";
+import path from "path";
+import { Worker } from "worker_threads";
+import { fileURLToPath } from "url";
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const createWorker = (urlsChunk, workerId) => {
+    return new Promise((resolve, reject) => {
+        const worker = new Worker(path.join(__dirname, "worker.js"), {
+            workerData: { urlsChunk, workerId },
+        });
+        worker.on("message", (msg) => console.log(`Worker ${workerId}: ${msg}.`));
+        worker.on("error", reject);
+        worker.on("exit", (code) => {
+            if (code !== 0) {
+                reject(new Error(`Worker ${workerId} stopped with exit code ${code}.`));
+            }
+            else {
+                resolve();
+            }
+        });
+    });
+};
+export const downloader = async () => {
+    try {
+        const linksPath = path.join(__dirname, "..", "data", "scraped.json");
+        const links = JSON.parse(await fs.readFile(linksPath, "utf-8"));
+        const numWorkers = 4;
+        const chunkSize = Math.ceil(links.length / numWorkers);
+        const workers = [];
+        for (let i = 0; i < numWorkers; i++) {
+            const chunk = links.slice(i * chunkSize, (i + 1) * chunkSize);
+            workers.push(createWorker(chunk, i));
+        }
+        await Promise.all(workers);
+        console.log("Scraped files successfully.");
+    }
+    catch (error) {
+        if (error instanceof Error) {
+            console.error(`Error: ${error.message}.`);
+        }
+        else {
+            console.error(`Unknown error.`);
+        }
+    }
+};