39 lines
1.3 KiB
JavaScript
39 lines
1.3 KiB
JavaScript
import { workerData, parentPort } from "worker_threads";
|
|
import fetch from "node-fetch";
|
|
import fs from "fs-extra";
|
|
import path from "path";
|
|
import { fileURLToPath } from "url";
|
|
import { randomBytes } from "crypto";
|
|
const generateRandomFilename = () => randomBytes(8).toString("hex");
|
|
const __filename = fileURLToPath(import.meta.url);
|
|
const __dirname = path.dirname(__filename);
|
|
const imgsDir = path.join(__dirname, "..", "..", "files");
|
|
fs.ensureDirSync(imgsDir);
|
|
const downloadFile = async (url) => {
|
|
try {
|
|
const response = await fetch(url);
|
|
if (!response.ok)
|
|
throw new Error(`Failed to fetch ${url}.`);
|
|
const buffer = await response.buffer();
|
|
const fileName = path.basename(url.split("?")[0]);
|
|
const filePath = path.join(imgsDir, `${generateRandomFilename()}-${fileName}`);
|
|
await fs.outputFile(filePath, buffer);
|
|
parentPort?.postMessage(`Scraped ${url} to ${filePath}.`);
|
|
}
|
|
catch (error) {
|
|
if (error instanceof Error) {
|
|
parentPort?.postMessage(`Error scraping ${url}: ${error.message}.`);
|
|
}
|
|
else {
|
|
parentPort?.postMessage(`Unknown error scraping ${url}.`);
|
|
}
|
|
}
|
|
};
|
|
const main = async () => {
|
|
const { urlsChunk } = workerData;
|
|
for (const url of urlsChunk) {
|
|
await downloadFile(url);
|
|
}
|
|
};
|
|
main();
|