Upload of project source code files.

src/index.ts

@@ -0,0 +1,94 @@
+import fetch from "node-fetch";
+import fs from "fs-extra";
+import { token, channels_to_fetch } from "../data/config.js";
+import { isMainThread } from "worker_threads";
+import { downloader } from "./downloader.js";
+
+type Attachment = {
+  id: string;
+  proxy_url: string;
+};
+
+type DiscordMessage = {
+  id: string;
+  attachments: Attachment[];
+};
+
+const createDataFolder = () => {
+  if (!fs.existsSync("../data")) {
+    fs.mkdirSync("../data");
+  }
+};
+
+const request = async (channel_id: string, before?: string) => {
+  const options = {
+    method: "GET",
+    headers: {
+      Authorization: token,
+      Accept: "application/json",
+    },
+  };
+
+  const url = `https://discord.com/api/channels/${channel_id}/messages?limit=100${
+    before ? `&before=${before}` : ""
+  }`;
+  const response = await fetch(url, options);
+  return (await response.json()) as DiscordMessage[];
+};
+
+const getAllMessages = async (channel_id: string) => {
+  let page: DiscordMessage[] = await request(channel_id);
+  let result: DiscordMessage[] = page;
+
+  while (page.length >= 100) {
+    page = await request(channel_id, page[page.length - 1].id);
+    result = result.concat(page);
+  }
+
+  return result;
+};
+
+const getAllAttachments = async (channel_id: string) => {
+  const messages = await getAllMessages(channel_id);
+  const attachments = messages
+    .map((msg) => msg.attachments)
+    .flat()
+    .map((attachment) => attachment.proxy_url);
+
+  return attachments.filter((attachment: string) => attachment);
+};
+
+const saveToFile = (fileName: string, data: any) => {
+  fs.writeFileSync(`../data/${fileName}`, JSON.stringify(data, null, 2));
+};
+
+const main = async () => {
+  console.log("Scraping links...");
+
+  let allAttachments: string[] = [];
+
+  for (const channel_id of channels_to_fetch) {
+    const channelAttachments = await getAllAttachments(channel_id);
+    allAttachments = allAttachments.concat(channelAttachments);
+    console.log(
+      `Scraped ${channelAttachments.length} attachment links from channel ${channel_id}.`
+    );
+  }
+
+  const uniqueAttachments = [...new Set(allAttachments)];
+
+  createDataFolder();
+  saveToFile("scraped.json", uniqueAttachments);
+
+  console.log(
+    `Scraped and saved ${uniqueAttachments.length} attachment links.`
+  );
+
+  console.log("Scraping files...");
+
+  if (isMainThread) {
+    downloader();
+  }
+};
+
+main();