Rewrite Mediafire Scraper

package.json

@@ -30,6 +30,7 @@
         "packages/scraper-games",
         "packages/scraper-images",
         "packages/scraper-instagram",
+        "packages/scraper-mediafire",
         "packages/scraper-news",
         "packages/scraper-others",
         "packages/scraper-primbon",

packages/scraper-instagram/src/util.ts

@@ -0,0 +1,23 @@
+import crypto from 'crypto'
+
+const SUFFIX = 'fa74751e1d977d092e2ee0b7467a8cb4448fb54b31284d96ac6a2324fb3'
+
+export function generateHash (url: string) {
+    const data = url + Date.now() + SUFFIX
+    const hash = crypto.createHash('sha256')
+    hash.update(data)
+    return hash.digest('hex')
+}
+
+export function stringifyCookies (cookies: string[]) {
+    return cookies.map((cookie) => cookie.split(';')[0])
+        .join('; ')
+}
+export function parseCookies (cookie: string): { [Key: string]: string } {
+    const cookies = cookie.split(';')
+    return cookies.reduce((prev, curr) => {
+        const [key, value] = curr.trim().split('=');
+        prev[key] = value
+        return prev
+    }, {})
+}

packages/scraper-instagram/types/instagramdl-v1.ts

@@ -3,7 +3,7 @@ import { z } from 'zod'
 export const InstagramdlArgsSchema = z.object({
     0: z.string().url()
 })
-export const InstagramdlItemSchema =z.object({
+export const InstagramdlItemSchema = z.object({
     thumbnail: z.string().url(),
     url: z.string().url()
 })

packages/scraper-mediafire/index.ts

@@ -0,0 +1 @@
+export { default as mediafiredl } from './src/mediafire-dl.js'

packages/scraper-mediafire/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "@bochilteam/scraper-mediafire",
+  "version": "0.0.1",
+  "description": "Mediafire scraper module",
+  "main": "lib/cjs/index.js",
+  "module": "lib/esm/index.js",
+  "types": "lib/@types/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./lib/esm/index.js",
+      "require": "./lib/cjs/index.js",
+      "types": "./lib/@types/index.d.ts"
+    }
+  },
+  "scripts": {
+    "test": "npm run test:cjs & npm run test:esm",
+    "test:cjs": "node --test ./lib/cjs/test",
+    "test:esm": "node --test ./lib/esm/test",
+    "build": "npm run build:cjs && npm run build:esm && npm run build:types",
+    "build:cjs": "tsc -p tsconfig.cjs.json",
+    "build:esm": "tsc -p tsconfig.esm.json",
+    "build:types": "tsc -p tsconfig.types.json",
+    "compile": "node ../../scripts/compile-typescript.js && npm run write:package",
+    "write:package": "node ../../scripts/write-package.js"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/BochilTeam/scraper.git"
+  },
+  "author": "BochilTeam",
+  "license": "GPL-3.0-or-later",
+  "bugs": {
+    "url": "https://github.com/BochilTeam/scraper/issues"
+  },
+  "homepage": "https://github.com/BochilTeam/scraper/tree/master/packages/scraper-mediafire#readme",
+  "files": [
+    "lib/**/*.js",
+    "lib/**/**.d.ts",
+    "lib/**/package.json"
+  ],
+  "dependencies": {
+    "got": "^11.8.6",
+    "human-readable": "^0.2.1",
+    "zod": "^3.20.2"
+  }
+}

packages/scraper-mediafire/src/constant.ts

@@ -0,0 +1,11 @@
+export const DEFAULT_HEADERS = {
+    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+    'accept-encoding': 'gzip, deflate, br',
+    'accept-language': 'en-US,en;q=0.9',
+    'sec-ch-ua': '"Google Chrome";v="117", "Not;A=Brand";v="8", "Chromium";v="117"',
+    'sec-ch-ua-mobile': '?0',
+    'sec-fetch-dest': 'empty',
+    'sec-fetch-mode': 'cors',
+    'sec-fetch-site': 'same-origin',
+    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36'
+}

packages/scraper-mediafire/src/mediafire-dl.ts

@@ -0,0 +1,38 @@
+import got from 'got'
+import cheerio from 'cheerio'
+import { DEFAULT_HEADERS } from './constant.js'
+import { parseFileSize } from './util.js'
+import { MediafiredlArgsSchema, MediafiredlSchema } from '../types/mediafire-dl.js'
+
+export default async function mediafiredl (url: string) {
+    MediafiredlArgsSchema.parse(arguments)
+
+    const data = await got(url, {
+        headers: {
+            ...DEFAULT_HEADERS
+        }
+    }).text()
+    const $ = cheerio.load(data)
+    const Url = ($('#downloadButton').attr('href') || '').trim()
+    const url2 = ($('#download_link > a.retry').attr('href') || '').trim()
+    const $intro = $('div.dl-info > div.intro')
+    const filename = $intro.find('div.filename').text().trim()
+    const filetype = $intro.find('div.filetype > span').eq(0).text().trim()
+    const ext = /\(\.(.*?)\)/.exec($intro.find('div.filetype > span').eq(1).text())?.[1]?.trim() || 'bin'
+    const $li = $('div.dl-info > ul.details > li')
+    const aploud = $li.eq(1).find('span').text().trim()
+    const filesizeH = $li.eq(0).find('span').text().trim()
+    const filesize = parseFileSize(filesizeH)
+
+    const result = {
+        url: Url || url2,
+        url2,
+        filename,
+        filetype,
+        ext,
+        aploud,
+        filesizeH,
+        filesize
+    }
+    return MediafiredlSchema.parse(result)
+}

packages/scraper-mediafire/src/util.ts

@@ -0,0 +1,18 @@
+/**
+ * @returns as a kilobit
+ */
+export function parseFileSize (size: string): number {
+    return parseFloat(size) * (
+      /GB/i.test(size)
+        ? 1000000
+        : /MB/i.test(size)
+          ? 1000
+          : /KB/i.test(size)
+            ? 1
+            : /bytes?/i.test(size)
+              ? 0.001
+              : /B/i.test(size)
+                ? 0.1
+                : 0
+    )
+  }

packages/scraper-mediafire/test/mediafire-dl.ts

@@ -0,0 +1,21 @@
+import got from 'got'
+import { test } from 'node:test'
+import assert from 'node:assert'
+import { mediafiredl } from '../index.js'
+import type { Mediafiredl } from '../types/mediafire-dl.js'
+
+const MEDIAFIRE_URL = 'https://www.mediafire.com/file/10q5c5wmzymsmhc/package.json/file'
+
+test('Mediafire Downloader', async (t) => {
+    let metadata: Mediafiredl
+    await t.test('Getting Metadata', async () => {
+        metadata = await mediafiredl(MEDIAFIRE_URL)
+    })
+
+    await t.test('Download Content', async () => {
+        if (!metadata)
+            return t.skip('Test skipped -- error in getting metadata!')
+        const buffer = await got(metadata.url).buffer()
+        assert.strictEqual(buffer.byteLength > 0, true)
+    })
+})

packages/scraper-mediafire/tsconfig.cjs.json

@@ -0,0 +1,14 @@
+{
+    "extends": "../../tsconfig.cjs.json",
+    "compilerOptions": {
+        "outDir": "./lib/cjs"
+    },
+    "include": [
+        "./index.ts",
+        "./src/**/*.ts",
+        "./test/*.ts"
+    ],
+    "exclude": [
+        "./node_modules/**"
+    ]
+}

packages/scraper-mediafire/tsconfig.esm.json

@@ -0,0 +1,14 @@
+{
+    "extends": "../../tsconfig.esm.json",
+    "compilerOptions": {
+        "outDir": "./lib/esm"
+    },
+    "include": [
+        "./index.ts",
+        "./src/**/*.ts",
+        "./test/*.ts"
+    ],
+    "exclude": [
+        "./node_modules/**"
+    ]
+}

packages/scraper-mediafire/tsconfig.types.json

@@ -0,0 +1,14 @@
+{
+    "extends": "../../tsconfig.types.json",
+    "compilerOptions": {
+        "outDir": "./lib/@types"
+    },
+    "include": [
+        "./index.ts",
+        "./src/**/*.ts",
+        "./test/*.ts"
+    ],
+    "exclude": [
+        "./node_modules/**"
+    ]
+}

packages/scraper-mediafire/typedoc.json

@@ -0,0 +1,10 @@
+{
+  "extends": [
+    "../../typedoc.base.json"
+  ],
+  "entryPoints": [
+    "index.ts",
+    "types/*"
+  ],
+  "tsconfig": "../../tsconfig.base.json"
+}

packages/scraper-mediafire/types/mediafire-dl.ts

@@ -0,0 +1,16 @@
+import { z } from 'zod'
+
+export const MediafiredlArgsSchema = z.object({
+    0: z.string().url()
+})
+export const MediafiredlSchema = z.object({
+    url: z.string().url(),
+    url2: z.string().url(),
+    filename: z.string(),
+    filetype: z.string(),
+    ext: z.string(),
+    aploud: z.string(),
+    filesizeH: z.string(),
+    filesize: z.number()
+})
+export type Mediafiredl = z.infer<typeof MediafiredlSchema>