Created a script to automatically download ddnet.sqlite and players.msgpack

index.js

@@ -3,8 +3,13 @@ import { config as loadEnv } from 'dotenv'
 import api from './api/api.js'
 import { generateDB } from './libs/database/generate.js'
 import { dbInit } from './libs/database/init.js'
+import { downloadEssentialData } from './libs/download/dowload.js'
+
 
 loadEnv()
+
+await downloadEssentialData()
+
 dbInit()
 generateDB()
 

libs/ddnss/handler.js

@@ -1,13 +1,13 @@
-import fetch from 'node-fetch'
 import { exec } from 'child_process'
 import { skinDB } from '../database/init.js'
 import initLog from '../utils/log.js'
+import { download } from '../download/dowload.js'
 
 const log = initLog("DDNSS")
 
 export async function ddnssStart() {
-    const getServers = await fetch('https://ddnet.tw/status/index.json');
-    const servers = await getServers.json();
+    const getServers = await download('https://ddnet.tw/status/index.json', "_RETURN_VALUE_")
+    const servers = await getServers.json()
 
     log(`Found ${servers.length} online servers!`)
 

libs/download/dowload.js

@@ -0,0 +1,84 @@
+import fs from 'fs'
+import https from 'https'
+import initLog from '../utils/log.js'
+import { exec } from 'child_process'
+
+const log = initLog("Downloader")
+
+
+/**
+ * This function can download and save data to files, or simply return the data.
+ * @param {string} url The URL of which to download from...
+ * @param {string} target This is the file path you want to save to. Alterntively use "_RETURN_VALUE_" to get value returned instead of saved to a file.
+ * 
+ * @returns {Promise}
+ * 
+ * @author BurnyLlama
+ */
+export function download(url, target) {
+    return new Promise(
+        (resolve, reject) => {
+            log(`Starting a download >> ${url}`)
+
+            https.get(
+                url,
+                data => {
+                    if (target === "_RETURN_VALUE_") {
+                        let result
+
+                        data.on(
+                            'data',
+                            chunk => result += chunk
+                        )
+
+                        data.on(
+                            'end',
+                            () => resolve(result)
+                        )
+                    }
+
+                    const file = fs.createWriteStream(target)
+
+                    data.pipe(file)
+
+                    data.on(
+                        'end',
+                        () => {
+                            log(`Done with a download >> ${url}`)
+                            file.close()
+                            resolve()
+                        }
+                    )
+                }
+            )
+        }
+    )
+}
+
+export function downloadEssentialData() {
+    return new Promise(
+        (resolve, reject) => {
+            log("Downloading 'ddnet.sqlite.zip' and 'players.msgpack'...")
+            Promise.all([
+                download("https://ddnet.tw/stats/ddnet.sqlite.zip", process.env.DDNET_SQLITE_PATH ? `${process.env.DDNET_SQLITE_PATH}.zip` : 'data/ddnet.sqlite.zip'),
+                download("https://ddnet.tw/players.msgpack", process.env.MSGPACK_PATH ?? 'data/players.msgpack')
+            ]).then(() => {
+                log("All downloads done! Going to unzip 'ddnet.sqlite.zip'...")
+
+                exec(
+                    `unzip -o ${process.env.DDNET_SQLITE_PATH ? `${process.env.DDNET_SQLITE_PATH}.zip` : 'data/ddnet.sqlite.zip'} \
+                           -d ${process.env.DDNET_SQLITE_PATH.replace(/\/[\s\S]*\.sqlite/, "") ?? 'data'}`,
+                    err => {
+                        if (err) {
+                            log("Error while unzipping!")
+                            reject()
+                        }
+
+                        log("Done unzipping!")
+                        resolve()
+                    }
+                )
+            })
+        }
+    )
+}

package.json

@@ -8,8 +8,7 @@
     "better-sqlite3": "^7.4.3",
     "dotenv": "^10.0.0",
     "express": "^4.17.1",
-    "mime-types": "^2.1.33",
-    "node-fetch": "^3.0.0"
+    "mime-types": "^2.1.33"
   },
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"