Created a script to automatically download ddnet.sqlite and players.msgpack

This commit is contained in:
BurnyLlama 2021-11-02 00:01:11 +01:00
parent 213477e212
commit 054a84f6fd
4 changed files with 93 additions and 5 deletions

View File

@ -3,8 +3,13 @@ import { config as loadEnv } from 'dotenv'
import api from './api/api.js'
import { generateDB } from './libs/database/generate.js'
import { dbInit } from './libs/database/init.js'
import { downloadEssentialData } from './libs/download/dowload.js'
loadEnv()
await downloadEssentialData()
dbInit()
generateDB()

View File

@ -1,13 +1,13 @@
import fetch from 'node-fetch'
import { exec } from 'child_process'
import { skinDB } from '../database/init.js'
import initLog from '../utils/log.js'
import { download } from '../download/dowload.js'
const log = initLog("DDNSS")
export async function ddnssStart() {
const getServers = await fetch('https://ddnet.tw/status/index.json');
const servers = await getServers.json();
const getServers = await download('https://ddnet.tw/status/index.json', "_RETURN_VALUE_")
const servers = await getServers.json()
log(`Found ${servers.length} online servers!`)

84
libs/download/dowload.js Normal file
View File

@ -0,0 +1,84 @@
import fs from 'fs'
import https from 'https'
import initLog from '../utils/log.js'
import { exec } from 'child_process'
const log = initLog("Downloader")
/**
* This function can download and save data to files, or simply return the data.
* @param {string} url The URL of which to download from...
* @param {string} target This is the file path you want to save to. Alterntively use "_RETURN_VALUE_" to get value returned instead of saved to a file.
*
* @returns {Promise}
*
* @author BurnyLlama
*/
export function download(url, target) {
return new Promise(
(resolve, reject) => {
log(`Starting a download >> ${url}`)
https.get(
url,
data => {
if (target === "_RETURN_VALUE_") {
let result
data.on(
'data',
chunk => result += chunk
)
data.on(
'end',
() => resolve(result)
)
}
const file = fs.createWriteStream(target)
data.pipe(file)
data.on(
'end',
() => {
log(`Done with a download >> ${url}`)
file.close()
resolve()
}
)
}
)
}
)
}
export function downloadEssentialData() {
return new Promise(
(resolve, reject) => {
log("Downloading 'ddnet.sqlite.zip' and 'players.msgpack'...")
Promise.all([
download("https://ddnet.tw/stats/ddnet.sqlite.zip", process.env.DDNET_SQLITE_PATH ? `${process.env.DDNET_SQLITE_PATH}.zip` : 'data/ddnet.sqlite.zip'),
download("https://ddnet.tw/players.msgpack", process.env.MSGPACK_PATH ?? 'data/players.msgpack')
]).then(() => {
log("All downloads done! Going to unzip 'ddnet.sqlite.zip'...")
exec(
`unzip -o ${process.env.DDNET_SQLITE_PATH ? `${process.env.DDNET_SQLITE_PATH}.zip` : 'data/ddnet.sqlite.zip'} \
-d ${process.env.DDNET_SQLITE_PATH.replace(/\/[\s\S]*\.sqlite/, "") ?? 'data'}`,
err => {
if (err) {
log("Error while unzipping!")
reject()
}
log("Done unzipping!")
resolve()
}
)
})
}
)
}

View File

@ -8,8 +8,7 @@
"better-sqlite3": "^7.4.3",
"dotenv": "^10.0.0",
"express": "^4.17.1",
"mime-types": "^2.1.33",
"node-fetch": "^3.0.0"
"mime-types": "^2.1.33"
},
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"