Major refactor... ^^

api/finishes.js

@@ -1,5 +1,5 @@
 import { Router } from 'express'
-import { sqlite } from '../libs/db/init.js'
+import { sqlite } from '../libs/database/init.js'
 
 const finishApi = Router()
 

api/graph.js

@@ -1,5 +1,5 @@
 import { Router } from 'express'
-import { sqlite } from '../libs/db/init.js'
+import { sqlite } from '../libs/database/init.js'
 
 const graphApi = Router()
 

api/maps.js

@@ -1,5 +1,5 @@
 import { Router } from 'express'
-import { sqlite } from '../libs/db/init.js'
+import { sqlite } from '../libs/database/init.js'
 
 const mapApi = Router()
 

api/players.js

@@ -1,5 +1,5 @@
 import { Router } from 'express'
-import { sqlite } from '../libs/db/init.js'
+import { sqlite } from '../libs/database/init.js'
 
 const playerApi = Router()
 

index.js

@@ -1,8 +1,8 @@
 import express from 'express'
 import dotenv from 'dotenv'
 import api from './api/api.js'
-import { generateDB } from "./libs/db/generate.js"
+import { generateDB } from "./libs/database/generate.js"
-import { sqlite, dbInit } from "./libs/db/init.js"
+import { sqlite, dbInit } from "./libs/database/init.js"
 import { ddnssStart, scrapeServer } from './libs/ddnss/handler.js'
 //import tasks from './db/tasks.js'
 

libs/database/generate.js

@@ -11,9 +11,8 @@ const log = initLog("DB Generation")
  */
 export function generateDB() {
     /* TODO: Clean this up as it is a mess */
+    /* TODO: Remove useless ones */
     log("Generating race index...")
-
-    /* Generate race index TODO: Remove useless ones */
     execMany([
         `CREATE INDEX IF NOT EXISTS "idx_race_Map_2" ON "race" ("Map","Name")`,
         `CREATE INDEX IF NOT EXISTS "idx_race_Name" ON "race" ("Name","Timestamp")`,
@@ -23,7 +22,6 @@ export function generateDB() {
         `CREATE INDEX IF NOT EXISTS "idx_race_MapNameTime" ON "race" ("Map", "Name", "Time")`
     ])
 
-    /* Create rankings table */
     log("Creating rankings table...")
     sqlite.exec(`
         CREATE TABLE IF NOT EXISTS "rankings" (
@@ -38,17 +36,19 @@ export function generateDB() {
     log("Calculating rankings for each map...")
     tasks.processRankings()
 
-    /* Generate rankings index */
     log("Generating rankings index...")
-    sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_rankings_map" ON "rankings" ("Map")`)
+    execMany([
-    sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_rankings_rank" ON "rankings" ("rank")`)
+        `CREATE INDEX IF NOT EXISTS "idx_rankings_map" ON "rankings" ("Map")`,
-    sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_rankings_player" ON "rankings" ("Name")`)
+        `CREATE INDEX IF NOT EXISTS "idx_rankings_rank" ON "rankings" ("rank")`,
+        `CREATE INDEX IF NOT EXISTS "idx_rankings_player" ON "rankings" ("Name")`
+    ])
 
-    /* Generate teamrace index */
     log("Generating teamrace index...")
-    sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrace_Map" ON "teamrace" ("Map")`);
+    execMany([
-    sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrace_ID" ON "teamrace" ("ID")`);
+        `CREATE INDEX IF NOT EXISTS "idx_teamrace_Map" ON "teamrace" ("Map")`,
-    sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrace_MapID" ON "teamrace" ("Map", "ID")`);
+        `CREATE INDEX IF NOT EXISTS "idx_teamrace_ID" ON "teamrace" ("ID")`,
+        `CREATE INDEX IF NOT EXISTS "idx_teamrace_MapID" ON "teamrace" ("Map", "ID")`
+    ])
 
     log("Creating teamrankings table...")
     sqlite.exec(`
@@ -66,9 +66,11 @@ export function generateDB() {
     tasks.processTeamRankings()
 
     log("Generating teamrankings index...")
-    sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrankings_map" ON "teamrankings" ("Map")`)
+    execMany([
-    sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrankings_rank" ON "teamrankings" ("teamrank")`)
+        `CREATE INDEX IF NOT EXISTS "idx_teamrankings_map" ON "teamrankings" ("Map")`,
-    sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrankings_player" ON "teamrankings" ("name")`)
+        `CREATE INDEX IF NOT EXISTS "idx_teamrankings_rank" ON "teamrankings" ("teamrank")`,
+        `CREATE INDEX IF NOT EXISTS "idx_teamrankings_player" ON "teamrankings" ("name")`
+    ])
 
     sqlite.exec(`
         CREATE TABLE IF NOT EXISTS "points" (
@@ -77,7 +79,6 @@ export function generateDB() {
             "points" INTEGER NOT NULL);
     `)
 
-    /* Process all types of points */
     log("Inserting points to DB...")
     tasks.processAllPoints()
 

libs/database/helper.js

@@ -3,7 +3,7 @@ import { sqlite } from './init.js'
 /**
  * This function takes an array of strings to be ran on the DB.
  * 
- * @param {array} instructions Array of instructions to be ran.
+ * @param {[string]} instructions Array of instructions to be ran.
  * @author BurnyLlama
  */
 export function execMany(instructions) {

libs/database/tasks.js

@@ -0,0 +1,150 @@
+import msgpack from '@msgpack/msgpack'
+import fs from 'fs'
+import { execMany } from './helper.js'
+
+import { sqlite } from './init.js'
+
+/**
+ * This module parses the msgpack provided by DDNet...
+ * @module db/decodeMsgpack
+ */
+export function decodeMsgpack() {
+    const data = fs.readFileSync('players.msgpack')
+    const decoded = msgpack.decodeMulti(data, { wrap: true })
+    const order = ['categories', 'maps', 'totalPoints', 'pointsRanks', 'pointsThisWeek', 'pointsThisMonth', 'teamRankPoints', 'rankPoints', 'serverRanks']
+    let final = {}
+
+    let i = 0
+    for (const part of decoded) {
+        final[order[i]] = part
+        ++i
+    }
+    return final
+}
+
+/**
+ * This generates rankings for each map...
+ * @module db/processRankings
+ */
+export function processRankings() {
+    const maps = sqlite.prepare(`SELECT map FROM maps`);
+
+    for (const map of maps.iterate())
+        sqlite
+            .prepare(`
+                INSERT INTO rankings
+                (
+                    map, name, time, timestamp, rank, server
+                )
+                SELECT   map, name, time, timestamp, rank, server
+                FROM     (
+                        SELECT   rank() OVER w AS rank,
+                                map,
+                                timestamp,
+                                NAME,
+                                min(time) AS time,
+                                server
+                        FROM     race
+                        WHERE    map = ?
+                        GROUP BY NAME window w AS (ORDER BY time) ) AS a
+                ORDER BY rank
+            `)
+            .run(map.Map)
+}
+
+/**
+ * This generates teamrankings for each map...
+ * @module db/processTeamRankings
+ */
+export function processTeamRankings() {
+    const maps = sqlite.prepare(`SELECT map FROM maps`);
+
+    for (const map of maps.iterate())
+        sqlite
+            .prepare(`
+                INSERT INTO teamrankings
+                (
+                    name, map, id, time, timestamp, server, teamrank
+                )
+                SELECT DISTINCT(r.NAME),
+                                r.map, r.id, r.time, r.timestamp,
+                                Substring(n.server, 1, 3),
+                                dense_rank() OVER w AS rank
+                FROM            ((
+                                    SELECT DISTINCT id
+                                    FROM            teamrace
+                                    WHERE           map = ?
+                                    ORDER BY        time) AS l
+                                )
+                LEFT JOIN       teamrace AS r
+                ON              l.id = r.id
+                INNER JOIN      race AS n
+                ON              r.map = n.map
+                AND             r.NAME = n.NAME
+                AND             r.time = n.time window w AS (ORDER BY r.time)
+            `)
+            .run(map.Map)
+}
+
+/**
+ * This inserts all types of points into a table...
+ * @module db/processAllPoints
+ */
+export function processAllPoints() {
+    const msgpack = decodeMsgpack()
+
+    const types = {
+        points: msgpack.pointsRanks,
+        pointsThisWeek: msgpack.pointsThisWeek,
+        pointsThisMonth: msgpack.pointsThisMonth,
+        pointsTeam: msgpack.teamRankPoints,
+        pointsRank: msgpack.rankPoints,
+    }
+
+    /* Generate tables */
+    sqlite.exec(`
+        CREATE TABLE IF NOT EXISTS "points"
+        (
+            "type" varchar(16) NOT NULL,
+            "rank" INTEGER NOT NULL,
+            "name" varchar(16) NOT NULL,
+            "points" INTEGER NOT NULL
+        );
+    `)
+
+    /* Insert data */
+    for(const type in types) {
+        let rank = 1
+
+        for (const entry of types[type]) {
+            sqlite
+                .prepare(`
+                    INSERT INTO "points"
+                    (
+                        type, rank, name, points
+                    ) VALUES (?, ?, ?, ?)
+                `)
+                .run(
+                    type,
+                    rank,
+                    entry[0],
+                    entry[1]
+                )
+
+            ++rank
+        }
+    }
+
+    /* Generate indexes */
+    execMany([
+        `CREATE INDEX IF NOT EXISTS "idx_points_type" ON "points" ("type")`,
+        `CREATE INDEX IF NOT EXISTS "Idx_points_rank" on "points" ("rank")`,
+        `CREATE INDEX IF NOT EXISTS "Idx_points_name" on "points" ("name")`
+    ])
+}
+
+export default {
+    processAllPoints,
+    processRankings,
+    processTeamRankings
+}

libs/db/tasks.js

@@ -1,141 +0,0 @@
-import msgpack from '@msgpack/msgpack'
-import fs from 'fs'
-
-import { sqlite } from "./init.js"
-
-/**
- * This module parses the msgpack provided by DDNet...
- * @module db/decodeMsgpack
- */
- export function decodeMsgpack() {
-    const data = fs.readFileSync('players.msgpack')
-    const decoded = msgpack.decodeMulti(data, {wrap: true})
-    const order = ['categories', 'maps', 'totalPoints', 'pointsRanks', 'pointsThisWeek', 'pointsThisMonth', 'teamRankPoints', 'rankPoints', 'serverRanks']
-    let final = {}
-
-    let i = 0
-    for (const part of decoded) {
-        final[order[i]] = part
-        ++i
-    }
-    return final
-}
-
-/**
- * This generates rankings for each map...
- * @module db/processRankings
- */
-export function processRankings() {
-    const maps = sqlite.prepare(`SELECT map FROM maps`);
-
-    for (const map of maps.iterate()) {
-            sqlite.prepare(
-            `
-            INSERT INTO rankings
-            (
-                map, name, time, timestamp, rank, server
-            )
-            SELECT   map, name, time, timestamp, rank, server
-            FROM     (
-                    SELECT   rank() OVER w AS rank,
-                            map,
-                            timestamp,
-                            NAME,
-                            min(time) AS time,
-                            server
-                    FROM     race
-                    WHERE    map = ?
-                    GROUP BY NAME window w AS (ORDER BY time) ) AS a
-            ORDER BY rank
-            `).run(map.Map)
-    }
-}
-
-/**
- * This generates teamrankings for each map...
- * @module db/processTeamRankings
- */
-export function processTeamRankings() {
-    const maps = sqlite.prepare(`SELECT map FROM maps`);
-
-    for (const map of maps.iterate()) {
-            sqlite.prepare(
-            `
-            INSERT INTO teamrankings
-            (
-                name, map, id, time, timestamp, server, teamrank
-            )
-            SELECT DISTINCT(r.NAME),
-                            r.map, r.id, r.time, r.timestamp,
-                            Substring(n.server, 1, 3),
-                            dense_rank() OVER w AS rank
-            FROM            ((
-                                SELECT DISTINCT id
-                                FROM            teamrace
-                                WHERE           map = ?
-                                ORDER BY        time) AS l
-                            )
-            LEFT JOIN       teamrace AS r
-            ON              l.id = r.id
-            INNER JOIN      race AS n
-            ON              r.map = n.map
-            AND             r.NAME = n.NAME
-            AND             r.time = n.time window w AS (ORDER BY r.time)
-            `).run(map.Map)
-    }
-}
-
-/**
- * This inserts all types of points into a table...
- * @module db/processAllPoints
- */
-export function processAllPoints() {
-    const msgpack = decodeMsgpack()
-
-    let types = {
-        points: msgpack.pointsRanks,
-        pointsThisWeek: msgpack.pointsThisWeek,
-        pointsThisMonth: msgpack.pointsThisMonth,
-        pointsTeam: msgpack.teamRankPoints,
-        pointsRank: msgpack.rankPoints,
-    }
-
-    /* Generate tables */
-    sqlite.exec(
-    `
-    CREATE TABLE IF NOT EXISTS "points"
-    (
-        "type" varchar(16) NOT NULL,
-        "rank" INTEGER NOT NULL,
-        "name" varchar(16) NOT NULL,
-        "points" INTEGER NOT NULL);
-    `)
-
-    /* Insert data */
-    for(const type in types) {
-        let rank = 1
-
-        for (const entry of types[type]) {
-            sqlite.prepare(
-            `
-            INSERT INTO "points"
-            (
-                type, rank, name, points
-            ) VALUES (?, ?, ?, ?)`).run(
-            type, rank, entry[0], entry[1])
-
-            ++rank
-        }
-    }
-
-    /* Generate indexes */
-    sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_points_type" ON "points" ("type")`)
-    sqlite.exec(`CREATE INDEX IF NOT EXISTS "Idx_points_rank" on "points" ("rank")`)
-    sqlite.exec(`CREATE INDEX IF NOT EXISTS "Idx_points_name" on "points" ("name")`)
-}
-
-export default {
-    processAllPoints,
-    processRankings,
-    processTeamRankings
-}

libs/ddnss/handler.js

@@ -1,85 +1,79 @@
 import fetch from 'node-fetch'
 import { exec } from 'child_process'
-import { skinDB } from "../db/init.js"
+import { skinDB } from '../database/init.js'
+import initLog from '../utils/log.js'
+
+const log = initLog("DDNSS")
 
 export async function ddnssStart() {
-    const response = await fetch('https://ddnet.tw/status/index.json');
+    const getServers = await fetch('https://ddnet.tw/status/index.json');
-    const data = await response.json();
+    const servers = await getServers.json();
 
-    console.log(data)
+    console.log(servers)
 
-    for (const servers of data) {
+    for (const server of servers) {
-        /* Check if server isn't empty and not full */
+        const connection = `${server.ip}:${server.port}`
-        if (servers.num_clients > 0 && servers.num_clients < 59) {
-            let server = `${servers.ip}:${servers.port}`
-            console.log(`Connecting: ${servers.ip}:${servers.port}`)
 
-            /* exec ddnss */
+        if (!(server.num_clients > 0 && server.num_clients < (server.max_clients - 2)))
-            await scrapeServer(`${server}`)
+            return log(`Server (essentially) full! >> ${server.ip}:${server.port} -> ${server.num_clients}/${server.max_clients} clients`)
-        }
+
-        else
+        log(`Connecting to server >> ${connection}`)
-            console.log(`${servers.num_clients}/63 Server full: ${servers.ip}:${servers.port}`)
+        await scrapeServer(`${connection}`)
     }
-    /* A bit hacky way of killing ddnss */
+
     exec(`pkill -9 -f ddnss`)
 }
 
-export function scrapeServer(server) {
+function scrapeServer(server) {
-    let command = `./ddnss/build/DDNet "ui_server_address ${server}" -f ddnss/build/config.conf`
-    let skinData
 
-    return new Promise((done, failed) => {
+    const command = `./ddnss/build/DDNet "ui_server_address ${server}" -f ddnss/build/config.conf`
-        exec(command, { encoding: 'utf8', timeout: 10000 }, (err, stdout, stderr) => {
-            if (err) {
-                err.stdout = stdout
-                err.stderr = stderr
-            }
 
-            /* Handle error from parsing of JSON */
+    return new Promise((resolve, reject) => {
+        exec(command, { encoding: 'utf8' }, (err, stdout, stderr) => {
             try {
-                skinData = JSON.parse(stdout)
+                const skinData = JSON.parse(stdout)
+
+                if (skinData === null) {
+                    resolve()
+                    return
+                }
+    
+                const currentTime = Date.now()
+    
+                for (const entry of skinData)
+                    skinDB
+                        .prepare(`
+                            INSERT INTO "skindata"
+                            (
+                                timestamp,
+                                player,
+                                clan,
+                                flag,
+                                skin,
+                                useColor,
+                                colorBodyRaw,
+                                colorBodyHex,
+                                colorFeetRaw,
+                                ColorFeetHex
+                            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                        `)
+                        .run(
+                            currentTime,
+                            entry.player,
+                            entry.clan,
+                            entry.flag,
+                            entry.skindata.skin,
+                            entry.skindata.useColor,
+                            entry.skindata.colorBody.raw,
+                            entry.skindata.colorBody.hex,
+                            entry.skindata.colorFeet.raw,
+                            entry.skindata.colorFeet.hex,
+                        )
             } catch (e) {
-                done()
+                log(`Failed to handle ${server}!`)
-                return
             }
 
-            if (skinData === null) {
+            resolve()
-                done()
-                return
-            }
-
-            /* Get timestamp */
-            let ts = (Date.now())
-
-            /* Insert skindata */
-            for (const entry of skinData) {
-                skinDB.prepare(`INSERT INTO "skindata"
-                (
-                    timestamp,
-                    player,
-                    clan,
-                    flag,
-                    skin,
-                    useColor,
-                    colorBodyRaw,
-                    colorBodyHex,
-                    colorFeetRaw,
-                    ColorFeetHex
-                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).
-                    run(
-                        ts,
-                        entry.player,
-                        entry.clan,
-                        entry.flag,
-                        entry.skindata.skin,
-                        entry.skindata.useColor,
-                        entry.skindata.colorBody.raw,
-                        entry.skindata.colorBody.hex,
-                        entry.skindata.colorFeet.raw,
-                        entry.skindata.colorFeet.hex,
-                    )
-            }
-            done()
         })
     })
 }