From bdd0b44561695d80872a38d5ca8808518ec63f23 Mon Sep 17 00:00:00 2001 From: BurnyLlama Date: Sun, 31 Oct 2021 23:43:36 +0100 Subject: [PATCH] Major refactor... ^^ --- api/finishes.js | 2 +- api/graph.js | 2 +- api/maps.js | 2 +- api/players.js | 2 +- index.js | 4 +- libs/{db => database}/generate.js | 31 +++--- libs/{db => database}/helper.js | 2 +- libs/{db => database}/init.js | 0 libs/database/tasks.js | 150 ++++++++++++++++++++++++++++++ libs/db/tasks.js | 141 ---------------------------- libs/ddnss/handler.js | 124 ++++++++++++------------ 11 files changed, 232 insertions(+), 228 deletions(-) rename libs/{db => database}/generate.js (73%) rename libs/{db => database}/helper.js (79%) rename libs/{db => database}/init.js (100%) create mode 100644 libs/database/tasks.js delete mode 100644 libs/db/tasks.js diff --git a/api/finishes.js b/api/finishes.js index f56655e..1fd68f6 100644 --- a/api/finishes.js +++ b/api/finishes.js @@ -1,5 +1,5 @@ import { Router } from 'express' -import { sqlite } from '../libs/db/init.js' +import { sqlite } from '../libs/database/init.js' const finishApi = Router() diff --git a/api/graph.js b/api/graph.js index 0a00067..c934aaf 100644 --- a/api/graph.js +++ b/api/graph.js @@ -1,5 +1,5 @@ import { Router } from 'express' -import { sqlite } from '../libs/db/init.js' +import { sqlite } from '../libs/database/init.js' const graphApi = Router() diff --git a/api/maps.js b/api/maps.js index 5ec29c3..d80d193 100644 --- a/api/maps.js +++ b/api/maps.js @@ -1,5 +1,5 @@ import { Router } from 'express' -import { sqlite } from '../libs/db/init.js' +import { sqlite } from '../libs/database/init.js' const mapApi = Router() diff --git a/api/players.js b/api/players.js index 75131cc..b8e82aa 100644 --- a/api/players.js +++ b/api/players.js @@ -1,5 +1,5 @@ import { Router } from 'express' -import { sqlite } from '../libs/db/init.js' +import { sqlite } from '../libs/database/init.js' const playerApi = Router() diff --git a/index.js b/index.js index 59388e6..eb79dd5 100644 --- a/index.js +++ b/index.js @@ -1,8 +1,8 @@ import express from 'express' import dotenv from 'dotenv' import api from './api/api.js' -import { generateDB } from "./libs/db/generate.js" -import { sqlite, dbInit } from "./libs/db/init.js" +import { generateDB } from "./libs/database/generate.js" +import { sqlite, dbInit } from "./libs/database/init.js" import { ddnssStart, scrapeServer } from './libs/ddnss/handler.js' //import tasks from './db/tasks.js' diff --git a/libs/db/generate.js b/libs/database/generate.js similarity index 73% rename from libs/db/generate.js rename to libs/database/generate.js index d147867..10fa71e 100644 --- a/libs/db/generate.js +++ b/libs/database/generate.js @@ -11,9 +11,8 @@ const log = initLog("DB Generation") */ export function generateDB() { /* TODO: Clean this up as it is a mess */ + /* TODO: Remove useless ones */ log("Generating race index...") - - /* Generate race index TODO: Remove useless ones */ execMany([ `CREATE INDEX IF NOT EXISTS "idx_race_Map_2" ON "race" ("Map","Name")`, `CREATE INDEX IF NOT EXISTS "idx_race_Name" ON "race" ("Name","Timestamp")`, @@ -23,7 +22,6 @@ export function generateDB() { `CREATE INDEX IF NOT EXISTS "idx_race_MapNameTime" ON "race" ("Map", "Name", "Time")` ]) - /* Create rankings table */ log("Creating rankings table...") sqlite.exec(` CREATE TABLE IF NOT EXISTS "rankings" ( @@ -38,17 +36,19 @@ export function generateDB() { log("Calculating rankings for each map...") tasks.processRankings() - /* Generate rankings index */ log("Generating rankings index...") - sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_rankings_map" ON "rankings" ("Map")`) - sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_rankings_rank" ON "rankings" ("rank")`) - sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_rankings_player" ON "rankings" ("Name")`) + execMany([ + `CREATE INDEX IF NOT EXISTS "idx_rankings_map" ON "rankings" ("Map")`, + `CREATE INDEX IF NOT EXISTS "idx_rankings_rank" ON "rankings" ("rank")`, + `CREATE INDEX IF NOT EXISTS "idx_rankings_player" ON "rankings" ("Name")` + ]) - /* Generate teamrace index */ log("Generating teamrace index...") - sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrace_Map" ON "teamrace" ("Map")`); - sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrace_ID" ON "teamrace" ("ID")`); - sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrace_MapID" ON "teamrace" ("Map", "ID")`); + execMany([ + `CREATE INDEX IF NOT EXISTS "idx_teamrace_Map" ON "teamrace" ("Map")`, + `CREATE INDEX IF NOT EXISTS "idx_teamrace_ID" ON "teamrace" ("ID")`, + `CREATE INDEX IF NOT EXISTS "idx_teamrace_MapID" ON "teamrace" ("Map", "ID")` + ]) log("Creating teamrankings table...") sqlite.exec(` @@ -66,9 +66,11 @@ export function generateDB() { tasks.processTeamRankings() log("Generating teamrankings index...") - sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrankings_map" ON "teamrankings" ("Map")`) - sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrankings_rank" ON "teamrankings" ("teamrank")`) - sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrankings_player" ON "teamrankings" ("name")`) + execMany([ + `CREATE INDEX IF NOT EXISTS "idx_teamrankings_map" ON "teamrankings" ("Map")`, + `CREATE INDEX IF NOT EXISTS "idx_teamrankings_rank" ON "teamrankings" ("teamrank")`, + `CREATE INDEX IF NOT EXISTS "idx_teamrankings_player" ON "teamrankings" ("name")` + ]) sqlite.exec(` CREATE TABLE IF NOT EXISTS "points" ( @@ -77,7 +79,6 @@ export function generateDB() { "points" INTEGER NOT NULL); `) - /* Process all types of points */ log("Inserting points to DB...") tasks.processAllPoints() diff --git a/libs/db/helper.js b/libs/database/helper.js similarity index 79% rename from libs/db/helper.js rename to libs/database/helper.js index 8382380..1565b68 100644 --- a/libs/db/helper.js +++ b/libs/database/helper.js @@ -3,7 +3,7 @@ import { sqlite } from './init.js' /** * This function takes an array of strings to be ran on the DB. * - * @param {array} instructions Array of instructions to be ran. + * @param {[string]} instructions Array of instructions to be ran. * @author BurnyLlama */ export function execMany(instructions) { diff --git a/libs/db/init.js b/libs/database/init.js similarity index 100% rename from libs/db/init.js rename to libs/database/init.js diff --git a/libs/database/tasks.js b/libs/database/tasks.js new file mode 100644 index 0000000..2d81072 --- /dev/null +++ b/libs/database/tasks.js @@ -0,0 +1,150 @@ +import msgpack from '@msgpack/msgpack' +import fs from 'fs' +import { execMany } from './helper.js' + +import { sqlite } from './init.js' + +/** + * This module parses the msgpack provided by DDNet... + * @module db/decodeMsgpack + */ +export function decodeMsgpack() { + const data = fs.readFileSync('players.msgpack') + const decoded = msgpack.decodeMulti(data, { wrap: true }) + const order = ['categories', 'maps', 'totalPoints', 'pointsRanks', 'pointsThisWeek', 'pointsThisMonth', 'teamRankPoints', 'rankPoints', 'serverRanks'] + let final = {} + + let i = 0 + for (const part of decoded) { + final[order[i]] = part + ++i + } + return final +} + +/** + * This generates rankings for each map... + * @module db/processRankings + */ +export function processRankings() { + const maps = sqlite.prepare(`SELECT map FROM maps`); + + for (const map of maps.iterate()) + sqlite + .prepare(` + INSERT INTO rankings + ( + map, name, time, timestamp, rank, server + ) + SELECT map, name, time, timestamp, rank, server + FROM ( + SELECT rank() OVER w AS rank, + map, + timestamp, + NAME, + min(time) AS time, + server + FROM race + WHERE map = ? + GROUP BY NAME window w AS (ORDER BY time) ) AS a + ORDER BY rank + `) + .run(map.Map) +} + +/** + * This generates teamrankings for each map... + * @module db/processTeamRankings + */ +export function processTeamRankings() { + const maps = sqlite.prepare(`SELECT map FROM maps`); + + for (const map of maps.iterate()) + sqlite + .prepare(` + INSERT INTO teamrankings + ( + name, map, id, time, timestamp, server, teamrank + ) + SELECT DISTINCT(r.NAME), + r.map, r.id, r.time, r.timestamp, + Substring(n.server, 1, 3), + dense_rank() OVER w AS rank + FROM (( + SELECT DISTINCT id + FROM teamrace + WHERE map = ? + ORDER BY time) AS l + ) + LEFT JOIN teamrace AS r + ON l.id = r.id + INNER JOIN race AS n + ON r.map = n.map + AND r.NAME = n.NAME + AND r.time = n.time window w AS (ORDER BY r.time) + `) + .run(map.Map) +} + +/** + * This inserts all types of points into a table... + * @module db/processAllPoints + */ +export function processAllPoints() { + const msgpack = decodeMsgpack() + + const types = { + points: msgpack.pointsRanks, + pointsThisWeek: msgpack.pointsThisWeek, + pointsThisMonth: msgpack.pointsThisMonth, + pointsTeam: msgpack.teamRankPoints, + pointsRank: msgpack.rankPoints, + } + + /* Generate tables */ + sqlite.exec(` + CREATE TABLE IF NOT EXISTS "points" + ( + "type" varchar(16) NOT NULL, + "rank" INTEGER NOT NULL, + "name" varchar(16) NOT NULL, + "points" INTEGER NOT NULL + ); + `) + + /* Insert data */ + for(const type in types) { + let rank = 1 + + for (const entry of types[type]) { + sqlite + .prepare(` + INSERT INTO "points" + ( + type, rank, name, points + ) VALUES (?, ?, ?, ?) + `) + .run( + type, + rank, + entry[0], + entry[1] + ) + + ++rank + } + } + + /* Generate indexes */ + execMany([ + `CREATE INDEX IF NOT EXISTS "idx_points_type" ON "points" ("type")`, + `CREATE INDEX IF NOT EXISTS "Idx_points_rank" on "points" ("rank")`, + `CREATE INDEX IF NOT EXISTS "Idx_points_name" on "points" ("name")` + ]) +} + +export default { + processAllPoints, + processRankings, + processTeamRankings +} \ No newline at end of file diff --git a/libs/db/tasks.js b/libs/db/tasks.js deleted file mode 100644 index ef3ca8e..0000000 --- a/libs/db/tasks.js +++ /dev/null @@ -1,141 +0,0 @@ -import msgpack from '@msgpack/msgpack' -import fs from 'fs' - -import { sqlite } from "./init.js" - -/** - * This module parses the msgpack provided by DDNet... - * @module db/decodeMsgpack - */ - export function decodeMsgpack() { - const data = fs.readFileSync('players.msgpack') - const decoded = msgpack.decodeMulti(data, {wrap: true}) - const order = ['categories', 'maps', 'totalPoints', 'pointsRanks', 'pointsThisWeek', 'pointsThisMonth', 'teamRankPoints', 'rankPoints', 'serverRanks'] - let final = {} - - let i = 0 - for (const part of decoded) { - final[order[i]] = part - ++i - } - return final -} - -/** - * This generates rankings for each map... - * @module db/processRankings - */ -export function processRankings() { - const maps = sqlite.prepare(`SELECT map FROM maps`); - - for (const map of maps.iterate()) { - sqlite.prepare( - ` - INSERT INTO rankings - ( - map, name, time, timestamp, rank, server - ) - SELECT map, name, time, timestamp, rank, server - FROM ( - SELECT rank() OVER w AS rank, - map, - timestamp, - NAME, - min(time) AS time, - server - FROM race - WHERE map = ? - GROUP BY NAME window w AS (ORDER BY time) ) AS a - ORDER BY rank - `).run(map.Map) - } -} - -/** - * This generates teamrankings for each map... - * @module db/processTeamRankings - */ -export function processTeamRankings() { - const maps = sqlite.prepare(`SELECT map FROM maps`); - - for (const map of maps.iterate()) { - sqlite.prepare( - ` - INSERT INTO teamrankings - ( - name, map, id, time, timestamp, server, teamrank - ) - SELECT DISTINCT(r.NAME), - r.map, r.id, r.time, r.timestamp, - Substring(n.server, 1, 3), - dense_rank() OVER w AS rank - FROM (( - SELECT DISTINCT id - FROM teamrace - WHERE map = ? - ORDER BY time) AS l - ) - LEFT JOIN teamrace AS r - ON l.id = r.id - INNER JOIN race AS n - ON r.map = n.map - AND r.NAME = n.NAME - AND r.time = n.time window w AS (ORDER BY r.time) - `).run(map.Map) - } -} - -/** - * This inserts all types of points into a table... - * @module db/processAllPoints - */ -export function processAllPoints() { - const msgpack = decodeMsgpack() - - let types = { - points: msgpack.pointsRanks, - pointsThisWeek: msgpack.pointsThisWeek, - pointsThisMonth: msgpack.pointsThisMonth, - pointsTeam: msgpack.teamRankPoints, - pointsRank: msgpack.rankPoints, - } - - /* Generate tables */ - sqlite.exec( - ` - CREATE TABLE IF NOT EXISTS "points" - ( - "type" varchar(16) NOT NULL, - "rank" INTEGER NOT NULL, - "name" varchar(16) NOT NULL, - "points" INTEGER NOT NULL); - `) - - /* Insert data */ - for(const type in types) { - let rank = 1 - - for (const entry of types[type]) { - sqlite.prepare( - ` - INSERT INTO "points" - ( - type, rank, name, points - ) VALUES (?, ?, ?, ?)`).run( - type, rank, entry[0], entry[1]) - - ++rank - } - } - - /* Generate indexes */ - sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_points_type" ON "points" ("type")`) - sqlite.exec(`CREATE INDEX IF NOT EXISTS "Idx_points_rank" on "points" ("rank")`) - sqlite.exec(`CREATE INDEX IF NOT EXISTS "Idx_points_name" on "points" ("name")`) -} - -export default { - processAllPoints, - processRankings, - processTeamRankings -} \ No newline at end of file diff --git a/libs/ddnss/handler.js b/libs/ddnss/handler.js index 3f30ab1..939efba 100644 --- a/libs/ddnss/handler.js +++ b/libs/ddnss/handler.js @@ -1,85 +1,79 @@ import fetch from 'node-fetch' import { exec } from 'child_process' -import { skinDB } from "../db/init.js" +import { skinDB } from '../database/init.js' +import initLog from '../utils/log.js' + +const log = initLog("DDNSS") export async function ddnssStart() { - const response = await fetch('https://ddnet.tw/status/index.json'); - const data = await response.json(); + const getServers = await fetch('https://ddnet.tw/status/index.json'); + const servers = await getServers.json(); - console.log(data) + console.log(servers) - for (const servers of data) { - /* Check if server isn't empty and not full */ - if (servers.num_clients > 0 && servers.num_clients < 59) { - let server = `${servers.ip}:${servers.port}` - console.log(`Connecting: ${servers.ip}:${servers.port}`) + for (const server of servers) { + const connection = `${server.ip}:${server.port}` - /* exec ddnss */ - await scrapeServer(`${server}`) - } - else - console.log(`${servers.num_clients}/63 Server full: ${servers.ip}:${servers.port}`) + if (!(server.num_clients > 0 && server.num_clients < (server.max_clients - 2))) + return log(`Server (essentially) full! >> ${server.ip}:${server.port} -> ${server.num_clients}/${server.max_clients} clients`) + + log(`Connecting to server >> ${connection}`) + await scrapeServer(`${connection}`) } - /* A bit hacky way of killing ddnss */ + exec(`pkill -9 -f ddnss`) } -export function scrapeServer(server) { - let command = `./ddnss/build/DDNet "ui_server_address ${server}" -f ddnss/build/config.conf` - let skinData +function scrapeServer(server) { - return new Promise((done, failed) => { - exec(command, { encoding: 'utf8', timeout: 10000 }, (err, stdout, stderr) => { - if (err) { - err.stdout = stdout - err.stderr = stderr - } + const command = `./ddnss/build/DDNet "ui_server_address ${server}" -f ddnss/build/config.conf` - /* Handle error from parsing of JSON */ + return new Promise((resolve, reject) => { + exec(command, { encoding: 'utf8' }, (err, stdout, stderr) => { try { - skinData = JSON.parse(stdout) + const skinData = JSON.parse(stdout) + + if (skinData === null) { + resolve() + return + } + + const currentTime = Date.now() + + for (const entry of skinData) + skinDB + .prepare(` + INSERT INTO "skindata" + ( + timestamp, + player, + clan, + flag, + skin, + useColor, + colorBodyRaw, + colorBodyHex, + colorFeetRaw, + ColorFeetHex + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `) + .run( + currentTime, + entry.player, + entry.clan, + entry.flag, + entry.skindata.skin, + entry.skindata.useColor, + entry.skindata.colorBody.raw, + entry.skindata.colorBody.hex, + entry.skindata.colorFeet.raw, + entry.skindata.colorFeet.hex, + ) } catch (e) { - done() - return + log(`Failed to handle ${server}!`) } - if (skinData === null) { - done() - return - } - - /* Get timestamp */ - let ts = (Date.now()) - - /* Insert skindata */ - for (const entry of skinData) { - skinDB.prepare(`INSERT INTO "skindata" - ( - timestamp, - player, - clan, - flag, - skin, - useColor, - colorBodyRaw, - colorBodyHex, - colorFeetRaw, - ColorFeetHex - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`). - run( - ts, - entry.player, - entry.clan, - entry.flag, - entry.skindata.skin, - entry.skindata.useColor, - entry.skindata.colorBody.raw, - entry.skindata.colorBody.hex, - entry.skindata.colorFeet.raw, - entry.skindata.colorFeet.hex, - ) - } - done() + resolve() }) }) } \ No newline at end of file