Major refactor... ^^

This commit is contained in:
BurnyLlama 2021-10-31 23:43:36 +01:00
parent 62dd9a4db0
commit bdd0b44561
11 changed files with 232 additions and 228 deletions

View File

@ -1,5 +1,5 @@
import { Router } from 'express' import { Router } from 'express'
import { sqlite } from '../libs/db/init.js' import { sqlite } from '../libs/database/init.js'
const finishApi = Router() const finishApi = Router()

View File

@ -1,5 +1,5 @@
import { Router } from 'express' import { Router } from 'express'
import { sqlite } from '../libs/db/init.js' import { sqlite } from '../libs/database/init.js'
const graphApi = Router() const graphApi = Router()

View File

@ -1,5 +1,5 @@
import { Router } from 'express' import { Router } from 'express'
import { sqlite } from '../libs/db/init.js' import { sqlite } from '../libs/database/init.js'
const mapApi = Router() const mapApi = Router()

View File

@ -1,5 +1,5 @@
import { Router } from 'express' import { Router } from 'express'
import { sqlite } from '../libs/db/init.js' import { sqlite } from '../libs/database/init.js'
const playerApi = Router() const playerApi = Router()

View File

@ -1,8 +1,8 @@
import express from 'express' import express from 'express'
import dotenv from 'dotenv' import dotenv from 'dotenv'
import api from './api/api.js' import api from './api/api.js'
import { generateDB } from "./libs/db/generate.js" import { generateDB } from "./libs/database/generate.js"
import { sqlite, dbInit } from "./libs/db/init.js" import { sqlite, dbInit } from "./libs/database/init.js"
import { ddnssStart, scrapeServer } from './libs/ddnss/handler.js' import { ddnssStart, scrapeServer } from './libs/ddnss/handler.js'
//import tasks from './db/tasks.js' //import tasks from './db/tasks.js'

View File

@ -11,9 +11,8 @@ const log = initLog("DB Generation")
*/ */
export function generateDB() { export function generateDB() {
/* TODO: Clean this up as it is a mess */ /* TODO: Clean this up as it is a mess */
/* TODO: Remove useless ones */
log("Generating race index...") log("Generating race index...")
/* Generate race index TODO: Remove useless ones */
execMany([ execMany([
`CREATE INDEX IF NOT EXISTS "idx_race_Map_2" ON "race" ("Map","Name")`, `CREATE INDEX IF NOT EXISTS "idx_race_Map_2" ON "race" ("Map","Name")`,
`CREATE INDEX IF NOT EXISTS "idx_race_Name" ON "race" ("Name","Timestamp")`, `CREATE INDEX IF NOT EXISTS "idx_race_Name" ON "race" ("Name","Timestamp")`,
@ -23,7 +22,6 @@ export function generateDB() {
`CREATE INDEX IF NOT EXISTS "idx_race_MapNameTime" ON "race" ("Map", "Name", "Time")` `CREATE INDEX IF NOT EXISTS "idx_race_MapNameTime" ON "race" ("Map", "Name", "Time")`
]) ])
/* Create rankings table */
log("Creating rankings table...") log("Creating rankings table...")
sqlite.exec(` sqlite.exec(`
CREATE TABLE IF NOT EXISTS "rankings" ( CREATE TABLE IF NOT EXISTS "rankings" (
@ -38,17 +36,19 @@ export function generateDB() {
log("Calculating rankings for each map...") log("Calculating rankings for each map...")
tasks.processRankings() tasks.processRankings()
/* Generate rankings index */
log("Generating rankings index...") log("Generating rankings index...")
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_rankings_map" ON "rankings" ("Map")`) execMany([
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_rankings_rank" ON "rankings" ("rank")`) `CREATE INDEX IF NOT EXISTS "idx_rankings_map" ON "rankings" ("Map")`,
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_rankings_player" ON "rankings" ("Name")`) `CREATE INDEX IF NOT EXISTS "idx_rankings_rank" ON "rankings" ("rank")`,
`CREATE INDEX IF NOT EXISTS "idx_rankings_player" ON "rankings" ("Name")`
])
/* Generate teamrace index */
log("Generating teamrace index...") log("Generating teamrace index...")
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrace_Map" ON "teamrace" ("Map")`); execMany([
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrace_ID" ON "teamrace" ("ID")`); `CREATE INDEX IF NOT EXISTS "idx_teamrace_Map" ON "teamrace" ("Map")`,
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrace_MapID" ON "teamrace" ("Map", "ID")`); `CREATE INDEX IF NOT EXISTS "idx_teamrace_ID" ON "teamrace" ("ID")`,
`CREATE INDEX IF NOT EXISTS "idx_teamrace_MapID" ON "teamrace" ("Map", "ID")`
])
log("Creating teamrankings table...") log("Creating teamrankings table...")
sqlite.exec(` sqlite.exec(`
@ -66,9 +66,11 @@ export function generateDB() {
tasks.processTeamRankings() tasks.processTeamRankings()
log("Generating teamrankings index...") log("Generating teamrankings index...")
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrankings_map" ON "teamrankings" ("Map")`) execMany([
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrankings_rank" ON "teamrankings" ("teamrank")`) `CREATE INDEX IF NOT EXISTS "idx_teamrankings_map" ON "teamrankings" ("Map")`,
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrankings_player" ON "teamrankings" ("name")`) `CREATE INDEX IF NOT EXISTS "idx_teamrankings_rank" ON "teamrankings" ("teamrank")`,
`CREATE INDEX IF NOT EXISTS "idx_teamrankings_player" ON "teamrankings" ("name")`
])
sqlite.exec(` sqlite.exec(`
CREATE TABLE IF NOT EXISTS "points" ( CREATE TABLE IF NOT EXISTS "points" (
@ -77,7 +79,6 @@ export function generateDB() {
"points" INTEGER NOT NULL); "points" INTEGER NOT NULL);
`) `)
/* Process all types of points */
log("Inserting points to DB...") log("Inserting points to DB...")
tasks.processAllPoints() tasks.processAllPoints()

View File

@ -3,7 +3,7 @@ import { sqlite } from './init.js'
/** /**
* This function takes an array of strings to be ran on the DB. * This function takes an array of strings to be ran on the DB.
* *
* @param {array} instructions Array of instructions to be ran. * @param {[string]} instructions Array of instructions to be ran.
* @author BurnyLlama * @author BurnyLlama
*/ */
export function execMany(instructions) { export function execMany(instructions) {

150
libs/database/tasks.js Normal file
View File

@ -0,0 +1,150 @@
import msgpack from '@msgpack/msgpack'
import fs from 'fs'
import { execMany } from './helper.js'
import { sqlite } from './init.js'
/**
* This module parses the msgpack provided by DDNet...
* @module db/decodeMsgpack
*/
export function decodeMsgpack() {
const data = fs.readFileSync('players.msgpack')
const decoded = msgpack.decodeMulti(data, { wrap: true })
const order = ['categories', 'maps', 'totalPoints', 'pointsRanks', 'pointsThisWeek', 'pointsThisMonth', 'teamRankPoints', 'rankPoints', 'serverRanks']
let final = {}
let i = 0
for (const part of decoded) {
final[order[i]] = part
++i
}
return final
}
/**
* This generates rankings for each map...
* @module db/processRankings
*/
export function processRankings() {
const maps = sqlite.prepare(`SELECT map FROM maps`);
for (const map of maps.iterate())
sqlite
.prepare(`
INSERT INTO rankings
(
map, name, time, timestamp, rank, server
)
SELECT map, name, time, timestamp, rank, server
FROM (
SELECT rank() OVER w AS rank,
map,
timestamp,
NAME,
min(time) AS time,
server
FROM race
WHERE map = ?
GROUP BY NAME window w AS (ORDER BY time) ) AS a
ORDER BY rank
`)
.run(map.Map)
}
/**
* This generates teamrankings for each map...
* @module db/processTeamRankings
*/
export function processTeamRankings() {
const maps = sqlite.prepare(`SELECT map FROM maps`);
for (const map of maps.iterate())
sqlite
.prepare(`
INSERT INTO teamrankings
(
name, map, id, time, timestamp, server, teamrank
)
SELECT DISTINCT(r.NAME),
r.map, r.id, r.time, r.timestamp,
Substring(n.server, 1, 3),
dense_rank() OVER w AS rank
FROM ((
SELECT DISTINCT id
FROM teamrace
WHERE map = ?
ORDER BY time) AS l
)
LEFT JOIN teamrace AS r
ON l.id = r.id
INNER JOIN race AS n
ON r.map = n.map
AND r.NAME = n.NAME
AND r.time = n.time window w AS (ORDER BY r.time)
`)
.run(map.Map)
}
/**
* This inserts all types of points into a table...
* @module db/processAllPoints
*/
export function processAllPoints() {
const msgpack = decodeMsgpack()
const types = {
points: msgpack.pointsRanks,
pointsThisWeek: msgpack.pointsThisWeek,
pointsThisMonth: msgpack.pointsThisMonth,
pointsTeam: msgpack.teamRankPoints,
pointsRank: msgpack.rankPoints,
}
/* Generate tables */
sqlite.exec(`
CREATE TABLE IF NOT EXISTS "points"
(
"type" varchar(16) NOT NULL,
"rank" INTEGER NOT NULL,
"name" varchar(16) NOT NULL,
"points" INTEGER NOT NULL
);
`)
/* Insert data */
for(const type in types) {
let rank = 1
for (const entry of types[type]) {
sqlite
.prepare(`
INSERT INTO "points"
(
type, rank, name, points
) VALUES (?, ?, ?, ?)
`)
.run(
type,
rank,
entry[0],
entry[1]
)
++rank
}
}
/* Generate indexes */
execMany([
`CREATE INDEX IF NOT EXISTS "idx_points_type" ON "points" ("type")`,
`CREATE INDEX IF NOT EXISTS "Idx_points_rank" on "points" ("rank")`,
`CREATE INDEX IF NOT EXISTS "Idx_points_name" on "points" ("name")`
])
}
export default {
processAllPoints,
processRankings,
processTeamRankings
}

View File

@ -1,141 +0,0 @@
import msgpack from '@msgpack/msgpack'
import fs from 'fs'
import { sqlite } from "./init.js"
/**
* This module parses the msgpack provided by DDNet...
* @module db/decodeMsgpack
*/
export function decodeMsgpack() {
const data = fs.readFileSync('players.msgpack')
const decoded = msgpack.decodeMulti(data, {wrap: true})
const order = ['categories', 'maps', 'totalPoints', 'pointsRanks', 'pointsThisWeek', 'pointsThisMonth', 'teamRankPoints', 'rankPoints', 'serverRanks']
let final = {}
let i = 0
for (const part of decoded) {
final[order[i]] = part
++i
}
return final
}
/**
* This generates rankings for each map...
* @module db/processRankings
*/
export function processRankings() {
const maps = sqlite.prepare(`SELECT map FROM maps`);
for (const map of maps.iterate()) {
sqlite.prepare(
`
INSERT INTO rankings
(
map, name, time, timestamp, rank, server
)
SELECT map, name, time, timestamp, rank, server
FROM (
SELECT rank() OVER w AS rank,
map,
timestamp,
NAME,
min(time) AS time,
server
FROM race
WHERE map = ?
GROUP BY NAME window w AS (ORDER BY time) ) AS a
ORDER BY rank
`).run(map.Map)
}
}
/**
* This generates teamrankings for each map...
* @module db/processTeamRankings
*/
export function processTeamRankings() {
const maps = sqlite.prepare(`SELECT map FROM maps`);
for (const map of maps.iterate()) {
sqlite.prepare(
`
INSERT INTO teamrankings
(
name, map, id, time, timestamp, server, teamrank
)
SELECT DISTINCT(r.NAME),
r.map, r.id, r.time, r.timestamp,
Substring(n.server, 1, 3),
dense_rank() OVER w AS rank
FROM ((
SELECT DISTINCT id
FROM teamrace
WHERE map = ?
ORDER BY time) AS l
)
LEFT JOIN teamrace AS r
ON l.id = r.id
INNER JOIN race AS n
ON r.map = n.map
AND r.NAME = n.NAME
AND r.time = n.time window w AS (ORDER BY r.time)
`).run(map.Map)
}
}
/**
* This inserts all types of points into a table...
* @module db/processAllPoints
*/
export function processAllPoints() {
const msgpack = decodeMsgpack()
let types = {
points: msgpack.pointsRanks,
pointsThisWeek: msgpack.pointsThisWeek,
pointsThisMonth: msgpack.pointsThisMonth,
pointsTeam: msgpack.teamRankPoints,
pointsRank: msgpack.rankPoints,
}
/* Generate tables */
sqlite.exec(
`
CREATE TABLE IF NOT EXISTS "points"
(
"type" varchar(16) NOT NULL,
"rank" INTEGER NOT NULL,
"name" varchar(16) NOT NULL,
"points" INTEGER NOT NULL);
`)
/* Insert data */
for(const type in types) {
let rank = 1
for (const entry of types[type]) {
sqlite.prepare(
`
INSERT INTO "points"
(
type, rank, name, points
) VALUES (?, ?, ?, ?)`).run(
type, rank, entry[0], entry[1])
++rank
}
}
/* Generate indexes */
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_points_type" ON "points" ("type")`)
sqlite.exec(`CREATE INDEX IF NOT EXISTS "Idx_points_rank" on "points" ("rank")`)
sqlite.exec(`CREATE INDEX IF NOT EXISTS "Idx_points_name" on "points" ("name")`)
}
export default {
processAllPoints,
processRankings,
processTeamRankings
}

View File

@ -1,85 +1,79 @@
import fetch from 'node-fetch' import fetch from 'node-fetch'
import { exec } from 'child_process' import { exec } from 'child_process'
import { skinDB } from "../db/init.js" import { skinDB } from '../database/init.js'
import initLog from '../utils/log.js'
const log = initLog("DDNSS")
export async function ddnssStart() { export async function ddnssStart() {
const response = await fetch('https://ddnet.tw/status/index.json'); const getServers = await fetch('https://ddnet.tw/status/index.json');
const data = await response.json(); const servers = await getServers.json();
console.log(data) console.log(servers)
for (const servers of data) { for (const server of servers) {
/* Check if server isn't empty and not full */ const connection = `${server.ip}:${server.port}`
if (servers.num_clients > 0 && servers.num_clients < 59) {
let server = `${servers.ip}:${servers.port}`
console.log(`Connecting: ${servers.ip}:${servers.port}`)
/* exec ddnss */ if (!(server.num_clients > 0 && server.num_clients < (server.max_clients - 2)))
await scrapeServer(`${server}`) return log(`Server (essentially) full! >> ${server.ip}:${server.port} -> ${server.num_clients}/${server.max_clients} clients`)
}
else log(`Connecting to server >> ${connection}`)
console.log(`${servers.num_clients}/63 Server full: ${servers.ip}:${servers.port}`) await scrapeServer(`${connection}`)
} }
/* A bit hacky way of killing ddnss */
exec(`pkill -9 -f ddnss`) exec(`pkill -9 -f ddnss`)
} }
export function scrapeServer(server) { function scrapeServer(server) {
let command = `./ddnss/build/DDNet "ui_server_address ${server}" -f ddnss/build/config.conf`
let skinData
return new Promise((done, failed) => { const command = `./ddnss/build/DDNet "ui_server_address ${server}" -f ddnss/build/config.conf`
exec(command, { encoding: 'utf8', timeout: 10000 }, (err, stdout, stderr) => {
if (err) {
err.stdout = stdout
err.stderr = stderr
}
/* Handle error from parsing of JSON */ return new Promise((resolve, reject) => {
exec(command, { encoding: 'utf8' }, (err, stdout, stderr) => {
try { try {
skinData = JSON.parse(stdout) const skinData = JSON.parse(stdout)
if (skinData === null) {
resolve()
return
}
const currentTime = Date.now()
for (const entry of skinData)
skinDB
.prepare(`
INSERT INTO "skindata"
(
timestamp,
player,
clan,
flag,
skin,
useColor,
colorBodyRaw,
colorBodyHex,
colorFeetRaw,
ColorFeetHex
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`)
.run(
currentTime,
entry.player,
entry.clan,
entry.flag,
entry.skindata.skin,
entry.skindata.useColor,
entry.skindata.colorBody.raw,
entry.skindata.colorBody.hex,
entry.skindata.colorFeet.raw,
entry.skindata.colorFeet.hex,
)
} catch (e) { } catch (e) {
done() log(`Failed to handle ${server}!`)
return
} }
if (skinData === null) { resolve()
done()
return
}
/* Get timestamp */
let ts = (Date.now())
/* Insert skindata */
for (const entry of skinData) {
skinDB.prepare(`INSERT INTO "skindata"
(
timestamp,
player,
clan,
flag,
skin,
useColor,
colorBodyRaw,
colorBodyHex,
colorFeetRaw,
ColorFeetHex
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).
run(
ts,
entry.player,
entry.clan,
entry.flag,
entry.skindata.skin,
entry.skindata.useColor,
entry.skindata.colorBody.raw,
entry.skindata.colorBody.hex,
entry.skindata.colorFeet.raw,
entry.skindata.colorFeet.hex,
)
}
done()
}) })
}) })
} }