Major refactor... ^^
This commit is contained in:
parent
62dd9a4db0
commit
bdd0b44561
|
|
@ -1,5 +1,5 @@
|
|||
import { Router } from 'express'
|
||||
import { sqlite } from '../libs/db/init.js'
|
||||
import { sqlite } from '../libs/database/init.js'
|
||||
|
||||
const finishApi = Router()
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { Router } from 'express'
|
||||
import { sqlite } from '../libs/db/init.js'
|
||||
import { sqlite } from '../libs/database/init.js'
|
||||
|
||||
const graphApi = Router()
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { Router } from 'express'
|
||||
import { sqlite } from '../libs/db/init.js'
|
||||
import { sqlite } from '../libs/database/init.js'
|
||||
|
||||
const mapApi = Router()
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { Router } from 'express'
|
||||
import { sqlite } from '../libs/db/init.js'
|
||||
import { sqlite } from '../libs/database/init.js'
|
||||
|
||||
const playerApi = Router()
|
||||
|
||||
|
|
|
|||
4
index.js
4
index.js
|
|
@ -1,8 +1,8 @@
|
|||
import express from 'express'
|
||||
import dotenv from 'dotenv'
|
||||
import api from './api/api.js'
|
||||
import { generateDB } from "./libs/db/generate.js"
|
||||
import { sqlite, dbInit } from "./libs/db/init.js"
|
||||
import { generateDB } from "./libs/database/generate.js"
|
||||
import { sqlite, dbInit } from "./libs/database/init.js"
|
||||
import { ddnssStart, scrapeServer } from './libs/ddnss/handler.js'
|
||||
//import tasks from './db/tasks.js'
|
||||
|
||||
|
|
|
|||
|
|
@ -11,9 +11,8 @@ const log = initLog("DB Generation")
|
|||
*/
|
||||
export function generateDB() {
|
||||
/* TODO: Clean this up as it is a mess */
|
||||
/* TODO: Remove useless ones */
|
||||
log("Generating race index...")
|
||||
|
||||
/* Generate race index TODO: Remove useless ones */
|
||||
execMany([
|
||||
`CREATE INDEX IF NOT EXISTS "idx_race_Map_2" ON "race" ("Map","Name")`,
|
||||
`CREATE INDEX IF NOT EXISTS "idx_race_Name" ON "race" ("Name","Timestamp")`,
|
||||
|
|
@ -23,7 +22,6 @@ export function generateDB() {
|
|||
`CREATE INDEX IF NOT EXISTS "idx_race_MapNameTime" ON "race" ("Map", "Name", "Time")`
|
||||
])
|
||||
|
||||
/* Create rankings table */
|
||||
log("Creating rankings table...")
|
||||
sqlite.exec(`
|
||||
CREATE TABLE IF NOT EXISTS "rankings" (
|
||||
|
|
@ -38,17 +36,19 @@ export function generateDB() {
|
|||
log("Calculating rankings for each map...")
|
||||
tasks.processRankings()
|
||||
|
||||
/* Generate rankings index */
|
||||
log("Generating rankings index...")
|
||||
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_rankings_map" ON "rankings" ("Map")`)
|
||||
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_rankings_rank" ON "rankings" ("rank")`)
|
||||
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_rankings_player" ON "rankings" ("Name")`)
|
||||
execMany([
|
||||
`CREATE INDEX IF NOT EXISTS "idx_rankings_map" ON "rankings" ("Map")`,
|
||||
`CREATE INDEX IF NOT EXISTS "idx_rankings_rank" ON "rankings" ("rank")`,
|
||||
`CREATE INDEX IF NOT EXISTS "idx_rankings_player" ON "rankings" ("Name")`
|
||||
])
|
||||
|
||||
/* Generate teamrace index */
|
||||
log("Generating teamrace index...")
|
||||
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrace_Map" ON "teamrace" ("Map")`);
|
||||
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrace_ID" ON "teamrace" ("ID")`);
|
||||
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrace_MapID" ON "teamrace" ("Map", "ID")`);
|
||||
execMany([
|
||||
`CREATE INDEX IF NOT EXISTS "idx_teamrace_Map" ON "teamrace" ("Map")`,
|
||||
`CREATE INDEX IF NOT EXISTS "idx_teamrace_ID" ON "teamrace" ("ID")`,
|
||||
`CREATE INDEX IF NOT EXISTS "idx_teamrace_MapID" ON "teamrace" ("Map", "ID")`
|
||||
])
|
||||
|
||||
log("Creating teamrankings table...")
|
||||
sqlite.exec(`
|
||||
|
|
@ -66,9 +66,11 @@ export function generateDB() {
|
|||
tasks.processTeamRankings()
|
||||
|
||||
log("Generating teamrankings index...")
|
||||
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrankings_map" ON "teamrankings" ("Map")`)
|
||||
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrankings_rank" ON "teamrankings" ("teamrank")`)
|
||||
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrankings_player" ON "teamrankings" ("name")`)
|
||||
execMany([
|
||||
`CREATE INDEX IF NOT EXISTS "idx_teamrankings_map" ON "teamrankings" ("Map")`,
|
||||
`CREATE INDEX IF NOT EXISTS "idx_teamrankings_rank" ON "teamrankings" ("teamrank")`,
|
||||
`CREATE INDEX IF NOT EXISTS "idx_teamrankings_player" ON "teamrankings" ("name")`
|
||||
])
|
||||
|
||||
sqlite.exec(`
|
||||
CREATE TABLE IF NOT EXISTS "points" (
|
||||
|
|
@ -77,7 +79,6 @@ export function generateDB() {
|
|||
"points" INTEGER NOT NULL);
|
||||
`)
|
||||
|
||||
/* Process all types of points */
|
||||
log("Inserting points to DB...")
|
||||
tasks.processAllPoints()
|
||||
|
||||
|
|
@ -3,7 +3,7 @@ import { sqlite } from './init.js'
|
|||
/**
|
||||
* This function takes an array of strings to be ran on the DB.
|
||||
*
|
||||
* @param {array} instructions Array of instructions to be ran.
|
||||
* @param {[string]} instructions Array of instructions to be ran.
|
||||
* @author BurnyLlama
|
||||
*/
|
||||
export function execMany(instructions) {
|
||||
150
libs/database/tasks.js
Normal file
150
libs/database/tasks.js
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
import msgpack from '@msgpack/msgpack'
|
||||
import fs from 'fs'
|
||||
import { execMany } from './helper.js'
|
||||
|
||||
import { sqlite } from './init.js'
|
||||
|
||||
/**
|
||||
* This module parses the msgpack provided by DDNet...
|
||||
* @module db/decodeMsgpack
|
||||
*/
|
||||
export function decodeMsgpack() {
|
||||
const data = fs.readFileSync('players.msgpack')
|
||||
const decoded = msgpack.decodeMulti(data, { wrap: true })
|
||||
const order = ['categories', 'maps', 'totalPoints', 'pointsRanks', 'pointsThisWeek', 'pointsThisMonth', 'teamRankPoints', 'rankPoints', 'serverRanks']
|
||||
let final = {}
|
||||
|
||||
let i = 0
|
||||
for (const part of decoded) {
|
||||
final[order[i]] = part
|
||||
++i
|
||||
}
|
||||
return final
|
||||
}
|
||||
|
||||
/**
|
||||
* This generates rankings for each map...
|
||||
* @module db/processRankings
|
||||
*/
|
||||
export function processRankings() {
|
||||
const maps = sqlite.prepare(`SELECT map FROM maps`);
|
||||
|
||||
for (const map of maps.iterate())
|
||||
sqlite
|
||||
.prepare(`
|
||||
INSERT INTO rankings
|
||||
(
|
||||
map, name, time, timestamp, rank, server
|
||||
)
|
||||
SELECT map, name, time, timestamp, rank, server
|
||||
FROM (
|
||||
SELECT rank() OVER w AS rank,
|
||||
map,
|
||||
timestamp,
|
||||
NAME,
|
||||
min(time) AS time,
|
||||
server
|
||||
FROM race
|
||||
WHERE map = ?
|
||||
GROUP BY NAME window w AS (ORDER BY time) ) AS a
|
||||
ORDER BY rank
|
||||
`)
|
||||
.run(map.Map)
|
||||
}
|
||||
|
||||
/**
|
||||
* This generates teamrankings for each map...
|
||||
* @module db/processTeamRankings
|
||||
*/
|
||||
export function processTeamRankings() {
|
||||
const maps = sqlite.prepare(`SELECT map FROM maps`);
|
||||
|
||||
for (const map of maps.iterate())
|
||||
sqlite
|
||||
.prepare(`
|
||||
INSERT INTO teamrankings
|
||||
(
|
||||
name, map, id, time, timestamp, server, teamrank
|
||||
)
|
||||
SELECT DISTINCT(r.NAME),
|
||||
r.map, r.id, r.time, r.timestamp,
|
||||
Substring(n.server, 1, 3),
|
||||
dense_rank() OVER w AS rank
|
||||
FROM ((
|
||||
SELECT DISTINCT id
|
||||
FROM teamrace
|
||||
WHERE map = ?
|
||||
ORDER BY time) AS l
|
||||
)
|
||||
LEFT JOIN teamrace AS r
|
||||
ON l.id = r.id
|
||||
INNER JOIN race AS n
|
||||
ON r.map = n.map
|
||||
AND r.NAME = n.NAME
|
||||
AND r.time = n.time window w AS (ORDER BY r.time)
|
||||
`)
|
||||
.run(map.Map)
|
||||
}
|
||||
|
||||
/**
|
||||
* This inserts all types of points into a table...
|
||||
* @module db/processAllPoints
|
||||
*/
|
||||
export function processAllPoints() {
|
||||
const msgpack = decodeMsgpack()
|
||||
|
||||
const types = {
|
||||
points: msgpack.pointsRanks,
|
||||
pointsThisWeek: msgpack.pointsThisWeek,
|
||||
pointsThisMonth: msgpack.pointsThisMonth,
|
||||
pointsTeam: msgpack.teamRankPoints,
|
||||
pointsRank: msgpack.rankPoints,
|
||||
}
|
||||
|
||||
/* Generate tables */
|
||||
sqlite.exec(`
|
||||
CREATE TABLE IF NOT EXISTS "points"
|
||||
(
|
||||
"type" varchar(16) NOT NULL,
|
||||
"rank" INTEGER NOT NULL,
|
||||
"name" varchar(16) NOT NULL,
|
||||
"points" INTEGER NOT NULL
|
||||
);
|
||||
`)
|
||||
|
||||
/* Insert data */
|
||||
for(const type in types) {
|
||||
let rank = 1
|
||||
|
||||
for (const entry of types[type]) {
|
||||
sqlite
|
||||
.prepare(`
|
||||
INSERT INTO "points"
|
||||
(
|
||||
type, rank, name, points
|
||||
) VALUES (?, ?, ?, ?)
|
||||
`)
|
||||
.run(
|
||||
type,
|
||||
rank,
|
||||
entry[0],
|
||||
entry[1]
|
||||
)
|
||||
|
||||
++rank
|
||||
}
|
||||
}
|
||||
|
||||
/* Generate indexes */
|
||||
execMany([
|
||||
`CREATE INDEX IF NOT EXISTS "idx_points_type" ON "points" ("type")`,
|
||||
`CREATE INDEX IF NOT EXISTS "Idx_points_rank" on "points" ("rank")`,
|
||||
`CREATE INDEX IF NOT EXISTS "Idx_points_name" on "points" ("name")`
|
||||
])
|
||||
}
|
||||
|
||||
export default {
|
||||
processAllPoints,
|
||||
processRankings,
|
||||
processTeamRankings
|
||||
}
|
||||
141
libs/db/tasks.js
141
libs/db/tasks.js
|
|
@ -1,141 +0,0 @@
|
|||
import msgpack from '@msgpack/msgpack'
|
||||
import fs from 'fs'
|
||||
|
||||
import { sqlite } from "./init.js"
|
||||
|
||||
/**
|
||||
* This module parses the msgpack provided by DDNet...
|
||||
* @module db/decodeMsgpack
|
||||
*/
|
||||
export function decodeMsgpack() {
|
||||
const data = fs.readFileSync('players.msgpack')
|
||||
const decoded = msgpack.decodeMulti(data, {wrap: true})
|
||||
const order = ['categories', 'maps', 'totalPoints', 'pointsRanks', 'pointsThisWeek', 'pointsThisMonth', 'teamRankPoints', 'rankPoints', 'serverRanks']
|
||||
let final = {}
|
||||
|
||||
let i = 0
|
||||
for (const part of decoded) {
|
||||
final[order[i]] = part
|
||||
++i
|
||||
}
|
||||
return final
|
||||
}
|
||||
|
||||
/**
|
||||
* This generates rankings for each map...
|
||||
* @module db/processRankings
|
||||
*/
|
||||
export function processRankings() {
|
||||
const maps = sqlite.prepare(`SELECT map FROM maps`);
|
||||
|
||||
for (const map of maps.iterate()) {
|
||||
sqlite.prepare(
|
||||
`
|
||||
INSERT INTO rankings
|
||||
(
|
||||
map, name, time, timestamp, rank, server
|
||||
)
|
||||
SELECT map, name, time, timestamp, rank, server
|
||||
FROM (
|
||||
SELECT rank() OVER w AS rank,
|
||||
map,
|
||||
timestamp,
|
||||
NAME,
|
||||
min(time) AS time,
|
||||
server
|
||||
FROM race
|
||||
WHERE map = ?
|
||||
GROUP BY NAME window w AS (ORDER BY time) ) AS a
|
||||
ORDER BY rank
|
||||
`).run(map.Map)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This generates teamrankings for each map...
|
||||
* @module db/processTeamRankings
|
||||
*/
|
||||
export function processTeamRankings() {
|
||||
const maps = sqlite.prepare(`SELECT map FROM maps`);
|
||||
|
||||
for (const map of maps.iterate()) {
|
||||
sqlite.prepare(
|
||||
`
|
||||
INSERT INTO teamrankings
|
||||
(
|
||||
name, map, id, time, timestamp, server, teamrank
|
||||
)
|
||||
SELECT DISTINCT(r.NAME),
|
||||
r.map, r.id, r.time, r.timestamp,
|
||||
Substring(n.server, 1, 3),
|
||||
dense_rank() OVER w AS rank
|
||||
FROM ((
|
||||
SELECT DISTINCT id
|
||||
FROM teamrace
|
||||
WHERE map = ?
|
||||
ORDER BY time) AS l
|
||||
)
|
||||
LEFT JOIN teamrace AS r
|
||||
ON l.id = r.id
|
||||
INNER JOIN race AS n
|
||||
ON r.map = n.map
|
||||
AND r.NAME = n.NAME
|
||||
AND r.time = n.time window w AS (ORDER BY r.time)
|
||||
`).run(map.Map)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This inserts all types of points into a table...
|
||||
* @module db/processAllPoints
|
||||
*/
|
||||
export function processAllPoints() {
|
||||
const msgpack = decodeMsgpack()
|
||||
|
||||
let types = {
|
||||
points: msgpack.pointsRanks,
|
||||
pointsThisWeek: msgpack.pointsThisWeek,
|
||||
pointsThisMonth: msgpack.pointsThisMonth,
|
||||
pointsTeam: msgpack.teamRankPoints,
|
||||
pointsRank: msgpack.rankPoints,
|
||||
}
|
||||
|
||||
/* Generate tables */
|
||||
sqlite.exec(
|
||||
`
|
||||
CREATE TABLE IF NOT EXISTS "points"
|
||||
(
|
||||
"type" varchar(16) NOT NULL,
|
||||
"rank" INTEGER NOT NULL,
|
||||
"name" varchar(16) NOT NULL,
|
||||
"points" INTEGER NOT NULL);
|
||||
`)
|
||||
|
||||
/* Insert data */
|
||||
for(const type in types) {
|
||||
let rank = 1
|
||||
|
||||
for (const entry of types[type]) {
|
||||
sqlite.prepare(
|
||||
`
|
||||
INSERT INTO "points"
|
||||
(
|
||||
type, rank, name, points
|
||||
) VALUES (?, ?, ?, ?)`).run(
|
||||
type, rank, entry[0], entry[1])
|
||||
|
||||
++rank
|
||||
}
|
||||
}
|
||||
|
||||
/* Generate indexes */
|
||||
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_points_type" ON "points" ("type")`)
|
||||
sqlite.exec(`CREATE INDEX IF NOT EXISTS "Idx_points_rank" on "points" ("rank")`)
|
||||
sqlite.exec(`CREATE INDEX IF NOT EXISTS "Idx_points_name" on "points" ("name")`)
|
||||
}
|
||||
|
||||
export default {
|
||||
processAllPoints,
|
||||
processRankings,
|
||||
processTeamRankings
|
||||
}
|
||||
|
|
@ -1,85 +1,79 @@
|
|||
import fetch from 'node-fetch'
|
||||
import { exec } from 'child_process'
|
||||
import { skinDB } from "../db/init.js"
|
||||
import { skinDB } from '../database/init.js'
|
||||
import initLog from '../utils/log.js'
|
||||
|
||||
const log = initLog("DDNSS")
|
||||
|
||||
export async function ddnssStart() {
|
||||
const response = await fetch('https://ddnet.tw/status/index.json');
|
||||
const data = await response.json();
|
||||
const getServers = await fetch('https://ddnet.tw/status/index.json');
|
||||
const servers = await getServers.json();
|
||||
|
||||
console.log(data)
|
||||
console.log(servers)
|
||||
|
||||
for (const servers of data) {
|
||||
/* Check if server isn't empty and not full */
|
||||
if (servers.num_clients > 0 && servers.num_clients < 59) {
|
||||
let server = `${servers.ip}:${servers.port}`
|
||||
console.log(`Connecting: ${servers.ip}:${servers.port}`)
|
||||
for (const server of servers) {
|
||||
const connection = `${server.ip}:${server.port}`
|
||||
|
||||
/* exec ddnss */
|
||||
await scrapeServer(`${server}`)
|
||||
}
|
||||
else
|
||||
console.log(`${servers.num_clients}/63 Server full: ${servers.ip}:${servers.port}`)
|
||||
if (!(server.num_clients > 0 && server.num_clients < (server.max_clients - 2)))
|
||||
return log(`Server (essentially) full! >> ${server.ip}:${server.port} -> ${server.num_clients}/${server.max_clients} clients`)
|
||||
|
||||
log(`Connecting to server >> ${connection}`)
|
||||
await scrapeServer(`${connection}`)
|
||||
}
|
||||
/* A bit hacky way of killing ddnss */
|
||||
|
||||
exec(`pkill -9 -f ddnss`)
|
||||
}
|
||||
|
||||
export function scrapeServer(server) {
|
||||
let command = `./ddnss/build/DDNet "ui_server_address ${server}" -f ddnss/build/config.conf`
|
||||
let skinData
|
||||
function scrapeServer(server) {
|
||||
|
||||
return new Promise((done, failed) => {
|
||||
exec(command, { encoding: 'utf8', timeout: 10000 }, (err, stdout, stderr) => {
|
||||
if (err) {
|
||||
err.stdout = stdout
|
||||
err.stderr = stderr
|
||||
}
|
||||
const command = `./ddnss/build/DDNet "ui_server_address ${server}" -f ddnss/build/config.conf`
|
||||
|
||||
/* Handle error from parsing of JSON */
|
||||
return new Promise((resolve, reject) => {
|
||||
exec(command, { encoding: 'utf8' }, (err, stdout, stderr) => {
|
||||
try {
|
||||
skinData = JSON.parse(stdout)
|
||||
const skinData = JSON.parse(stdout)
|
||||
|
||||
if (skinData === null) {
|
||||
resolve()
|
||||
return
|
||||
}
|
||||
|
||||
const currentTime = Date.now()
|
||||
|
||||
for (const entry of skinData)
|
||||
skinDB
|
||||
.prepare(`
|
||||
INSERT INTO "skindata"
|
||||
(
|
||||
timestamp,
|
||||
player,
|
||||
clan,
|
||||
flag,
|
||||
skin,
|
||||
useColor,
|
||||
colorBodyRaw,
|
||||
colorBodyHex,
|
||||
colorFeetRaw,
|
||||
ColorFeetHex
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`)
|
||||
.run(
|
||||
currentTime,
|
||||
entry.player,
|
||||
entry.clan,
|
||||
entry.flag,
|
||||
entry.skindata.skin,
|
||||
entry.skindata.useColor,
|
||||
entry.skindata.colorBody.raw,
|
||||
entry.skindata.colorBody.hex,
|
||||
entry.skindata.colorFeet.raw,
|
||||
entry.skindata.colorFeet.hex,
|
||||
)
|
||||
} catch (e) {
|
||||
done()
|
||||
return
|
||||
log(`Failed to handle ${server}!`)
|
||||
}
|
||||
|
||||
if (skinData === null) {
|
||||
done()
|
||||
return
|
||||
}
|
||||
|
||||
/* Get timestamp */
|
||||
let ts = (Date.now())
|
||||
|
||||
/* Insert skindata */
|
||||
for (const entry of skinData) {
|
||||
skinDB.prepare(`INSERT INTO "skindata"
|
||||
(
|
||||
timestamp,
|
||||
player,
|
||||
clan,
|
||||
flag,
|
||||
skin,
|
||||
useColor,
|
||||
colorBodyRaw,
|
||||
colorBodyHex,
|
||||
colorFeetRaw,
|
||||
ColorFeetHex
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).
|
||||
run(
|
||||
ts,
|
||||
entry.player,
|
||||
entry.clan,
|
||||
entry.flag,
|
||||
entry.skindata.skin,
|
||||
entry.skindata.useColor,
|
||||
entry.skindata.colorBody.raw,
|
||||
entry.skindata.colorBody.hex,
|
||||
entry.skindata.colorFeet.raw,
|
||||
entry.skindata.colorFeet.hex,
|
||||
)
|
||||
}
|
||||
done()
|
||||
resolve()
|
||||
})
|
||||
})
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user