Initial commit

This commit is contained in:
furo 2021-10-30 20:26:37 +02:00
commit 08b0f6d36c
14 changed files with 896 additions and 0 deletions

125
.gitignore vendored Normal file
View File

@ -0,0 +1,125 @@
# ---> Node
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
# Diagnostic reports (https://nodejs.org/api/report.html)
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
*.lcov
# nyc test coverage
.nyc_output
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# Snowpack dependency directory (https://snowpack.dev/)
web_modules/
# TypeScript cache
*.tsbuildinfo
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Microbundle cache
.rpt2_cache/
.rts2_cache_cjs/
.rts2_cache_es/
.rts2_cache_umd/
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variables file
.env
.env.test
# parcel-bundler cache (https://parceljs.org/)
.cache
.parcel-cache
# Next.js build output
.next
out
# Nuxt.js build / generate output
.nuxt
dist
# Gatsby files
.cache/
# Comment in the public line in if your project uses Gatsby and not Next.js
# https://nextjs.org/blog/next-9-1#public-directory-support
# public
# vuepress build output
.vuepress/dist
# Serverless directories
.serverless/
# FuseBox cache
.fusebox/
# DynamoDB Local files
.dynamodb/
# TernJS port file
.tern-port
# Stores VSCode versions used for testing VSCode extensions
.vscode-test
# yarn v2
.yarn/cache
.yarn/unplugged
.yarn/build-state.yml
.yarn/install-state.gz
.pnp.*
package-lock.json
pnpm-lock.yaml
*.sql*
players.msgpack
.env
ddnss/build

26
api/api.js Normal file
View File

@ -0,0 +1,26 @@
import { Router } from 'express'
import playerApi from './players.js'
import mapApi from './maps.js'
import finishesApi from './finishes.js'
import graphApi from './graph.js'
const api = Router()
api.get(
'/',
(req, res) => res.json({
success: true,
response: "You connected to DDStats API! :D"
})
)
api.use('/players', playerApi)
api.use('/maps', mapApi)
api.use('/finishes', finishesApi)
api.use('/graph', graphApi)
/**
* This module is the entrypoint for the API.
* @module api/api
*/
export default api

19
api/finishes.js Normal file
View File

@ -0,0 +1,19 @@
import { Router } from 'express'
import { sqlite } from '../db/init.js'
const finishApi = Router()
/* TODO: precalculate this */
finishApi.get(
'/count',
(req, res) => {
const finishes = sqlite.prepare(`SELECT COUNT(*) as count FROM race`).get()
return res.json({
success: true,
response: finishes.count,
})
}
)
export default finishApi

46
api/graph.js Normal file
View File

@ -0,0 +1,46 @@
import { Router } from 'express'
import { sqlite } from '../db/init.js'
const graphApi = Router()
/* TODO: precalculate this */
graphApi.get(
'/points',
(req, res) => {
/* Check if a query was provided */
if (!req.query.q) {
return res.json({
success: false,
response: "No query ('host/path?q=query') provided!"
})
}
let player = req.query.q
const finishes = sqlite.prepare(
`
SELECT DISTINCT(a.map), a.timestamp, b.points
FROM race AS a
INNER JOIN maps AS b
ON a.map = b.map
WHERE a.NAME = ?
AND a.map LIKE '%'
GROUP BY a.map
ORDER BY a.timestamp;
`)
let currentPoints = 0
let array = []
for (const finish of finishes.iterate(player)) {
console.log(finish)
currentPoints += finish.Points
array.push({ t: new Date(finish.Timestamp), y: currentPoints })
}
return res.json({
success: true,
response: array,
})
}
)
export default graphApi

175
api/maps.js Normal file
View File

@ -0,0 +1,175 @@
import { Router } from 'express'
import { sqlite } from '../db/init.js'
const mapApi = Router()
mapApi.get(
'/count',
(req, res) => {
const totalMaps = sqlite.prepare(`SELECT COUNT(*) as amount FROM maps`).get()
return res.json({
success: true,
response: totalMaps.amount,
})
}
)
mapApi.get(
'/get/:map',
(req, res) => {
let map = req.params.map
/* Check if map exists */
const check = sqlite.prepare(`SELECT map FROM maps WHERE map = ?`).get(map)
if (!check) {
return res.json({
success: false,
response: "No map found!",
})
}
const info = sqlite.prepare(`SELECT * FROM maps WHERE map = ?`).get(map)
/* TODO: Generate a table with this as a cache */
/* Also generate indexes */
const avgTime = sqlite.prepare(`SELECT avg(Time) as 'averageTime' FROM race WHERE map = ?`).get(map)
const total = sqlite.prepare(`SELECT COUNT(*) as 'total' FROM race WHERE map = ?`).get(map)
const unique = sqlite.prepare(`SELECT COUNT(distinct(name)) as 'unique' FROM race WHERE map = ?`).get(map)
const teams = sqlite.prepare(`SELECT COUNT(distinct(ID)) as 'teams' FROM teamrace WHERE map = ?`).get(map)
return res.json({
success: true,
response: {
name: info.Map,
category: info.Server,
awardPoints: info.Points,
rating: info.Stars,
mapper: info.Mapper,
release: info.Timestamp,
/* TODO Get median time*/
averageTime: avgTime.averageTime,
finishes: {
unique: unique.unique,
total: total.total,
teams: teams.teams,
}
}
})
}
)
mapApi.get(
'/getAll',
(req, res) => {
const allMaps = sqlite.prepare(
`SELECT
map as name,
server as category,
points as awardPoints,
stars as rating,
mapper as mapper,
timestamp as release
FROM maps`).all()
return res.json({
success: true,
response: allMaps,
})
}
)
mapApi.get(
'/category/:category',
(req, res) => {
let category = req.params.category
/* Check if category exists */
const check = sqlite.prepare(`SELECT server FROM maps WHERE server = ? LIMIT 1`).get(category)
if (!check) {
return res.json({
success: false,
response: "Invalid category name!",
})
}
const allMaps = sqlite.prepare(
`SELECT
map as name,
server as category,
points as awardPoints,
stars as rating,
mapper as mapper,
timestamp as release
FROM maps WHERE server = ?`).all(category)
return res.json({
success: true,
response: allMaps,
})
}
)
/* Searching allows you to attach sql LIKE %
* Example to search for maps beginning with Kobra
* You can do search for Kobra%
* This thing is a complete mess, send help. BUT it does work <3 */
mapApi.get(
'/search',
async (req, res) => {
/* Check if a query was provided */
if (!req.query.q || !req.query.byMapper) {
return res.json({
success: false,
response: "No query ('host/path?q=query') provided!"
})
}
let query = req.query.q
/* Set defaults */
let limit = 20
let offset = 0
let sortBy = "timestamp"
let order = "asc"
let column = "map"
let startsWith = ""
if (req.query.limit)
limit = req.query.limit
if (req.query.sortBy)
sortBy = req.query.sortBy
if (req.query.order)
order = req.query.order
if (req.query.byMapper)
column = "mapper"
/* TODO: do this in one query? */
const amount = sqlite.prepare(
`SELECT COUNT(*) as amount FROM maps WHERE ${column} LIKE "${query}"`).get()
let pages = Math.floor(amount.amount / limit)
if (req.query.page)
offset = (req.query.page * limit)
const maps = sqlite.prepare(
`SELECT * FROM maps WHERE ${column} LIKE "${query}" ORDER BY ${sortBy} ${order} LIMIT ? OFFSET ?`).all(limit, offset)
return res.json({
success: true,
response: {
amount: amount.amount,
pages,
maps,
}
})
}
)
export default mapApi

91
api/players.js Normal file
View File

@ -0,0 +1,91 @@
import { Router } from 'express'
import { sqlite } from '../db/init.js'
const playerApi = Router()
playerApi.get(
'/get/:player',
async (req, res) => {
let player = req.params.player
/* Misc */
const firstFinish = sqlite.prepare(`SELECT server as server, map as map, time as time, Timestamp as date FROM race WHERE name = ? ORDER BY Timestamp ASC LIMIT 1`).get(player)
/* TODO, make points a single table. Would be alot more efficent but longer cache creation time */
/* Points */
const points = sqlite.prepare(`SELECT rank, points FROM points WHERE name = ?`).get(player)
const pointsRank = sqlite.prepare(`SELECT rank, points FROM pointsRank WHERE name = ?`).get(player)
const pointsTeam = sqlite.prepare(`SELECT rank, points FROM pointsTeam WHERE name = ?`).get(player)
const pointsThisWeek = sqlite.prepare(`SELECT rank, points FROM pointsThisWeek WHERE name = ?`).get(player)
const pointsThisMonth = sqlite.prepare(`SELECT rank, points FROM pointsThisMonth WHERE name = ?`).get(player)
return res.json({
success: true,
response: {
firstFinish,
points,
pointsRank,
pointsTeam,
pointsThisWeek,
pointsThisMonth,
}
})
}
)
/* Searching allows you to attach sql LIKE %
* Example to search for players beginning with Test
* You can do search for Test%
*/
playerApi.get(
'/search',
async (req, res) => {
/* Check if a query was provided */
if (!req.query.q) {
return res.json({
success: false,
response: "No query ('host/path?q=query') provided!"
})
}
1
let name = req.query.q
/* Set defaults */
let limit = 20
let offset = 0
if (req.query.limit) {
limit = req.query.limit
}
const amount = sqlite.prepare(
`SELECT COUNT(*) as amount FROM points
WHERE name LIKE "${name}"
`).get()
let pages = Math.floor(amount.amount / limit)
if (req.query.page)
offset = (req.query.page * limit)
const players = sqlite.prepare(
`SELECT Rank, Name, Points FROM points
WHERE name LIKE "${name}" LIMIT ? OFFSET ?
`).all(limit, offset)
return res.json({
success: true,
response: {
amount: amount.amount,
pages: pages,
players: players,
}
})
}
)
export default playerApi

91
db/generate.js Normal file
View File

@ -0,0 +1,91 @@
import { sqlite, skinDB } from "./init.js"
import tasks from "./tasks.js"
/**
* This constructs the DB with indexes and rankings...
* @module db/generateDB
*/
export function generateDB() {
/* TODO: Clean this up as it is a mess */
console.log("Generating race index")
/* Generate race index TODO: Remove useless ones */
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_race_Map_2" ON "race" ("Map","Name")`)
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_race_Name" ON "race" ("Name","Timestamp")`)
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_race_Server" ON "race" ("Server")`)
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_race_MapTimestamp" ON "race" ("Map","Timestamp")`)
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_race_Timestamp" ON "race" ("Timestamp")`)
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_race_MapNameTime" ON "race" ("Map", "Name", "Time")`)
/* Create rankings table */
console.log("Creating rankings table")
sqlite.exec(`
CREATE TABLE IF NOT EXISTS "rankings" (
"Map" varchar(128) NOT NULL,
"Name" varchar(16) NOT NULL,
"Time" float NOT NULL DEFAULT 0,
"Timestamp" timestamp NOT NULL DEFAULT current_timestamp,
"Server" char(4) NOT NULL DEFAULT '',
"rank" INTEGER NOT NULL);
`)
console.log("Calculating rankings for each map")
tasks.processRankings()
/* Generate rankings index */
console.log("Generating rankings index")
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_rankings_map" ON "rankings" ("Map")`)
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_rankings_rank" ON "rankings" ("rank")`)
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_rankings_player" ON "rankings" ("Name")`)
/* Generate teamrace index */
console.log("Generating teamrace index")
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrace_Map" ON "teamrace" ("Map")`);
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrace_ID" ON "teamrace" ("ID")`);
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrace_MapID" ON "teamrace" ("Map", "ID")`);
console.log("Creating teamrankings table")
sqlite.exec(`
CREATE TABLE IF NOT EXISTS "teamrankings" (
"Map" varchar(128) NOT NULL,
"ID" varbinary(16) NOT NULL,
"Name" varchar(16) NOT NULL,
"Time" float NOT NULL DEFAULT 0,
"Timestamp" timestamp NOT NULL DEFAULT current_timestamp,
"Server" char(4) NOT NULL DEFAULT '',
"teamrank" INTEGER NOT NULL);
`)
console.log("Calculating teamrankings for each map")
tasks.processTeamRankings()
console.log("Generating teamrankings index")
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrankings_map" ON "teamrankings" ("Map")`)
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrankings_rank" ON "teamrankings" ("teamrank")`)
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_teamrankings_player" ON "teamrankings" ("name")`)
sqlite.exec(`
CREATE TABLE IF NOT EXISTS "points" (
"rank" INTEGER NOT NULL,
"name" varchar(16) NOT NULL,
"points" INTEGER NOT NULL);
`)
/* Process all types of points */
console.log("Inserting points to DB")
tasks.processAllPoints()
skinDB.exec(`
CREATE TABLE IF NOT EXISTS "skindata" (
"timestamp" INTEGER NOT NULL,
"player" varchar(16) NOT NULL,
"clan" varchar(12) NOT NULL,
"flag" INTEGER NOT NULL,
"skin" varchar(16) NOT NULL,
"useColor" INTEGER NOT NULL,
"colorBodyRaw" INTEGER NOT NULL,
"colorBodyHex" varchar(8) NOT NULL,
"colorFeetRaw" INTEGER NOT NULL,
"colorFeetHex" varchar(8) NOT NULL);
`)
}

26
db/init.js Normal file
View File

@ -0,0 +1,26 @@
import Database from 'better-sqlite3'
/* Export DB for use in other files */
export let sqlite = undefined
export let skinDB = undefined
/**
* This initalizes the ddnet.sqlite and skindata.sqlite DB...
* @module db/dbInit
*/
export function dbInit() {
console.log("Starting up databases...")
/* load in db using better-sqlite3 */
sqlite = new Database('ddnet.sqlite', { verbose: console.log });
skinDB = new Database('skindata.sqlite', { });
/* WAL mode */
sqlite.pragma('journal_mode = WAL');
/* Unsafe mode */
sqlite.unsafeMode()
console.log("Loaded in 'ddnet.sqlite'!")
console.log("Loaded in 'skindata.sqlite'!")
}

143
db/tasks.js Normal file
View File

@ -0,0 +1,143 @@
import msgpack from '@msgpack/msgpack'
import fs from 'fs'
import { sqlite } from "./init.js"
/**
* This module parses the msgpack provided by DDNet...
* @module db/decodeMsgpack
*/
export function decodeMsgpack() {
const data = fs.readFileSync('players.msgpack')
const decoded = msgpack.decodeMulti(data, {wrap: true})
const order = ['categories', 'maps', 'totalPoints', 'pointsRanks', 'pointsThisWeek', 'pointsThisMonth', 'teamRankPoints', 'rankPoints', 'serverRanks']
let final = {}
let i = 0
for (const part of decoded) {
final[order[i]] = part
++i
}
return final
}
/**
* This generates rankings for each map...
* @module db/processRankings
*/
export function processRankings() {
const maps = sqlite.prepare(`SELECT map FROM maps`);
for (const map of maps.iterate()) {
sqlite.prepare(
`
INSERT INTO rankings
(
map, name, time, timestamp, rank, server
)
SELECT map, name, time, timestamp, rank, server
FROM (
SELECT rank() OVER w AS rank,
map,
timestamp,
NAME,
min(time) AS time,
server
FROM race
WHERE map = ?
GROUP BY NAME window w AS (ORDER BY time) ) AS a
ORDER BY rank
`).run(map.Map)
}
}
/**
* This generates teamrankings for each map...
* @module db/processTeamRankings
*/
export function processTeamRankings() {
const maps = sqlite.prepare(`SELECT map FROM maps`);
for (const map of maps.iterate()) {
sqlite.prepare(
`
INSERT INTO teamrankings
(
name, map, id, time, timestamp, server, teamrank
)
SELECT DISTINCT(r.NAME),
r.map, r.id, r.time, r.timestamp,
Substring(n.server, 1, 3),
dense_rank() OVER w AS rank
FROM ((
SELECT DISTINCT id
FROM teamrace
WHERE map = ?
ORDER BY time) AS l
)
LEFT JOIN teamrace AS r
ON l.id = r.id
INNER JOIN race AS n
ON r.map = n.map
AND r.NAME = n.NAME
AND r.time = n.time window w AS (ORDER BY r.time)
`).run(map.Map)
}
}
/**
* This inserts all types of points into a table...
* @module db/processAllPoints
*/
export function processAllPoints() {
const msgpack = decodeMsgpack()
let types = {
points: msgpack.pointsRanks,
pointsThisWeek: msgpack.pointsThisWeek,
pointsThisMonth: msgpack.pointsThisMonth,
pointsTeam: msgpack.teamRankPoints,
pointsRank: msgpack.rankPoints,
}
/* Generate tables */
for(const type in types) {
sqlite.exec(
`
CREATE TABLE IF NOT EXISTS "${type}"
(
"rank" INTEGER NOT NULL,
"name" varchar(16) NOT NULL,
"points" INTEGER NOT NULL);
`)
}
/* Insert data */
for(const type in types) {
let rank = 1
for (const entry of types[type]) {
sqlite.prepare(
`
INSERT INTO "${type}"
(
rank, name, points
) VALUES (?, ?, ?)`).run(
rank, entry[0], entry[1])
++rank
}
}
/* Generate indexes */
for(const type in types) {
sqlite.exec(`CREATE INDEX IF NOT EXISTS "idx_${type}_player" ON "${type}" ("Name")`)
sqlite.exec(`CREATE INDEX IF NOT EXISTS "Idx_${type}_rank" on "${type}" ("Rank")`)
}
}
export default {
processAllPoints,
processRankings,
processTeamRankings
}

3
ddnet-links.txt Normal file
View File

@ -0,0 +1,3 @@
http://ddnet.tw/players.msgpack
https://ddnet.tw/status/index.json
https://ddnet.tw/stats/ddnet.sqlite.zip

102
ddnss/handler.js Normal file
View File

@ -0,0 +1,102 @@
import fetch from 'node-fetch'
import { exec } from 'child_process'
import { skinDB } from "../db/init.js"
export async function ddnssStart() {
const response = await fetch('https://ddnet.tw/status/index.json');
const data = await response.json();
console.log(data)
for (const servers of data) {
/* Check if server isn't empty and not full */
if (servers.num_clients > 0 && servers.num_clients < 59) {
let server = `${servers.ip}:${servers.port}`
console.log(`Connecting: ${servers.ip}:${servers.port}`)
/* exec ddnss */
await scrapeServer(`${server}`)
}
else
console.log(`Server full: ${servers.ip}:${servers.port}`)
}
}
export function scrapeServer(server) {
let command = `./ddnss/build/DDNet "ui_server_address ${server}" -f ddnss/build/config.conf`
let skinData
return new Promise((done, failed) => {
exec(command, { encoding: 'utf8', timeout: 10000 }, (err, stdout, stderr) => {
if (err) {
err.stdout = stdout
err.stderr = stderr
}
/* Handle error from parsing of JSON */
try {
skinData = JSON.parse(stdout)
} catch (e) {
done()
return
}
if (skinData === null) {
done()
return
}
/* Get timestamp */
let ts = (Date.now())
/* Insert skindata */
for (const entry of skinData) {
skinDB.prepare(`INSERT INTO "skindata"
(
timestamp,
player,
clan,
flag,
skin,
useColor,
colorBodyRaw,
colorBodyHex,
colorFeetRaw,
ColorFeetHex
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).
run(
ts,
entry.player,
entry.clan,
entry.flag,
entry.skindata.skin,
entry.skindata.useColor,
entry.skindata.colorBody.raw,
entry.skindata.colorBody.hex,
entry.skindata.colorFeet.raw,
entry.skindata.colorFeet.hex,
)
}
done()
/* A bit hacky way of killing ddnss */
//exec(`pkill -9 -f ddnss`)
})
})
}
/*
CREATE TABLE IF NOT EXISTS "skindata"
(
"timestamp" INTEGER NOT NULL,
"player" varchar(16) NOT NULL,
"clan" varchar(12) NOT NULL,
"flag" INTEGER NOT NULL,
"skin" varchar(16) NOT NULL,
"useColor" INTEGER NOT NULL,
"colorBodyRaw" INTEGER NOT NULL,
"colorBodyHex" varchar(8) NOT NULL,
"colorFeetRaw" INTEGER NOT NULL,
"colorFeetHex" varchar(8) NOT NULL);
*/

1
dotenv-template Normal file
View File

@ -0,0 +1 @@
PORT = 12345

28
index.js Normal file
View File

@ -0,0 +1,28 @@
import express from 'express'
import dotenv from 'dotenv'
import api from './api/api.js'
import { generateDB } from "./db/generate.js"
import { sqlite, dbInit } from "./db/init.js"
import { ddnssStart, scrapeServer } from './ddnss/handler.js'
/* Read the .env file */
dotenv.config()
/* Init db */
dbInit()
/* check if the table points exists */
const exists = sqlite.prepare(`SELECT count(*) as a FROM sqlite_master WHERE type='table' AND name='points'`).get()
/* Generate the DB if false */
if(!exists.a)
generateDB()
/* Init express */
const Server = express()
Server.use('/api', api)
Server.listen(process.env.PORT, () => console.log(`Server started and listening on port ${process.env.PORT}.`))
//ddnssStart()

20
package.json Normal file
View File

@ -0,0 +1,20 @@
{
"name": "ddstats-server-sqlite",
"version": "1.0.0",
"description": "",
"main": "index.js",
"dependencies": {
"@msgpack/msgpack": "^2.7.1",
"better-sqlite3": "^7.4.3",
"dotenv": "^10.0.0",
"express": "^4.17.1",
"mime-types": "^2.1.33",
"node-fetch": "^3.0.0"
},
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"type": "module",
"license": "ISC"
}