From b2e9827a5d386f73ab038be5d5373499454a54d0 Mon Sep 17 00:00:00 2001 From: BurnyLlama Date: Wed, 29 Sep 2021 20:29:06 +0200 Subject: [PATCH] sqlite2mongo can now import players. --- index.js | 11 +++-- libs/database/init.js | 29 ++++++++++++ libs/database/sqlite2mongo.js | 89 +++++++++++++++++++++++++++++++++++ libs/databaseHandler.js | 25 ---------- libs/sqlite2mongo.js | 77 ------------------------------ libs/utills/log.js | 3 ++ package.json | 5 +- schemas/Player.js | 3 +- 8 files changed, 134 insertions(+), 108 deletions(-) create mode 100644 libs/database/init.js create mode 100644 libs/database/sqlite2mongo.js delete mode 100644 libs/databaseHandler.js delete mode 100644 libs/sqlite2mongo.js create mode 100644 libs/utills/log.js diff --git a/index.js b/index.js index d68cef5..c9e42d7 100644 --- a/index.js +++ b/index.js @@ -1,6 +1,10 @@ import express from 'express' import dotenv from 'dotenv' -import sqlite2mongo from './libs/sqlite2mongo.js' +import sqlite2mongo from './libs/database/sqlite2mongo.js' +import databaseInit from './libs/database/init.js' +import initLog from './libs/utills/log.js' + +const log = initLog("[ MAIN ]") // Read the .env file dotenv.config() @@ -17,6 +21,7 @@ Server.get('/', (req, res) => ) ) -sqlite2mongo() +await databaseInit() +await sqlite2mongo() -Server.listen(process.env.PORT, () => console.log("Server has started!")) \ No newline at end of file +Server.listen(process.env.PORT, () => log(`Server started and listening on port ${process.env.PORT}.`)) \ No newline at end of file diff --git a/libs/database/init.js b/libs/database/init.js new file mode 100644 index 0000000..db976f0 --- /dev/null +++ b/libs/database/init.js @@ -0,0 +1,29 @@ +import initLog from '../utills/log.js' +import mongoose from 'mongoose' +import sqlite3 from 'sqlite3' +import { open } from 'sqlite' + +const log = initLog("database") + +export let sqlite = undefined + +async function init() { + log("Starting up databases...") + + sqlite = await open({ + filename: 'ddnet.sqlite', + driver: sqlite3.cached.Database + }) + log("Loaded in 'ddnet.sqlite'!") + + + await mongoose.connect( + process.env.MONGO_URI, + { + useNewUrlParser: true, + useUnifiedTopology: true + } + ).then(() => log("Connected to mongodb!")) +} + +export default init \ No newline at end of file diff --git a/libs/database/sqlite2mongo.js b/libs/database/sqlite2mongo.js new file mode 100644 index 0000000..5ee2dfd --- /dev/null +++ b/libs/database/sqlite2mongo.js @@ -0,0 +1,89 @@ +import initLog from "../utills/log.js" +import { sqlite } from "./init.js" +import Level from '../../schemas/Level.js' +import Player from '../../schemas/Player.js' + +const log = initLog("sqlite2mongo") + + + +async function checkForMaps() { + log("Checking for new maps...") + + const latestMap = await Level.findOne({}).sort({ "release": "desc" }) + const date = latestMap ? latestMap.release.toISOString().replace(/[TZ]/g, " ").replace(/\.[0-9\s]+/, "") : undefined + const newMapsAmount = (await sqlite.get(`SELECT count(Map) FROM maps ${date ? `WHERE Timestamp > datetime('${date}')` : ''} ORDER BY Timestamp`))['count(Map)'] + log(`Found ${newMapsAmount} new maps!`) + + let addedMaps = 0 + await sqlite.each( + `SELECT Map, Server, Points, Stars, Mapper, Timestamp FROM maps ${date ? `WHERE Timestamp > datetime('${date}')` : ''} ORDER BY Timestamp`, + [], + (err, map) => { + if (err) return log(err) + + Level.create({ + name: map.Map, + mapper: map.Mapper, + release: map.Timestamp === '0000-00-00 00:00:00' ? new Date('January 1, 1970 00:00:00 UTC') : new Date(`${map.Timestamp}+00:00`), + category: map.Server, + rating: map.Stars, + awardPoints: map.Points + }).then(() => { + ++addedMaps + log(`Added map ${addedMaps}/${newMapsAmount} -> ${map.Map}`) + }) + } + ) +} + + + +async function checkForPlayers() { + log("Checking for new players...") + + const latestPlayer = await Player.findOne({}).sort({ "firstFinish": "desc" }) + const date = latestPlayer ? latestPlayer.firstFinish.toISOString().replace(/[TZ]/g, " ").replace(/\.[0-9\s]+/, "") : undefined + const newPlayerAmount = (await sqlite.get(`SELECT count(Name) FROM (SELECT Name, Timestamp FROM (SELECT * FROM race ORDER BY Timestamp ASC, Name ASC) GROUP BY Name) ${date ? `WHERE Timestamp > datetime('${date}')` : ''}`))['count(Name)'] + if (newPlayerAmount === 0) return log("No new players found...") + + log(`Found ${newPlayerAmount} new players!`) + log("Importing new players...") + await sqlite.exec("DROP TABLE IF EXISTS temp") + await sqlite.exec("CREATE TABLE temp(Name varchar(128) NOT NULL, Timestamp timestamp NOT NULL)") + await sqlite.exec(`INSERT INTO TEMP (Name, Timestamp) SELECT Name, Timestamp FROM (SELECT Name, Timestamp FROM (SELECT * FROM race ORDER BY Timestamp ASC, Name ASC) GROUP BY Name) ${date ? `WHERE Timestamp > datetime('${date}')` : ''}`) + log("Imported new players into 'temp'!") + + let addedPlayers = 0 + let offset = -1 + while (offset < newPlayerAmount) { + await sqlite.each( + `SELECT Name, Timestamp FROM temp LIMIT 10000 OFFSET ${offset + 1}`, + [], + async (err, player) => { + if (err) return log(err) + if (addedPlayers >= newPlayerAmount) return {} + + Player.create({ + name: player.Name, + firstFinish: player.Timestamp === '0000-00-00 00:00:00' ? new Date('January 1, 1970 00:00:00 UTC') : new Date(`${player.Timestamp}+00:00`) + }).then(() => { + ++addedPlayers + log(`Added player ${addedPlayers}/${newPlayerAmount} -> ${player.Name}`) + }) + } + ) + offset += 10000 + } +} + + + +async function sqlite2mongo() { + log("Checking for additions to 'ddnet.sqlite'...") + + await checkForMaps() + await checkForPlayers() +} + +export default sqlite2mongo \ No newline at end of file diff --git a/libs/databaseHandler.js b/libs/databaseHandler.js deleted file mode 100644 index 23a0353..0000000 --- a/libs/databaseHandler.js +++ /dev/null @@ -1,25 +0,0 @@ -import betterSqlite3 from 'better-sqlite3' -import mongoose from 'mongoose' - -const sqlite = new betterSqlite3('ddnet.sqlite', { readonly: true }) - -function initMongo() { - mongoose.connect( - process.env.MONGO_URI, - { - useNewUrlParser: true, - useUnifiedTopology: true - }, - () => "Connected to mongodb!" - ) -} - -function getAllMaps(player) { - const stmt = sqlite.prepare('SELECT Name, Map, Time, Time, Timestamp, Server FROM race WHERE Name = ?') - return stmt.all(player) -} - -export default { - initMongo, - getAllMaps -} \ No newline at end of file diff --git a/libs/sqlite2mongo.js b/libs/sqlite2mongo.js deleted file mode 100644 index 046bc26..0000000 --- a/libs/sqlite2mongo.js +++ /dev/null @@ -1,77 +0,0 @@ -import betterSqlite3 from 'better-sqlite3' -import mongoose from 'mongoose' -import Level from '../schemas/Level.js' - -function log(string) { - console.log(`sqlite2mongo >>> ${string}`) -} - -export default async function() { - log("Reconstructing the Database!") - await mongoose.connect( - process.env.MONGO_URI, - { - useNewUrlParser: true, - useUnifiedTopology: true - } - ).then(async () => { - log("Connected to mongodb!") - - const sqlite = new betterSqlite3('ddnet.sqlite') - log("Initialised sqlite!") - - log("Adding levels (maps)...") - const latestMapAddition = (await Level.findOne({}).sort({ "release": "desc" })).release.toISOString().replace(/[TZ]/g, " ").replace(/\.[0-9\s]+/, "") - const allMaps = sqlite.prepare("SELECT Map, Server, Points, Stars, Mapper, Timestamp FROM maps WHERE Timestamp > ? ORDER BY Timestamp").iterate(`datetime('${latestMapAddition}')`) - const newMapAmount = sqlite.prepare("SELECT count(Map) FROM maps WHERE Timestamp > ? ORDER BY Timestamp").get(`datetime('${latestMapAddition}')`)['count(Map)'] - let addedMapCount = 0 - console.log({ allMaps, newMapAmount, addedMapCount }) - for (let map of allMaps) { - Level.create({ - name: map.Map, - mapper: map.Mapper, - release: map.Timestamp === '0000-00-00 00:00:00' ? new Date('January 1, 1970 00:00:00 UTC') : new Date(map.Timestamp), - category: map.Server, - rating: map.Stars, - awardPoints: map.Points - }).then(() => { - ++addedMapCount - log(`Adding map ${addedMapCount}/${newMapAmount} -> ${map.Map}`) - }) - } - - // log("Adding players...") - // let addedPlayerCount = 0 - // const allPlayers = sqlite.prepare(`SELECT distinct Name FROM race LIMIT 10000`).iterate() - // for (const player of allPlayers) { - // Player.create({ - // name: player.name - // }).then(() => { - // ++addedPlayerCount - // log(`Adding player ${addedPlayerCount}/10000 -> ${player.Name}`) - // }) - // } - -// log("Adding finishes (races)...") -// const rows = sqlite.prepare("SELECT count(Name) FROM race;").get()['count(Name)'] -// let addedFinishCount = 0 - -// for (let i = 0; i < rows; i += 5000) { -// const allFinishes = sqlite.prepare(`SELECT Map, Name, Time, Timestamp, Server FROM race LIMIT 5000 OFFSET ${i + 1}`).all() -// for (let finish of allFinishes) { -// Level.findOne({ name: finish.Map }).then(map => { -// Player.findOne({ name: finish.Name }).then(async player => { -// await Finish.create({ -// map: map._id, -// time: finish.Time, -// date: finish.Timestamp === '0000-00-00 00:00:00' ? new Date('January 1, 1970 00:00:00 UTC') : new Date(finish.Timestamp), -// players: player._id -// }) -// ++addedFinishCount -// log(`Adding race ${addedFinishCount}/${rows} -> ${finish.Name} @ ${finish.Map}`) -// }) -// }) -// } -// } - }) -} \ No newline at end of file diff --git a/libs/utills/log.js b/libs/utills/log.js new file mode 100644 index 0000000..9b88482 --- /dev/null +++ b/libs/utills/log.js @@ -0,0 +1,3 @@ +export default function initLog(prefix) { + return string => console.log(`${prefix} >>> ${string}`) +} \ No newline at end of file diff --git a/package.json b/package.json index 88ec613..5af0f44 100644 --- a/package.json +++ b/package.json @@ -17,9 +17,10 @@ "author": "BurnyLlama, furo", "license": "WTFPL", "dependencies": { - "better-sqlite3": "^7.4.3", "dotenv": "^10.0.0", "express": "^4.17.1", - "mongoose": "^6.0.7" + "mongoose": "^6.0.7", + "sqlite": "^4.0.23", + "sqlite3": "^5.0.2" } } diff --git a/schemas/Player.js b/schemas/Player.js index 6a77c07..e801751 100644 --- a/schemas/Player.js +++ b/schemas/Player.js @@ -4,7 +4,8 @@ const Player = new mongoose.Schema({ name: String, points: Number, rankPoints: Number, - teamPoints: Number + teamPoints: Number, + firstFinish: Date }) export default mongoose.model("Player", Player) \ No newline at end of file