diff --git a/libs/database/sqlite2mongo.js b/libs/database/sqlite2mongo.js index d4bc5c9..0baabb3 100644 --- a/libs/database/sqlite2mongo.js +++ b/libs/database/sqlite2mongo.js @@ -1,128 +1,17 @@ -import initLog from "../utills/log.js" -import { sqlite } from "./init.js" -import Level from '../../schemas/Level.js' -import Player from '../../schemas/Player.js' -import Finish from '../../schemas/Finish.js' +import initLog from '../utills/log.js' +import { checkForFinishes, checkForMaps, checkForPlayers } from './sqlite2mongo/checks.js' +import postTasks from './sqlite2mongo/postTasks.js' const log = initLog("sqlite2mongo") - - -async function checkForMaps() { - log("Checking for new maps...") - - const latestMap = await Level.findOne({}).sort({ "release": "desc" }) - const date = latestMap ? latestMap.release.toISOString().replace(/[TZ]/g, " ").replace(/\.[0-9\s]+/, "") : undefined - const newMapsAmount = (await sqlite.get(`SELECT count(Map) FROM maps ${date ? `WHERE Timestamp > datetime('${date}')` : ''} ORDER BY Timestamp`))['count(Map)'] - log(`Found ${newMapsAmount} new maps!`) - - let addedMaps = 0 - await sqlite.each( - `SELECT Map, Server, Points, Stars, Mapper, Timestamp FROM maps ${date ? `WHERE Timestamp > datetime('${date}')` : ''} ORDER BY Timestamp`, - [], - (err, map) => { - if (err) return log(err) - - Level.create({ - name: map.Map, - mapper: map.Mapper, - release: map.Timestamp === '0000-00-00 00:00:00' ? new Date('January 1, 1970 00:00:00 UTC') : new Date(`${map.Timestamp}+00:00`), - category: map.Server, - rating: map.Stars, - awardPoints: map.Points - }).then(() => { - ++addedMaps - log(`Added map ${addedMaps}/${newMapsAmount} -> ${map.Map}`) - }) - } - ) -} - - - -async function checkForPlayers() { - log("Checking for new players...") - - const latestPlayer = await Player.findOne({}).sort({ "firstFinish": "desc" }) - const date = latestPlayer ? latestPlayer.firstFinish.toISOString().replace(/[TZ]/g, " ").replace(/\.[0-9\s]+/, "") : undefined - const newPlayerAmount = (await sqlite.get(`select count(Name) FROM (SELECT Name, Timestamp FROM (SELECT Name, Timestamp FROM (SELECT * FROM race ORDER BY Timestamp ASC, Name ASC) GROUP BY Name) ${date ? `WHERE Timestamp > datetime('${date}')` : ''} ORDER BY Timestamp ASC)`))['count(Name)'] - if (newPlayerAmount === 0) return log("No new players found...") - - log(`Found ${newPlayerAmount} new players!`) - log("Importing new players...") - await sqlite.exec("DROP TABLE IF EXISTS temp") - await sqlite.exec("CREATE TABLE temp(Name varchar(128) NOT NULL, Timestamp timestamp NOT NULL)") - await sqlite.exec(`INSERT INTO TEMP (Name, Timestamp) SELECT Name, Timestamp FROM (SELECT Name, Timestamp FROM (SELECT Name, Timestamp FROM (SELECT * FROM race ORDER BY Timestamp ASC, Name ASC) GROUP BY Name) ${date ? `WHERE Timestamp > datetime('${date}')` : ''} ORDER BY Timestamp ASC)`) - log("Imported new players into 'temp'!") - - let addedPlayers = 0 - let offset = -1 - while (offset < newPlayerAmount) { - await sqlite.each( - `SELECT Name, Timestamp FROM temp LIMIT 10000 OFFSET ${offset + 1}`, - [], - async (err, player) => { - if (err) return log(err) - if (addedPlayers >= newPlayerAmount) return {} - - Player.create({ - name: player.Name, - firstFinish: player.Timestamp === '0000-00-00 00:00:00' ? new Date('January 1, 1970 00:00:00 UTC') : new Date(`${player.Timestamp}+00:00`) - }).then(() => { - ++addedPlayers - log(`Added player ${addedPlayers}/${newPlayerAmount} -> ${player.Name}`) - }) - } - ) - offset += 10000 - } -} - - -async function checkForFinishes() { - log("Checking for new finishes...") - - const latestFinish = await Finish.findOne({}).sort({ "date": "desc" }) - const date = latestFinish ? latestFinish.date.toISOString().replace(/[TZ]/g, " ").replace(/\.[0-9\s]+/, "") : undefined - const newFinishAmount = (await sqlite.get(`SELECT count(Name) FROM race ${date ? `WHERE Timestamp > datetime('${date}')` : ''}`))['count(Name)'] - if (newFinishAmount === 0) return log("No new finishes found...") - - log(`Found ${newFinishAmount} new finishes!`) - - let addedFinishes = 0 - let offset = -1 - while (offset < newFinishAmount) { - await sqlite.each( - `SELECT * FROM race ${date ? `WHERE Timestamp > datetime('${date}')` : ''} ORDER BY Timestamp LIMIT 5000 OFFSET ${offset + 1}`, - [], - async (err, finish) => { - if (err) return log(err) - if (addedFinishes >= newFinishAmount) return {} - - Finish.create({ - map: finish.Map, - time: finish.Time, - date: finish.Timestamp === '0000-00-00 00:00:00' ? new Date('January 1, 1970 00:00:00 UTC') : new Date(`${finish.Timestamp}+00:00`), - serverLocation: finish.Server ?? '', - player: finish.Name - }).then(() => { - ++addedFinishes - log(`Added finish ${addedFinishes}/${newFinishAmount} -> At ${finish.Timestamp} «${finish.Name}» completed «${finish.Map}» in ${finish.Time} s`) - }) - } - ) - offset += 5000 - } -} - - - async function sqlite2mongo() { log("Checking for additions to 'ddnet.sqlite'...") await checkForMaps() await checkForPlayers() await checkForFinishes() + + await postTasks() } export default sqlite2mongo \ No newline at end of file diff --git a/libs/database/sqlite2mongo/checks.js b/libs/database/sqlite2mongo/checks.js new file mode 100644 index 0000000..6656495 --- /dev/null +++ b/libs/database/sqlite2mongo/checks.js @@ -0,0 +1,116 @@ +import { sqlite } from "../init.js" +import Level from '../../../schemas/Level.js' +import Player from '../../../schemas/Player.js' +import Finish from '../../../schemas/Finish.js' +import initLog from '../../utills/log.js' + + +const log = initLog("sqlite2mongo") + + +export async function checkForMaps() { + log("Checking for new maps...") + + const latestMap = await Level.findOne({}).sort({ "release": "desc" }) + const date = latestMap ? latestMap.release.toISOString().replace(/[TZ]/g, " ").replace(/\.[0-9\s]+/, "") : undefined + const newMapsAmount = (await sqlite.get(`SELECT count(Map) FROM maps ${date ? `WHERE Timestamp > datetime('${date}')` : ''} ORDER BY Timestamp`))['count(Map)'] + log(`Found ${newMapsAmount} new maps!`) + + let addedMaps = 0 + await sqlite.each( + `SELECT Map, Server, Points, Stars, Mapper, Timestamp FROM maps ${date ? `WHERE Timestamp > datetime('${date}')` : ''} ORDER BY Timestamp`, + [], + (err, map) => { + if (err) return log(err) + + Level.create({ + name: map.Map, + mapper: map.Mapper, + release: map.Timestamp === '0000-00-00 00:00:00' ? new Date('January 1, 1970 00:00:00 UTC') : new Date(`${map.Timestamp}+00:00`), + category: map.Server, + rating: map.Stars, + awardPoints: map.Points + }).then(() => { + ++addedMaps + log(`Added map ${addedMaps}/${newMapsAmount} -> ${map.Map}`) + }) + } + ) +} + + + +export async function checkForPlayers() { + log("Checking for new players...") + + const latestPlayer = await Player.findOne({}).sort({ "firstFinish": "desc" }) + const date = latestPlayer ? latestPlayer.firstFinish.toISOString().replace(/[TZ]/g, " ").replace(/\.[0-9\s]+/, "") : undefined + const newPlayerAmount = (await sqlite.get(`select count(Name) FROM (SELECT Name, Timestamp FROM (SELECT Name, Timestamp FROM (SELECT * FROM race ORDER BY Timestamp ASC, Name ASC) GROUP BY Name) ${date ? `WHERE Timestamp > datetime('${date}')` : ''} ORDER BY Timestamp ASC)`))['count(Name)'] + if (newPlayerAmount === 0) return log("No new players found...") + + log(`Found ${newPlayerAmount} new players!`) + log("Importing new players...") + await sqlite.exec("DROP TABLE IF EXISTS temp") + await sqlite.exec("CREATE TABLE temp(Name varchar(128) NOT NULL, Timestamp timestamp NOT NULL)") + await sqlite.exec(`INSERT INTO TEMP (Name, Timestamp) SELECT Name, Timestamp FROM (SELECT Name, Timestamp FROM (SELECT Name, Timestamp FROM (SELECT * FROM race ORDER BY Timestamp ASC, Name ASC) GROUP BY Name) ${date ? `WHERE Timestamp > datetime('${date}')` : ''} ORDER BY Timestamp ASC)`) + log("Imported new players into 'temp'!") + + let addedPlayers = 0 + let offset = -1 + while (offset < newPlayerAmount) { + await sqlite.each( + `SELECT Name, Timestamp FROM temp LIMIT 10000 OFFSET ${offset + 1}`, + [], + async (err, player) => { + if (err) return log(err) + if (addedPlayers >= newPlayerAmount) return {} + + Player.create({ + name: player.Name, + firstFinish: player.Timestamp === '0000-00-00 00:00:00' ? new Date('January 1, 1970 00:00:00 UTC') : new Date(`${player.Timestamp}+00:00`) + }).then(() => { + ++addedPlayers + log(`Added player ${addedPlayers}/${newPlayerAmount} -> ${player.Name}`) + }) + } + ) + offset += 10000 + } +} + + +export async function checkForFinishes() { + log("Checking for new finishes...") + + const latestFinish = await Finish.findOne({}).sort({ "date": "desc" }) + const date = latestFinish ? latestFinish.date.toISOString().replace(/[TZ]/g, " ").replace(/\.[0-9\s]+/, "") : undefined + const newFinishAmount = (await sqlite.get(`SELECT count(Name) FROM race ${date ? `WHERE Timestamp > datetime('${date}')` : ''}`))['count(Name)'] + if (newFinishAmount === 0) return log("No new finishes found...") + + log(`Found ${newFinishAmount} new finishes!`) + + let addedFinishes = 0 + let offset = -1 + while (offset < newFinishAmount) { + await sqlite.each( + `SELECT * FROM race ${date ? `WHERE Timestamp > datetime('${date}')` : ''} ORDER BY Timestamp LIMIT 5000 OFFSET ${offset + 1}`, + [], + async (err, finish) => { + if (err) return log(err) + if (addedFinishes >= newFinishAmount) return {} + + Finish.create({ + map: finish.Map, + time: finish.Time, + date: finish.Timestamp === '0000-00-00 00:00:00' ? new Date('January 1, 1970 00:00:00 UTC') : new Date(`${finish.Timestamp}+00:00`), + serverLocation: finish.Server ?? '', + player: finish.Name + }).then(() => { + ++addedFinishes + log(`Added finish ${addedFinishes}/${newFinishAmount} -> At ${finish.Timestamp} «${finish.Name}» completed «${finish.Map}» in ${finish.Time} s`) + }) + } + ) + offset += 5000 + } +} \ No newline at end of file diff --git a/libs/database/sqlite2mongo/postTasks.js b/libs/database/sqlite2mongo/postTasks.js new file mode 100644 index 0000000..baa6a07 --- /dev/null +++ b/libs/database/sqlite2mongo/postTasks.js @@ -0,0 +1,26 @@ +import Finish from '../../../schemas/Finish.js' +import Level from '../../../schemas/Level.js' +import Player from '../../../schemas/Player.js' +import initLog from '../../utills/log.js' +import { sqlite } from '../init.js' + +const log = initLog("sqlite2mongo") + +export default async function postTasks() { + log("Post tasks...") + + log("Calculating player points...") + const playerAmount = await Player.find({}).count() + const playerAgg = Player.aggregate([]) + + let processedPlayers = 0 + for await (const player of playerAgg) { + const points = (await sqlite.get(`SELECT sum(Points) FROM maps JOIN (SELECT DISTINCT Map FROM race WHERE Name = "${player.name}") AS r ON maps.Map = r.Map`))['sum(Points)'] + player.points = points + + Player.updateOne({ name: player.name }, { rankPoints: points }, () => log(`Task ${processedPlayers}/${playerAmount} >> Player «${player.name}» has ${points} points!`)) + ++processedPlayers + } + + log("Done calculating points!") +} \ No newline at end of file