sqlite2mongo can now import players.

This commit is contained in:
BurnyLlama 2021-09-29 20:29:06 +02:00
parent 13cd9d643b
commit b2e9827a5d
8 changed files with 134 additions and 108 deletions

View File

@ -1,6 +1,10 @@
import express from 'express' import express from 'express'
import dotenv from 'dotenv' import dotenv from 'dotenv'
import sqlite2mongo from './libs/sqlite2mongo.js' import sqlite2mongo from './libs/database/sqlite2mongo.js'
import databaseInit from './libs/database/init.js'
import initLog from './libs/utills/log.js'
const log = initLog("[ MAIN ]")
// Read the .env file // Read the .env file
dotenv.config() dotenv.config()
@ -17,6 +21,7 @@ Server.get('/', (req, res) =>
) )
) )
sqlite2mongo() await databaseInit()
await sqlite2mongo()
Server.listen(process.env.PORT, () => console.log("Server has started!")) Server.listen(process.env.PORT, () => log(`Server started and listening on port ${process.env.PORT}.`))

29
libs/database/init.js Normal file
View File

@ -0,0 +1,29 @@
import initLog from '../utills/log.js'
import mongoose from 'mongoose'
import sqlite3 from 'sqlite3'
import { open } from 'sqlite'
const log = initLog("database")
export let sqlite = undefined
async function init() {
log("Starting up databases...")
sqlite = await open({
filename: 'ddnet.sqlite',
driver: sqlite3.cached.Database
})
log("Loaded in 'ddnet.sqlite'!")
await mongoose.connect(
process.env.MONGO_URI,
{
useNewUrlParser: true,
useUnifiedTopology: true
}
).then(() => log("Connected to mongodb!"))
}
export default init

View File

@ -0,0 +1,89 @@
import initLog from "../utills/log.js"
import { sqlite } from "./init.js"
import Level from '../../schemas/Level.js'
import Player from '../../schemas/Player.js'
const log = initLog("sqlite2mongo")
async function checkForMaps() {
log("Checking for new maps...")
const latestMap = await Level.findOne({}).sort({ "release": "desc" })
const date = latestMap ? latestMap.release.toISOString().replace(/[TZ]/g, " ").replace(/\.[0-9\s]+/, "") : undefined
const newMapsAmount = (await sqlite.get(`SELECT count(Map) FROM maps ${date ? `WHERE Timestamp > datetime('${date}')` : ''} ORDER BY Timestamp`))['count(Map)']
log(`Found ${newMapsAmount} new maps!`)
let addedMaps = 0
await sqlite.each(
`SELECT Map, Server, Points, Stars, Mapper, Timestamp FROM maps ${date ? `WHERE Timestamp > datetime('${date}')` : ''} ORDER BY Timestamp`,
[],
(err, map) => {
if (err) return log(err)
Level.create({
name: map.Map,
mapper: map.Mapper,
release: map.Timestamp === '0000-00-00 00:00:00' ? new Date('January 1, 1970 00:00:00 UTC') : new Date(`${map.Timestamp}+00:00`),
category: map.Server,
rating: map.Stars,
awardPoints: map.Points
}).then(() => {
++addedMaps
log(`Added map ${addedMaps}/${newMapsAmount} -> ${map.Map}`)
})
}
)
}
async function checkForPlayers() {
log("Checking for new players...")
const latestPlayer = await Player.findOne({}).sort({ "firstFinish": "desc" })
const date = latestPlayer ? latestPlayer.firstFinish.toISOString().replace(/[TZ]/g, " ").replace(/\.[0-9\s]+/, "") : undefined
const newPlayerAmount = (await sqlite.get(`SELECT count(Name) FROM (SELECT Name, Timestamp FROM (SELECT * FROM race ORDER BY Timestamp ASC, Name ASC) GROUP BY Name) ${date ? `WHERE Timestamp > datetime('${date}')` : ''}`))['count(Name)']
if (newPlayerAmount === 0) return log("No new players found...")
log(`Found ${newPlayerAmount} new players!`)
log("Importing new players...")
await sqlite.exec("DROP TABLE IF EXISTS temp")
await sqlite.exec("CREATE TABLE temp(Name varchar(128) NOT NULL, Timestamp timestamp NOT NULL)")
await sqlite.exec(`INSERT INTO TEMP (Name, Timestamp) SELECT Name, Timestamp FROM (SELECT Name, Timestamp FROM (SELECT * FROM race ORDER BY Timestamp ASC, Name ASC) GROUP BY Name) ${date ? `WHERE Timestamp > datetime('${date}')` : ''}`)
log("Imported new players into 'temp'!")
let addedPlayers = 0
let offset = -1
while (offset < newPlayerAmount) {
await sqlite.each(
`SELECT Name, Timestamp FROM temp LIMIT 10000 OFFSET ${offset + 1}`,
[],
async (err, player) => {
if (err) return log(err)
if (addedPlayers >= newPlayerAmount) return {}
Player.create({
name: player.Name,
firstFinish: player.Timestamp === '0000-00-00 00:00:00' ? new Date('January 1, 1970 00:00:00 UTC') : new Date(`${player.Timestamp}+00:00`)
}).then(() => {
++addedPlayers
log(`Added player ${addedPlayers}/${newPlayerAmount} -> ${player.Name}`)
})
}
)
offset += 10000
}
}
async function sqlite2mongo() {
log("Checking for additions to 'ddnet.sqlite'...")
await checkForMaps()
await checkForPlayers()
}
export default sqlite2mongo

View File

@ -1,25 +0,0 @@
import betterSqlite3 from 'better-sqlite3'
import mongoose from 'mongoose'
const sqlite = new betterSqlite3('ddnet.sqlite', { readonly: true })
function initMongo() {
mongoose.connect(
process.env.MONGO_URI,
{
useNewUrlParser: true,
useUnifiedTopology: true
},
() => "Connected to mongodb!"
)
}
function getAllMaps(player) {
const stmt = sqlite.prepare('SELECT Name, Map, Time, Time, Timestamp, Server FROM race WHERE Name = ?')
return stmt.all(player)
}
export default {
initMongo,
getAllMaps
}

View File

@ -1,77 +0,0 @@
import betterSqlite3 from 'better-sqlite3'
import mongoose from 'mongoose'
import Level from '../schemas/Level.js'
function log(string) {
console.log(`sqlite2mongo >>> ${string}`)
}
export default async function() {
log("Reconstructing the Database!")
await mongoose.connect(
process.env.MONGO_URI,
{
useNewUrlParser: true,
useUnifiedTopology: true
}
).then(async () => {
log("Connected to mongodb!")
const sqlite = new betterSqlite3('ddnet.sqlite')
log("Initialised sqlite!")
log("Adding levels (maps)...")
const latestMapAddition = (await Level.findOne({}).sort({ "release": "desc" })).release.toISOString().replace(/[TZ]/g, " ").replace(/\.[0-9\s]+/, "")
const allMaps = sqlite.prepare("SELECT Map, Server, Points, Stars, Mapper, Timestamp FROM maps WHERE Timestamp > ? ORDER BY Timestamp").iterate(`datetime('${latestMapAddition}')`)
const newMapAmount = sqlite.prepare("SELECT count(Map) FROM maps WHERE Timestamp > ? ORDER BY Timestamp").get(`datetime('${latestMapAddition}')`)['count(Map)']
let addedMapCount = 0
console.log({ allMaps, newMapAmount, addedMapCount })
for (let map of allMaps) {
Level.create({
name: map.Map,
mapper: map.Mapper,
release: map.Timestamp === '0000-00-00 00:00:00' ? new Date('January 1, 1970 00:00:00 UTC') : new Date(map.Timestamp),
category: map.Server,
rating: map.Stars,
awardPoints: map.Points
}).then(() => {
++addedMapCount
log(`Adding map ${addedMapCount}/${newMapAmount} -> ${map.Map}`)
})
}
// log("Adding players...")
// let addedPlayerCount = 0
// const allPlayers = sqlite.prepare(`SELECT distinct Name FROM race LIMIT 10000`).iterate()
// for (const player of allPlayers) {
// Player.create({
// name: player.name
// }).then(() => {
// ++addedPlayerCount
// log(`Adding player ${addedPlayerCount}/10000 -> ${player.Name}`)
// })
// }
// log("Adding finishes (races)...")
// const rows = sqlite.prepare("SELECT count(Name) FROM race;").get()['count(Name)']
// let addedFinishCount = 0
// for (let i = 0; i < rows; i += 5000) {
// const allFinishes = sqlite.prepare(`SELECT Map, Name, Time, Timestamp, Server FROM race LIMIT 5000 OFFSET ${i + 1}`).all()
// for (let finish of allFinishes) {
// Level.findOne({ name: finish.Map }).then(map => {
// Player.findOne({ name: finish.Name }).then(async player => {
// await Finish.create({
// map: map._id,
// time: finish.Time,
// date: finish.Timestamp === '0000-00-00 00:00:00' ? new Date('January 1, 1970 00:00:00 UTC') : new Date(finish.Timestamp),
// players: player._id
// })
// ++addedFinishCount
// log(`Adding race ${addedFinishCount}/${rows} -> ${finish.Name} @ ${finish.Map}`)
// })
// })
// }
// }
})
}

3
libs/utills/log.js Normal file
View File

@ -0,0 +1,3 @@
export default function initLog(prefix) {
return string => console.log(`${prefix} >>> ${string}`)
}

View File

@ -17,9 +17,10 @@
"author": "BurnyLlama, furo", "author": "BurnyLlama, furo",
"license": "WTFPL", "license": "WTFPL",
"dependencies": { "dependencies": {
"better-sqlite3": "^7.4.3",
"dotenv": "^10.0.0", "dotenv": "^10.0.0",
"express": "^4.17.1", "express": "^4.17.1",
"mongoose": "^6.0.7" "mongoose": "^6.0.7",
"sqlite": "^4.0.23",
"sqlite3": "^5.0.2"
} }
} }

View File

@ -4,7 +4,8 @@ const Player = new mongoose.Schema({
name: String, name: String,
points: Number, points: Number,
rankPoints: Number, rankPoints: Number,
teamPoints: Number teamPoints: Number,
firstFinish: Date
}) })
export default mongoose.model("Player", Player) export default mongoose.model("Player", Player)