sqlite2mongo can now import players.
This commit is contained in:
parent
13cd9d643b
commit
b2e9827a5d
11
index.js
11
index.js
|
|
@ -1,6 +1,10 @@
|
|||
import express from 'express'
|
||||
import dotenv from 'dotenv'
|
||||
import sqlite2mongo from './libs/sqlite2mongo.js'
|
||||
import sqlite2mongo from './libs/database/sqlite2mongo.js'
|
||||
import databaseInit from './libs/database/init.js'
|
||||
import initLog from './libs/utills/log.js'
|
||||
|
||||
const log = initLog("[ MAIN ]")
|
||||
|
||||
// Read the .env file
|
||||
dotenv.config()
|
||||
|
|
@ -17,6 +21,7 @@ Server.get('/', (req, res) =>
|
|||
)
|
||||
)
|
||||
|
||||
sqlite2mongo()
|
||||
await databaseInit()
|
||||
await sqlite2mongo()
|
||||
|
||||
Server.listen(process.env.PORT, () => console.log("Server has started!"))
|
||||
Server.listen(process.env.PORT, () => log(`Server started and listening on port ${process.env.PORT}.`))
|
||||
29
libs/database/init.js
Normal file
29
libs/database/init.js
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
import initLog from '../utills/log.js'
|
||||
import mongoose from 'mongoose'
|
||||
import sqlite3 from 'sqlite3'
|
||||
import { open } from 'sqlite'
|
||||
|
||||
const log = initLog("database")
|
||||
|
||||
export let sqlite = undefined
|
||||
|
||||
async function init() {
|
||||
log("Starting up databases...")
|
||||
|
||||
sqlite = await open({
|
||||
filename: 'ddnet.sqlite',
|
||||
driver: sqlite3.cached.Database
|
||||
})
|
||||
log("Loaded in 'ddnet.sqlite'!")
|
||||
|
||||
|
||||
await mongoose.connect(
|
||||
process.env.MONGO_URI,
|
||||
{
|
||||
useNewUrlParser: true,
|
||||
useUnifiedTopology: true
|
||||
}
|
||||
).then(() => log("Connected to mongodb!"))
|
||||
}
|
||||
|
||||
export default init
|
||||
89
libs/database/sqlite2mongo.js
Normal file
89
libs/database/sqlite2mongo.js
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
import initLog from "../utills/log.js"
|
||||
import { sqlite } from "./init.js"
|
||||
import Level from '../../schemas/Level.js'
|
||||
import Player from '../../schemas/Player.js'
|
||||
|
||||
const log = initLog("sqlite2mongo")
|
||||
|
||||
|
||||
|
||||
async function checkForMaps() {
|
||||
log("Checking for new maps...")
|
||||
|
||||
const latestMap = await Level.findOne({}).sort({ "release": "desc" })
|
||||
const date = latestMap ? latestMap.release.toISOString().replace(/[TZ]/g, " ").replace(/\.[0-9\s]+/, "") : undefined
|
||||
const newMapsAmount = (await sqlite.get(`SELECT count(Map) FROM maps ${date ? `WHERE Timestamp > datetime('${date}')` : ''} ORDER BY Timestamp`))['count(Map)']
|
||||
log(`Found ${newMapsAmount} new maps!`)
|
||||
|
||||
let addedMaps = 0
|
||||
await sqlite.each(
|
||||
`SELECT Map, Server, Points, Stars, Mapper, Timestamp FROM maps ${date ? `WHERE Timestamp > datetime('${date}')` : ''} ORDER BY Timestamp`,
|
||||
[],
|
||||
(err, map) => {
|
||||
if (err) return log(err)
|
||||
|
||||
Level.create({
|
||||
name: map.Map,
|
||||
mapper: map.Mapper,
|
||||
release: map.Timestamp === '0000-00-00 00:00:00' ? new Date('January 1, 1970 00:00:00 UTC') : new Date(`${map.Timestamp}+00:00`),
|
||||
category: map.Server,
|
||||
rating: map.Stars,
|
||||
awardPoints: map.Points
|
||||
}).then(() => {
|
||||
++addedMaps
|
||||
log(`Added map ${addedMaps}/${newMapsAmount} -> ${map.Map}`)
|
||||
})
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
|
||||
async function checkForPlayers() {
|
||||
log("Checking for new players...")
|
||||
|
||||
const latestPlayer = await Player.findOne({}).sort({ "firstFinish": "desc" })
|
||||
const date = latestPlayer ? latestPlayer.firstFinish.toISOString().replace(/[TZ]/g, " ").replace(/\.[0-9\s]+/, "") : undefined
|
||||
const newPlayerAmount = (await sqlite.get(`SELECT count(Name) FROM (SELECT Name, Timestamp FROM (SELECT * FROM race ORDER BY Timestamp ASC, Name ASC) GROUP BY Name) ${date ? `WHERE Timestamp > datetime('${date}')` : ''}`))['count(Name)']
|
||||
if (newPlayerAmount === 0) return log("No new players found...")
|
||||
|
||||
log(`Found ${newPlayerAmount} new players!`)
|
||||
log("Importing new players...")
|
||||
await sqlite.exec("DROP TABLE IF EXISTS temp")
|
||||
await sqlite.exec("CREATE TABLE temp(Name varchar(128) NOT NULL, Timestamp timestamp NOT NULL)")
|
||||
await sqlite.exec(`INSERT INTO TEMP (Name, Timestamp) SELECT Name, Timestamp FROM (SELECT Name, Timestamp FROM (SELECT * FROM race ORDER BY Timestamp ASC, Name ASC) GROUP BY Name) ${date ? `WHERE Timestamp > datetime('${date}')` : ''}`)
|
||||
log("Imported new players into 'temp'!")
|
||||
|
||||
let addedPlayers = 0
|
||||
let offset = -1
|
||||
while (offset < newPlayerAmount) {
|
||||
await sqlite.each(
|
||||
`SELECT Name, Timestamp FROM temp LIMIT 10000 OFFSET ${offset + 1}`,
|
||||
[],
|
||||
async (err, player) => {
|
||||
if (err) return log(err)
|
||||
if (addedPlayers >= newPlayerAmount) return {}
|
||||
|
||||
Player.create({
|
||||
name: player.Name,
|
||||
firstFinish: player.Timestamp === '0000-00-00 00:00:00' ? new Date('January 1, 1970 00:00:00 UTC') : new Date(`${player.Timestamp}+00:00`)
|
||||
}).then(() => {
|
||||
++addedPlayers
|
||||
log(`Added player ${addedPlayers}/${newPlayerAmount} -> ${player.Name}`)
|
||||
})
|
||||
}
|
||||
)
|
||||
offset += 10000
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
async function sqlite2mongo() {
|
||||
log("Checking for additions to 'ddnet.sqlite'...")
|
||||
|
||||
await checkForMaps()
|
||||
await checkForPlayers()
|
||||
}
|
||||
|
||||
export default sqlite2mongo
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
import betterSqlite3 from 'better-sqlite3'
|
||||
import mongoose from 'mongoose'
|
||||
|
||||
const sqlite = new betterSqlite3('ddnet.sqlite', { readonly: true })
|
||||
|
||||
function initMongo() {
|
||||
mongoose.connect(
|
||||
process.env.MONGO_URI,
|
||||
{
|
||||
useNewUrlParser: true,
|
||||
useUnifiedTopology: true
|
||||
},
|
||||
() => "Connected to mongodb!"
|
||||
)
|
||||
}
|
||||
|
||||
function getAllMaps(player) {
|
||||
const stmt = sqlite.prepare('SELECT Name, Map, Time, Time, Timestamp, Server FROM race WHERE Name = ?')
|
||||
return stmt.all(player)
|
||||
}
|
||||
|
||||
export default {
|
||||
initMongo,
|
||||
getAllMaps
|
||||
}
|
||||
|
|
@ -1,77 +0,0 @@
|
|||
import betterSqlite3 from 'better-sqlite3'
|
||||
import mongoose from 'mongoose'
|
||||
import Level from '../schemas/Level.js'
|
||||
|
||||
function log(string) {
|
||||
console.log(`sqlite2mongo >>> ${string}`)
|
||||
}
|
||||
|
||||
export default async function() {
|
||||
log("Reconstructing the Database!")
|
||||
await mongoose.connect(
|
||||
process.env.MONGO_URI,
|
||||
{
|
||||
useNewUrlParser: true,
|
||||
useUnifiedTopology: true
|
||||
}
|
||||
).then(async () => {
|
||||
log("Connected to mongodb!")
|
||||
|
||||
const sqlite = new betterSqlite3('ddnet.sqlite')
|
||||
log("Initialised sqlite!")
|
||||
|
||||
log("Adding levels (maps)...")
|
||||
const latestMapAddition = (await Level.findOne({}).sort({ "release": "desc" })).release.toISOString().replace(/[TZ]/g, " ").replace(/\.[0-9\s]+/, "")
|
||||
const allMaps = sqlite.prepare("SELECT Map, Server, Points, Stars, Mapper, Timestamp FROM maps WHERE Timestamp > ? ORDER BY Timestamp").iterate(`datetime('${latestMapAddition}')`)
|
||||
const newMapAmount = sqlite.prepare("SELECT count(Map) FROM maps WHERE Timestamp > ? ORDER BY Timestamp").get(`datetime('${latestMapAddition}')`)['count(Map)']
|
||||
let addedMapCount = 0
|
||||
console.log({ allMaps, newMapAmount, addedMapCount })
|
||||
for (let map of allMaps) {
|
||||
Level.create({
|
||||
name: map.Map,
|
||||
mapper: map.Mapper,
|
||||
release: map.Timestamp === '0000-00-00 00:00:00' ? new Date('January 1, 1970 00:00:00 UTC') : new Date(map.Timestamp),
|
||||
category: map.Server,
|
||||
rating: map.Stars,
|
||||
awardPoints: map.Points
|
||||
}).then(() => {
|
||||
++addedMapCount
|
||||
log(`Adding map ${addedMapCount}/${newMapAmount} -> ${map.Map}`)
|
||||
})
|
||||
}
|
||||
|
||||
// log("Adding players...")
|
||||
// let addedPlayerCount = 0
|
||||
// const allPlayers = sqlite.prepare(`SELECT distinct Name FROM race LIMIT 10000`).iterate()
|
||||
// for (const player of allPlayers) {
|
||||
// Player.create({
|
||||
// name: player.name
|
||||
// }).then(() => {
|
||||
// ++addedPlayerCount
|
||||
// log(`Adding player ${addedPlayerCount}/10000 -> ${player.Name}`)
|
||||
// })
|
||||
// }
|
||||
|
||||
// log("Adding finishes (races)...")
|
||||
// const rows = sqlite.prepare("SELECT count(Name) FROM race;").get()['count(Name)']
|
||||
// let addedFinishCount = 0
|
||||
|
||||
// for (let i = 0; i < rows; i += 5000) {
|
||||
// const allFinishes = sqlite.prepare(`SELECT Map, Name, Time, Timestamp, Server FROM race LIMIT 5000 OFFSET ${i + 1}`).all()
|
||||
// for (let finish of allFinishes) {
|
||||
// Level.findOne({ name: finish.Map }).then(map => {
|
||||
// Player.findOne({ name: finish.Name }).then(async player => {
|
||||
// await Finish.create({
|
||||
// map: map._id,
|
||||
// time: finish.Time,
|
||||
// date: finish.Timestamp === '0000-00-00 00:00:00' ? new Date('January 1, 1970 00:00:00 UTC') : new Date(finish.Timestamp),
|
||||
// players: player._id
|
||||
// })
|
||||
// ++addedFinishCount
|
||||
// log(`Adding race ${addedFinishCount}/${rows} -> ${finish.Name} @ ${finish.Map}`)
|
||||
// })
|
||||
// })
|
||||
// }
|
||||
// }
|
||||
})
|
||||
}
|
||||
3
libs/utills/log.js
Normal file
3
libs/utills/log.js
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
export default function initLog(prefix) {
|
||||
return string => console.log(`${prefix} >>> ${string}`)
|
||||
}
|
||||
|
|
@ -17,9 +17,10 @@
|
|||
"author": "BurnyLlama, furo",
|
||||
"license": "WTFPL",
|
||||
"dependencies": {
|
||||
"better-sqlite3": "^7.4.3",
|
||||
"dotenv": "^10.0.0",
|
||||
"express": "^4.17.1",
|
||||
"mongoose": "^6.0.7"
|
||||
"mongoose": "^6.0.7",
|
||||
"sqlite": "^4.0.23",
|
||||
"sqlite3": "^5.0.2"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,8 @@ const Player = new mongoose.Schema({
|
|||
name: String,
|
||||
points: Number,
|
||||
rankPoints: Number,
|
||||
teamPoints: Number
|
||||
teamPoints: Number,
|
||||
firstFinish: Date
|
||||
})
|
||||
|
||||
export default mongoose.model("Player", Player)
|
||||
Loading…
Reference in New Issue
Block a user