ddstats-server/libs/database/sqlite2mongo.js

import initLog from "../utills/log.js"
import { sqlite } from "./init.js"
import Level from '../../schemas/Level.js'
import Player from '../../schemas/Player.js'
import Finish from '../../schemas/Finish.js'

const log = initLog("sqlite2mongo")



async function checkForMaps() {
    log("Checking for new maps...")

    const latestMap = await Level.findOne({}).sort({ "release": "desc" })
    const date = latestMap ? latestMap.release.toISOString().replace(/[TZ]/g, " ").replace(/\.[0-9\s]+/, "") : undefined
    const newMapsAmount = (await sqlite.get(`SELECT count(Map) FROM maps ${date ? `WHERE Timestamp > datetime('${date}')` : ''} ORDER BY Timestamp`))['count(Map)']
    log(`Found ${newMapsAmount} new maps!`)

    let addedMaps = 0
    await sqlite.each(
        `SELECT Map, Server, Points, Stars, Mapper, Timestamp FROM maps ${date ? `WHERE Timestamp > datetime('${date}')` : ''} ORDER BY Timestamp`,
        [],
        (err, map) => {
            if (err) return log(err)

            Level.create({
                name: map.Map,
                mapper: map.Mapper,
                release: map.Timestamp === '0000-00-00 00:00:00' ? new Date('January 1, 1970 00:00:00 UTC') : new Date(`${map.Timestamp}+00:00`),
                category: map.Server,
                rating: map.Stars,
                awardPoints: map.Points
            }).then(() => {
                ++addedMaps
                log(`Added map ${addedMaps}/${newMapsAmount} -> ${map.Map}`)
            })
        }
    )
}



async function checkForPlayers() {
    log("Checking for new players...")

    const latestPlayer = await Player.findOne({}).sort({ "firstFinish": "desc" })
    const date = latestPlayer ? latestPlayer.firstFinish.toISOString().replace(/[TZ]/g, " ").replace(/\.[0-9\s]+/, "") : undefined
    const newPlayerAmount = (await sqlite.get(`SELECT count(Name) FROM (SELECT Name, Timestamp FROM (SELECT * FROM race ORDER BY Timestamp ASC, Name ASC) GROUP BY Name) ${date ? `WHERE Timestamp > datetime('${date}')` : ''}`))['count(Name)']
    if (newPlayerAmount === 0) return log("No new players found...")
    
    log(`Found ${newPlayerAmount} new players!`)
    log("Importing new players...")
    await sqlite.exec("DROP TABLE IF EXISTS temp")
    await sqlite.exec("CREATE TABLE temp(Name varchar(128) NOT NULL, Timestamp timestamp NOT NULL)")
    await sqlite.exec(`INSERT INTO TEMP (Name, Timestamp) SELECT Name, Timestamp FROM (SELECT Name, Timestamp FROM (SELECT * FROM race ORDER BY Timestamp ASC, Name ASC) GROUP BY Name) ${date ? `WHERE Timestamp > datetime('${date}')` : ''}`)
    log("Imported new players into 'temp'!")

    let addedPlayers = 0
    let offset = -1
    while (offset < newPlayerAmount) {
        await sqlite.each(
            `SELECT Name, Timestamp FROM temp LIMIT 10000 OFFSET ${offset + 1}`,
            [],
            async (err, player) => {
                if (err) return log(err)
                if (addedPlayers >= newPlayerAmount) return {}

                Player.create({
                    name: player.Name,
                    firstFinish: player.Timestamp === '0000-00-00 00:00:00' ? new Date('January 1, 1970 00:00:00 UTC') : new Date(`${player.Timestamp}+00:00`)
                }).then(() => {
                    ++addedPlayers
                    log(`Added player ${addedPlayers}/${newPlayerAmount} -> ${player.Name}`)
                })
            }
        )
        offset += 10000
    }
}


async function checkForFinishes() {
    log("Checking for new players...")

    const latestFinish = await Finish.findOne({}).sort({ "firstFinish": "desc" })
    const date = latestFinish ? latestFinish.firstFinish.toISOString().replace(/[TZ]/g, " ").replace(/\.[0-9\s]+/, "") : undefined
    const newFinishAmount = (await sqlite.get(`SELECT count(Name) FROM race ${date ? `WHERE Timestamp > datetime('${date}')` : ''}`))['count(Name)']
    if (newFinishAmount === 0) return log("No new finishes found...")

    log(`Found ${newFinishAmount} new players!`)
    log("Importing new players...")
    // await sqlite.exec("DROP TABLE IF EXISTS temp")
    // await sqlite.exec("CREATE TABLE temp(Name varchar(128) NOT NULL, Timestamp timestamp NOT NULL)")
    // await sqlite.exec(`INSERT INTO TEMP (Name, Timestamp) SELECT Name, Timestamp FROM (SELECT Name, Timestamp FROM (SELECT * FROM race ORDER BY Timestamp ASC, Name ASC) GROUP BY Name) ${date ? `WHERE Timestamp > datetime('${date}')` : ''}`)
    // log("Imported new players into 'temp'!")

    // let addedPlayers = 0
    // let offset = -1
    // while (offset < newFinishAmount) {
    //     await sqlite.each(
    //         `SELECT Name, Timestamp FROM temp LIMIT 10000 OFFSET ${offset + 1}`,
    //         [],
    //         async (err, player) => {
    //             if (err) return log(err)
    //             if (addedPlayers >= newFinishAmount) return {}

    //             Player.create({
    //                 name: player.Name,
    //                 firstFinish: player.Timestamp === '0000-00-00 00:00:00' ? new Date('January 1, 1970 00:00:00 UTC') : new Date(`${player.Timestamp}+00:00`)
    //             }).then(() => {
    //                 ++addedPlayers
    //                 log(`Added player ${addedPlayers}/${newFinishAmount} -> ${player.Name}`)
    //             })
    //         }
    //     )
    //     offset += 10000
    // }
}



async function sqlite2mongo() {
    log("Checking for additions to 'ddnet.sqlite'...")

    await checkForMaps()
    await checkForPlayers()
}

export default sqlite2mongo