sqlite2mongo can now import players.

index.js

@@ -1,6 +1,10 @@
 import express from 'express'
 import dotenv from 'dotenv'
-import sqlite2mongo from './libs/sqlite2mongo.js'
+import sqlite2mongo from './libs/database/sqlite2mongo.js'
+import databaseInit from './libs/database/init.js'
+import initLog from './libs/utills/log.js'
+
+const log = initLog("[ MAIN ]")
 
 // Read the .env file
 dotenv.config()
@@ -17,6 +21,7 @@ Server.get('/', (req, res) =>
         )
 )
 
-sqlite2mongo()
+await databaseInit()
+await sqlite2mongo()
 
-Server.listen(process.env.PORT, () => console.log("Server has started!"))
+Server.listen(process.env.PORT, () => log(`Server started and listening on port ${process.env.PORT}.`))

libs/database/init.js

@@ -0,0 +1,29 @@
+import initLog from '../utills/log.js'
+import mongoose from 'mongoose'
+import sqlite3 from 'sqlite3'
+import { open } from 'sqlite'
+
+const log = initLog("database")
+
+export let sqlite = undefined
+
+async function init() {
+    log("Starting up databases...")
+
+    sqlite = await open({
+        filename: 'ddnet.sqlite',
+        driver: sqlite3.cached.Database
+    })
+    log("Loaded in 'ddnet.sqlite'!")
+
+
+    await mongoose.connect(
+        process.env.MONGO_URI,
+        {
+            useNewUrlParser: true,
+            useUnifiedTopology: true
+        }
+    ).then(() => log("Connected to mongodb!"))
+}
+
+export default init

libs/database/sqlite2mongo.js

@@ -0,0 +1,89 @@
+import initLog from "../utills/log.js"
+import { sqlite } from "./init.js"
+import Level from '../../schemas/Level.js'
+import Player from '../../schemas/Player.js'
+
+const log = initLog("sqlite2mongo")
+
+
+
+async function checkForMaps() {
+    log("Checking for new maps...")
+
+    const latestMap = await Level.findOne({}).sort({ "release": "desc" })
+    const date = latestMap ? latestMap.release.toISOString().replace(/[TZ]/g, " ").replace(/\.[0-9\s]+/, "") : undefined
+    const newMapsAmount = (await sqlite.get(`SELECT count(Map) FROM maps ${date ? `WHERE Timestamp > datetime('${date}')` : ''} ORDER BY Timestamp`))['count(Map)']
+    log(`Found ${newMapsAmount} new maps!`)
+
+    let addedMaps = 0
+    await sqlite.each(
+        `SELECT Map, Server, Points, Stars, Mapper, Timestamp FROM maps ${date ? `WHERE Timestamp > datetime('${date}')` : ''} ORDER BY Timestamp`,
+        [],
+        (err, map) => {
+            if (err) return log(err)
+
+            Level.create({
+                name: map.Map,
+                mapper: map.Mapper,
+                release: map.Timestamp === '0000-00-00 00:00:00' ? new Date('January 1, 1970 00:00:00 UTC') : new Date(`${map.Timestamp}+00:00`),
+                category: map.Server,
+                rating: map.Stars,
+                awardPoints: map.Points
+            }).then(() => {
+                ++addedMaps
+                log(`Added map ${addedMaps}/${newMapsAmount} -> ${map.Map}`)
+            })
+        }
+    )
+}
+
+
+
+async function checkForPlayers() {
+    log("Checking for new players...")
+
+    const latestPlayer = await Player.findOne({}).sort({ "firstFinish": "desc" })
+    const date = latestPlayer ? latestPlayer.firstFinish.toISOString().replace(/[TZ]/g, " ").replace(/\.[0-9\s]+/, "") : undefined
+    const newPlayerAmount = (await sqlite.get(`SELECT count(Name) FROM (SELECT Name, Timestamp FROM (SELECT * FROM race ORDER BY Timestamp ASC, Name ASC) GROUP BY Name) ${date ? `WHERE Timestamp > datetime('${date}')` : ''}`))['count(Name)']
+    if (newPlayerAmount === 0) return log("No new players found...")
+    
+    log(`Found ${newPlayerAmount} new players!`)
+    log("Importing new players...")
+    await sqlite.exec("DROP TABLE IF EXISTS temp")
+    await sqlite.exec("CREATE TABLE temp(Name varchar(128) NOT NULL, Timestamp timestamp NOT NULL)")
+    await sqlite.exec(`INSERT INTO TEMP (Name, Timestamp) SELECT Name, Timestamp FROM (SELECT Name, Timestamp FROM (SELECT * FROM race ORDER BY Timestamp ASC, Name ASC) GROUP BY Name) ${date ? `WHERE Timestamp > datetime('${date}')` : ''}`)
+    log("Imported new players into 'temp'!")
+
+    let addedPlayers = 0
+    let offset = -1
+    while (offset < newPlayerAmount) {
+        await sqlite.each(
+            `SELECT Name, Timestamp FROM temp LIMIT 10000 OFFSET ${offset + 1}`,
+            [],
+            async (err, player) => {
+                if (err) return log(err)
+                if (addedPlayers >= newPlayerAmount) return {}
+
+                Player.create({
+                    name: player.Name,
+                    firstFinish: player.Timestamp === '0000-00-00 00:00:00' ? new Date('January 1, 1970 00:00:00 UTC') : new Date(`${player.Timestamp}+00:00`)
+                }).then(() => {
+                    ++addedPlayers
+                    log(`Added player ${addedPlayers}/${newPlayerAmount} -> ${player.Name}`)
+                })
+            }
+        )
+        offset += 10000
+    }
+}
+
+
+
+async function sqlite2mongo() {
+    log("Checking for additions to 'ddnet.sqlite'...")
+
+    await checkForMaps()
+    await checkForPlayers()
+}
+
+export default sqlite2mongo

libs/databaseHandler.js

@@ -1,25 +0,0 @@
-import betterSqlite3 from 'better-sqlite3'
-import mongoose from 'mongoose'
-
-const sqlite = new betterSqlite3('ddnet.sqlite', { readonly: true })
-
-function initMongo() {
-    mongoose.connect(
-        process.env.MONGO_URI,
-        {
-            useNewUrlParser: true,
-            useUnifiedTopology: true
-        },
-        () => "Connected to mongodb!"
-    )
-}
-
-function getAllMaps(player) {
-    const stmt = sqlite.prepare('SELECT Name, Map, Time, Time, Timestamp, Server FROM race WHERE Name = ?')
-    return stmt.all(player)
-}
-
-export default {
-    initMongo,
-    getAllMaps
-}

libs/sqlite2mongo.js

@@ -1,77 +0,0 @@
-import betterSqlite3 from 'better-sqlite3'
-import mongoose from 'mongoose'
-import Level from '../schemas/Level.js'
-
-function log(string) {
-    console.log(`sqlite2mongo >>> ${string}`)
-}
-
-export default async function() {
-    log("Reconstructing the Database!")
-    await mongoose.connect(
-        process.env.MONGO_URI,
-        {
-            useNewUrlParser: true,
-            useUnifiedTopology: true
-        }
-    ).then(async () => {
-        log("Connected to mongodb!")
-
-        const sqlite = new betterSqlite3('ddnet.sqlite')
-        log("Initialised sqlite!")
-
-        log("Adding levels (maps)...")
-        const latestMapAddition = (await Level.findOne({}).sort({ "release": "desc" })).release.toISOString().replace(/[TZ]/g, " ").replace(/\.[0-9\s]+/, "")
-        const allMaps = sqlite.prepare("SELECT Map, Server, Points, Stars, Mapper, Timestamp FROM maps WHERE Timestamp > ? ORDER BY Timestamp").iterate(`datetime('${latestMapAddition}')`)
-        const newMapAmount = sqlite.prepare("SELECT count(Map) FROM maps WHERE Timestamp > ? ORDER BY Timestamp").get(`datetime('${latestMapAddition}')`)['count(Map)']
-        let addedMapCount = 0
-        console.log({ allMaps, newMapAmount, addedMapCount })
-        for (let map of allMaps) {
-            Level.create({
-                name: map.Map,
-                mapper: map.Mapper,
-                release: map.Timestamp === '0000-00-00 00:00:00' ? new Date('January 1, 1970 00:00:00 UTC') : new Date(map.Timestamp),
-                category: map.Server,
-                rating: map.Stars,
-                awardPoints: map.Points
-            }).then(() => {
-                ++addedMapCount
-                log(`Adding map ${addedMapCount}/${newMapAmount} -> ${map.Map}`)
-            })
-        }
-
-        // log("Adding players...")
-        // let addedPlayerCount = 0
-        // const allPlayers = sqlite.prepare(`SELECT distinct Name FROM race LIMIT 10000`).iterate()
-        // for (const player of allPlayers) {
-        //     Player.create({
-        //         name: player.name
-        //     }).then(() => {
-        //         ++addedPlayerCount
-        //         log(`Adding player ${addedPlayerCount}/10000 -> ${player.Name}`)
-        //     })
-        // }
-
-//     log("Adding finishes (races)...")
-//     const rows = sqlite.prepare("SELECT count(Name) FROM race;").get()['count(Name)']
-//     let addedFinishCount = 0
-
-//     for (let i = 0; i < rows; i += 5000) {
-//         const allFinishes = sqlite.prepare(`SELECT Map, Name, Time, Timestamp, Server FROM race LIMIT 5000 OFFSET ${i + 1}`).all()
-//         for (let finish of allFinishes) {
-//             Level.findOne({ name: finish.Map }).then(map => {
-//                 Player.findOne({ name: finish.Name }).then(async player => {
-//                     await Finish.create({
-//                         map: map._id,
-//                         time: finish.Time,
-//                         date: finish.Timestamp === '0000-00-00 00:00:00' ? new Date('January 1, 1970 00:00:00 UTC') : new Date(finish.Timestamp),
-//                         players: player._id
-//                     })
-//                     ++addedFinishCount
-//                     log(`Adding race ${addedFinishCount}/${rows} -> ${finish.Name} @ ${finish.Map}`)
-//                 })
-//             })
-//         }
-//     }
-    })
-}

libs/utills/log.js

@@ -0,0 +1,3 @@
+export default function initLog(prefix) {
+    return string => console.log(`${prefix} >>> ${string}`)
+}

package.json

@@ -17,9 +17,10 @@
   "author": "BurnyLlama, furo",
   "license": "WTFPL",
   "dependencies": {
-    "better-sqlite3": "^7.4.3",
     "dotenv": "^10.0.0",
     "express": "^4.17.1",
-    "mongoose": "^6.0.7"
+    "mongoose": "^6.0.7",
+    "sqlite": "^4.0.23",
+    "sqlite3": "^5.0.2"
   }
 }

schemas/Player.js

@@ -4,7 +4,8 @@ const Player = new mongoose.Schema({
     name: String,
     points: Number,
     rankPoints: Number,
-    teamPoints: Number
+    teamPoints: Number,
+    firstFinish: Date
 })
 
 export default mongoose.model("Player", Player)