perf: Delayed writing to database

- Implement delayed writing feature to database for improved performance and reduced write frequency
- Update docs

.gitignore

@@ -1,5 +1,6 @@
 # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
 
+count.db
 /.history
 
 # dependencies

Readme.md

@@ -1,8 +1,8 @@
-# Moe-counter
+# Moe-Counter
 
 多种风格可选的萌萌计数器
 
-![Moe-counter](https://count.getloli.com/get/@Moe-counter.github)
+![Moe-Counter](https://count.getloli.com/get/@Moe-counter.github)
 
 <details>
 <summary>More theme</summary>
@@ -26,17 +26,17 @@
 
 ### Install
 
-#### Run on Repl.it
+#### Run on Replit
 
-- Open the url [https://repl.it/@journeyad/Moe-counter](https://repl.it/@journeyad/Moe-counter)
+- Open the url [https://replit.com/@journeyad/Moe-Counter](https://replit.com/@journeyad/Moe-Counter)
 - Just hit the **Fork** button
 - And hit the **Run** button
 
 #### Deploying on your own server
 
 ```shell
-$ git clone https://github.com/journey-ad/Moe-counter.git
+$ git clone https://github.com/journey-ad/Moe-Counter.git
-$ cd Moe-counter
+$ cd Moe-Counter
 $ yarn install
 
 $ yarn start
@@ -51,7 +51,7 @@ app:
   port: 3000
 
 db:
-  type: mongodb # sqlite or mongodb
+  type: sqlite # sqlite or mongodb
 ```
 
 If you use mongodb, you need to specify the environment variable `DB_URL`
@@ -61,7 +61,7 @@ If you use mongodb, you need to specify the environment variable `DB_URL`
 export DB_URL=mongodb+srv://account:passwd@***.***.***.mongodb.net/db_count
 ```
 
-repl.it can use `.env` file, [documentation](https://docs.repl.it/repls/secret-keys)
+replit can use Secrets, [documentation](https://docs.replit.com/programming-ide/storing-sensitive-information-environment-variables)
 
 ```
 DB_URL="mongodb+srv://account:passwd@***.***.***.mongodb.net/db_count"
@@ -69,8 +69,8 @@ DB_URL="mongodb+srv://account:passwd@***.***.***.mongodb.net/db_count"
 
 ## Credits
 
-*   [repl.it](https://repl.it/)
+*   [replit](https://replit.com/)
-*   [A-SOUL](https://www.asoulworld.com/) <sup>(非官方导航站)</sup>
+*   [A-SOUL_Official](https://space.bilibili.com/703007996)
 *   [moebooru](https://github.com/moebooru/moebooru)
 *   rule34.xxx NSFW
 *   gelbooru.com NSFW
@@ -78,4 +78,4 @@ DB_URL="mongodb+srv://account:passwd@***.***.***.mongodb.net/db_count"
 
 ## License
 
-[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fjourney-ad%2FMoe-counter.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fjourney-ad%2FMoe-counter?ref=badge_large)
+[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fjourney-ad%2FMoe-Counter.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fjourney-ad%2FMoe-Counter?ref=badge_large)

config.yml

@@ -2,4 +2,4 @@ app:
   port: 3000
 
 db:
-  type: mongodb # sqlite or mongodb
+  type: sqlite # sqlite or mongodb

db/mongodb.js

@@ -32,8 +32,24 @@ function setNum(name, num) {
           .exec()
 }
 
+function setNumMulti(counters) {
+  const bulkOps = counters.map(obj => {
+    const { name, num } = obj
+    return {
+      updateOne: {
+        filter: { name },
+        update: { name, num },
+        upsert: true
+      }
+    }
+  })
+
+  return Count.bulkWrite(bulkOps, { ordered : false })
+}
+
 module.exports = {
   getNum,
   getAll,
-  setNum
+  setNum,
+  setNumMulti
 }

db/sqlite.js

@@ -1,11 +1,11 @@
 'use strict'
 
 const path = require('path')
-const sqlite3 = require('sqlite3')
+const Database = require('better-sqlite3')
 
-const db = new sqlite3.Database(path.resolve(__dirname, '../count.db'))
+const db = new Database(path.resolve(__dirname, '../count.db'))
 
-db.run(`CREATE TABLE IF NOT EXISTS tb_count (
+db.exec(`CREATE TABLE IF NOT EXISTS tb_count (
     id    INTEGER      PRIMARY KEY AUTOINCREMENT
                        NOT NULL
                        UNIQUE,
@@ -17,44 +17,53 @@ db.run(`CREATE TABLE IF NOT EXISTS tb_count (
 
 function getNum(name) {
   return new Promise((resolve, reject) => {
-    db.get('SELECT `name`, `num` from tb_count WHERE `name` = ?', name, (err, row) => {
+    const stmt = db.prepare('SELECT `name`, `num` from tb_count WHERE `name` = ?')
-      if (err) reject(err)
+    const row = stmt.get(name)
-
+    resolve(row || { name, num: 0 })
-      resolve(row || { name, num: 0 })
-    })
   })
 }
 
 function getAll(name) {
   return new Promise((resolve, reject) => {
-    db.get('SELECT * from tb_count', (err, row) => {
+    const stmt = db.prepare('SELECT * from tb_count')
-      if (err) reject(err)
+    const rows = stmt.all()
-
+    resolve(rows)
-      resolve(row)
-    })
   })
 }
 
 function setNum(name, num) {
   return new Promise((resolve, reject) => {
-    db.run(`INSERT INTO tb_count(\`name\`, \`num\`)
+    db.exec(`INSERT INTO tb_count(\`name\`, \`num\`)
             VALUES($name, $num)
             ON CONFLICT(name) DO
             UPDATE SET \`num\` = $num;`
-      , {
+      ,
-        $name: name,
+      { $name: name, $num: num }
-        $num: num
+    )
-      }
-      , (err, row) => {
-        if (err) reject(err)
 
-        resolve(row)
+    resolve()
-      })
+  })
+}
+
+function setNumMulti(counters) {
+  return new Promise((resolve, reject) => {
+    const stmt = db.prepare(`INSERT INTO tb_count(\`name\`, \`num\`)
+    VALUES($name, $num)
+    ON CONFLICT(name) DO
+    UPDATE SET \`num\` = $num;`)
+
+    const setMany = db.transaction((counters) => {
+      for (const counter of counters) stmt.run(counter)
+    })
+
+    setMany(counters)
+    resolve()
   })
 }
 
 module.exports = {
   getNum,
   getAll,
-  setNum
+  setNum,
-}
+  setNumMulti
+}

index.js

@@ -45,7 +45,7 @@ app.get('/get/@:name', async (req, res) => {
   const renderSvg = themify.getCountImage({ count: data.num, theme, length })
   res.send(renderSvg)
 
-  console.log(data, `theme: ${theme}`)
+  console.log(data, `theme: ${theme}`, `ref: ${req.get('Referrer') || null}`, `ua: ${req.get('User-Agent') || null}`)
 })
 
 // JSON record
@@ -70,16 +70,51 @@ const listener = app.listen(config.app.port || 3000, () => {
   console.log('Your app is listening on port ' + listener.address().port)
 })
 
+let __cache_counter = {}, shouldPush = false
+
+setInterval(() => {
+  shouldPush = true
+}, 1000 * 60);
+
+async function pushDB() {
+  if (!shouldPush) return
+
+  try {
+    shouldPush = false
+    if (Object.keys(__cache_counter).length === 0) return
+
+    console.log("pushDB", __cache_counter)
+
+    const counters = Object.keys(__cache_counter).map(key => {
+      return {
+        name: key,
+        num: __cache_counter[key]
+      }
+    })
+
+    await db.setNumMulti(counters)
+    __cache_counter = {}
+  } catch (error) {
+    console.log("pushDB is error: ", error)
+  }
+}
+
 async function getCountByName(name) {
   const defaultCount = { name, num: 0 }
 
   if (name === 'demo') return { name, num: '0123456789' }
 
   try {
-    const counter = await db.getNum(name) || defaultCount
+    if (!(name in __cache_counter)) {
-    const num = counter.num + 1
+      const counter = await db.getNum(name) || defaultCount
-    db.setNum(counter.name, num)
+      __cache_counter[name] = counter.num + 1
-    return counter
+    } else {
+      __cache_counter[name]++
+    }
+
+    pushDB()
+
+    return { name, num: __cache_counter[name] }
 
   } catch (error) {
     console.log("get count by name is error: ", error)

package.json

@@ -1,5 +1,5 @@
 {
-  "name": "kawaii-counter",
+  "name": "moe-counter",
   "version": "1.0.0",
   "description": "",
   "main": "index.js",
@@ -11,13 +11,13 @@
   "author": "journey-ad",
   "license": "MIT",
   "dependencies": {
+    "better-sqlite3": "^8.1.0",
     "compression": "^1.7.4",
     "config-yml": "^0.10.3",
     "express": "^4.17.1",
     "image-size": "^0.8.3",
     "mime-types": "^2.1.27",
     "mongoose": "^5.9.28",
-    "pug": "^3.0.0",
+    "pug": "^3.0.0"
-    "sqlite3": "^5.0.0"
   }
 }

views/index.pug

@@ -62,7 +62,7 @@ html
       li
         a(href='https://repl.it/', target='_blank', rel='nofollow') repl.it
       li
-        a(href='https://www.asoulworld.com/', target='_blank', title='A-SOUL导航站(非官方)') A-SOUL
+        a(href='https://space.bilibili.com/703007996', target='_blank', title='A-SOUL_Official') A-SOUL
       li
         a(href='https://github.com/moebooru/moebooru', target='_blank', rel='nofollow') moebooru
       li
@@ -109,4 +109,4 @@ html
     iframe(src="https://chat.getloli.com/room/@Moe-counter?title=%E8%90%8C%E8%90%8C%E8%AE%A1%E6%95%B0%E5%99%A8%E7%9A%84%E7%95%99%E8%A8%80%E6%9D%BF", scrolling="no", frameborder="0", height="70%", width="26%", style="position: fixed;top: 2%;right: 5%;")
 
     p.copy
-      a(href='https://github.com/journey-ad/Moe-counter', target='_blank', onclick='_evt_push("click", "normal", "go_github")') source code
+      a(href='https://github.com/journey-ad/Moe-Counter', target='_blank', onclick='_evt_push("click", "normal", "go_github")') source code