perf: Delayed writing to database
- Implement delayed writing feature to database for improved performance and reduced write frequency - Update docs
This commit is contained in:
journey-ad
Jad
parent
c3cf12769c
commit
c3f18f8b34
1
.gitignore
vendored
1
.gitignore
vendored
@ -1,5 +1,6 @@
|
||||
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
|
||||
|
||||
count.db
|
||||
/.history
|
||||
|
||||
# dependencies
|
||||
|
||||
22
Readme.md
22
Readme.md
@ -1,8 +1,8 @@
|
||||
# Moe-counter
|
||||
# Moe-Counter
|
||||
|
||||
多种风格可选的萌萌计数器
|
||||
|
||||
|
||||
|
||||
|
||||
<details>
|
||||
<summary>More theme</summary>
|
||||
@ -26,17 +26,17 @@
|
||||
|
||||
### Install
|
||||
|
||||
#### Run on Repl.it
|
||||
#### Run on Replit
|
||||
|
||||
- Open the url [https://repl.it/@journeyad/Moe-counter](https://repl.it/@journeyad/Moe-counter)
|
||||
- Open the url [https://replit.com/@journeyad/Moe-Counter](https://replit.com/@journeyad/Moe-Counter)
|
||||
- Just hit the **Fork** button
|
||||
- And hit the **Run** button
|
||||
|
||||
#### Deploying on your own server
|
||||
|
||||
```shell
|
||||
$ git clone https://github.com/journey-ad/Moe-counter.git
|
||||
$ cd Moe-counter
|
||||
$ git clone https://github.com/journey-ad/Moe-Counter.git
|
||||
$ cd Moe-Counter
|
||||
$ yarn install
|
||||
|
||||
$ yarn start
|
||||
@ -51,7 +51,7 @@ app:
|
||||
port: 3000
|
||||
|
||||
db:
|
||||
type: mongodb # sqlite or mongodb
|
||||
type: sqlite # sqlite or mongodb
|
||||
```
|
||||
|
||||
If you use mongodb, you need to specify the environment variable `DB_URL`
|
||||
@ -61,7 +61,7 @@ If you use mongodb, you need to specify the environment variable `DB_URL`
|
||||
export DB_URL=mongodb+srv://account:passwd@***.***.***.mongodb.net/db_count
|
||||
```
|
||||
|
||||
repl.it can use `.env` file, [documentation](https://docs.repl.it/repls/secret-keys)
|
||||
replit can use Secrets, [documentation](https://docs.replit.com/programming-ide/storing-sensitive-information-environment-variables)
|
||||
|
||||
```
|
||||
DB_URL="mongodb+srv://account:passwd@***.***.***.mongodb.net/db_count"
|
||||
@ -69,8 +69,8 @@ DB_URL="mongodb+srv://account:passwd@***.***.***.mongodb.net/db_count"
|
||||
|
||||
## Credits
|
||||
|
||||
* [repl.it](https://repl.it/)
|
||||
* [A-SOUL](https://www.asoulworld.com/) <sup>(非官方导航站)</sup>
|
||||
* [replit](https://replit.com/)
|
||||
* [A-SOUL_Official](https://space.bilibili.com/703007996)
|
||||
* [moebooru](https://github.com/moebooru/moebooru)
|
||||
* rule34.xxx NSFW
|
||||
* gelbooru.com NSFW
|
||||
@ -78,4 +78,4 @@ DB_URL="mongodb+srv://account:passwd@***.***.***.mongodb.net/db_count"
|
||||
|
||||
## License
|
||||
|
||||
[](https://app.fossa.com/projects/git%2Bgithub.com%2Fjourney-ad%2FMoe-counter?ref=badge_large)
|
||||
[](https://app.fossa.com/projects/git%2Bgithub.com%2Fjourney-ad%2FMoe-Counter?ref=badge_large)
|
||||
|
||||
@ -2,4 +2,4 @@ app:
|
||||
port: 3000
|
||||
|
||||
db:
|
||||
type: mongodb # sqlite or mongodb
|
||||
type: sqlite # sqlite or mongodb
|
||||
|
||||
@ -32,8 +32,24 @@ function setNum(name, num) {
|
||||
.exec()
|
||||
}
|
||||
|
||||
function setNumMulti(counters) {
|
||||
const bulkOps = counters.map(obj => {
|
||||
const { name, num } = obj
|
||||
return {
|
||||
updateOne: {
|
||||
filter: { name },
|
||||
update: { name, num },
|
||||
upsert: true
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
return Count.bulkWrite(bulkOps, { ordered : false })
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
getNum,
|
||||
getAll,
|
||||
setNum
|
||||
setNum,
|
||||
setNumMulti
|
||||
}
|
||||
|
||||
57
db/sqlite.js
57
db/sqlite.js
@ -1,11 +1,11 @@
|
||||
'use strict'
|
||||
|
||||
const path = require('path')
|
||||
const sqlite3 = require('sqlite3')
|
||||
const Database = require('better-sqlite3')
|
||||
|
||||
const db = new sqlite3.Database(path.resolve(__dirname, '../count.db'))
|
||||
const db = new Database(path.resolve(__dirname, '../count.db'))
|
||||
|
||||
db.run(`CREATE TABLE IF NOT EXISTS tb_count (
|
||||
db.exec(`CREATE TABLE IF NOT EXISTS tb_count (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT
|
||||
NOT NULL
|
||||
UNIQUE,
|
||||
@ -17,44 +17,53 @@ db.run(`CREATE TABLE IF NOT EXISTS tb_count (
|
||||
|
||||
function getNum(name) {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.get('SELECT `name`, `num` from tb_count WHERE `name` = ?', name, (err, row) => {
|
||||
if (err) reject(err)
|
||||
|
||||
resolve(row || { name, num: 0 })
|
||||
})
|
||||
const stmt = db.prepare('SELECT `name`, `num` from tb_count WHERE `name` = ?')
|
||||
const row = stmt.get(name)
|
||||
resolve(row || { name, num: 0 })
|
||||
})
|
||||
}
|
||||
|
||||
function getAll(name) {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.get('SELECT * from tb_count', (err, row) => {
|
||||
if (err) reject(err)
|
||||
|
||||
resolve(row)
|
||||
})
|
||||
const stmt = db.prepare('SELECT * from tb_count')
|
||||
const rows = stmt.all()
|
||||
resolve(rows)
|
||||
})
|
||||
}
|
||||
|
||||
function setNum(name, num) {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.run(`INSERT INTO tb_count(\`name\`, \`num\`)
|
||||
db.exec(`INSERT INTO tb_count(\`name\`, \`num\`)
|
||||
VALUES($name, $num)
|
||||
ON CONFLICT(name) DO
|
||||
UPDATE SET \`num\` = $num;`
|
||||
, {
|
||||
$name: name,
|
||||
$num: num
|
||||
}
|
||||
, (err, row) => {
|
||||
if (err) reject(err)
|
||||
,
|
||||
{ $name: name, $num: num }
|
||||
)
|
||||
|
||||
resolve(row)
|
||||
})
|
||||
resolve()
|
||||
})
|
||||
}
|
||||
|
||||
function setNumMulti(counters) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const stmt = db.prepare(`INSERT INTO tb_count(\`name\`, \`num\`)
|
||||
VALUES($name, $num)
|
||||
ON CONFLICT(name) DO
|
||||
UPDATE SET \`num\` = $num;`)
|
||||
|
||||
const setMany = db.transaction((counters) => {
|
||||
for (const counter of counters) stmt.run(counter)
|
||||
})
|
||||
|
||||
setMany(counters)
|
||||
resolve()
|
||||
})
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
getNum,
|
||||
getAll,
|
||||
setNum
|
||||
}
|
||||
setNum,
|
||||
setNumMulti
|
||||
}
|
||||
|
||||
45
index.js
45
index.js
@ -45,7 +45,7 @@ app.get('/get/@:name', async (req, res) => {
|
||||
const renderSvg = themify.getCountImage({ count: data.num, theme, length })
|
||||
res.send(renderSvg)
|
||||
|
||||
console.log(data, `theme: ${theme}`)
|
||||
console.log(data, `theme: ${theme}`, `ref: ${req.get('Referrer') || null}`, `ua: ${req.get('User-Agent') || null}`)
|
||||
})
|
||||
|
||||
// JSON record
|
||||
@ -70,16 +70,51 @@ const listener = app.listen(config.app.port || 3000, () => {
|
||||
console.log('Your app is listening on port ' + listener.address().port)
|
||||
})
|
||||
|
||||
let __cache_counter = {}, shouldPush = false
|
||||
|
||||
setInterval(() => {
|
||||
shouldPush = true
|
||||
}, 1000 * 60);
|
||||
|
||||
async function pushDB() {
|
||||
if (!shouldPush) return
|
||||
|
||||
try {
|
||||
shouldPush = false
|
||||
if (Object.keys(__cache_counter).length === 0) return
|
||||
|
||||
console.log("pushDB", __cache_counter)
|
||||
|
||||
const counters = Object.keys(__cache_counter).map(key => {
|
||||
return {
|
||||
name: key,
|
||||
num: __cache_counter[key]
|
||||
}
|
||||
})
|
||||
|
||||
await db.setNumMulti(counters)
|
||||
__cache_counter = {}
|
||||
} catch (error) {
|
||||
console.log("pushDB is error: ", error)
|
||||
}
|
||||
}
|
||||
|
||||
async function getCountByName(name) {
|
||||
const defaultCount = { name, num: 0 }
|
||||
|
||||
if (name === 'demo') return { name, num: '0123456789' }
|
||||
|
||||
try {
|
||||
const counter = await db.getNum(name) || defaultCount
|
||||
const num = counter.num + 1
|
||||
db.setNum(counter.name, num)
|
||||
return counter
|
||||
if (!(name in __cache_counter)) {
|
||||
const counter = await db.getNum(name) || defaultCount
|
||||
__cache_counter[name] = counter.num + 1
|
||||
} else {
|
||||
__cache_counter[name]++
|
||||
}
|
||||
|
||||
pushDB()
|
||||
|
||||
return { name, num: __cache_counter[name] }
|
||||
|
||||
} catch (error) {
|
||||
console.log("get count by name is error: ", error)
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
{
|
||||
"name": "kawaii-counter",
|
||||
"name": "moe-counter",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
@ -11,13 +11,13 @@
|
||||
"author": "journey-ad",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"better-sqlite3": "^8.1.0",
|
||||
"compression": "^1.7.4",
|
||||
"config-yml": "^0.10.3",
|
||||
"express": "^4.17.1",
|
||||
"image-size": "^0.8.3",
|
||||
"mime-types": "^2.1.27",
|
||||
"mongoose": "^5.9.28",
|
||||
"pug": "^3.0.0",
|
||||
"sqlite3": "^5.0.0"
|
||||
"pug": "^3.0.0"
|
||||
}
|
||||
}
|
||||
|
||||
@ -62,7 +62,7 @@ html
|
||||
li
|
||||
a(href='https://repl.it/', target='_blank', rel='nofollow') repl.it
|
||||
li
|
||||
a(href='https://www.asoulworld.com/', target='_blank', title='A-SOUL导航站(非官方)') A-SOUL
|
||||
a(href='https://space.bilibili.com/703007996', target='_blank', title='A-SOUL_Official') A-SOUL
|
||||
li
|
||||
a(href='https://github.com/moebooru/moebooru', target='_blank', rel='nofollow') moebooru
|
||||
li
|
||||
@ -109,4 +109,4 @@ html
|
||||
iframe(src="https://chat.getloli.com/room/@Moe-counter?title=%E8%90%8C%E8%90%8C%E8%AE%A1%E6%95%B0%E5%99%A8%E7%9A%84%E7%95%99%E8%A8%80%E6%9D%BF", scrolling="no", frameborder="0", height="70%", width="26%", style="position: fixed;top: 2%;right: 5%;")
|
||||
|
||||
p.copy
|
||||
a(href='https://github.com/journey-ad/Moe-counter', target='_blank', onclick='_evt_push("click", "normal", "go_github")') source code
|
||||
a(href='https://github.com/journey-ad/Moe-Counter', target='_blank', onclick='_evt_push("click", "normal", "go_github")') source code
|
||||
Loading…
Reference in New Issue
Block a user