From 0243b5f4917ef3ad3d863eec6a7e5c7a56f2fe64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Losada=20Hern=C3=A1ndez?= Date: Tue, 23 Jan 2024 08:09:17 +0100 Subject: [PATCH 1/5] inital commit --- server/admin.api.js | 64 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 50 insertions(+), 14 deletions(-) diff --git a/server/admin.api.js b/server/admin.api.js index b9b2afbd7..0884ca03f 100644 --- a/server/admin.api.js +++ b/server/admin.api.js @@ -55,15 +55,28 @@ router.post('/admin/cleanup', mw.adminOnly, (req, res)=>{ }); /* Searches for matching edit or share id, also attempts to partial match */ -router.get('/admin/lookup/:id', mw.adminOnly, (req, res, next)=>{ - HomebrewModel.findOne({ $or : [ - { editId: { '$regex': req.params.id, '$options': 'i' } }, - { shareId: { '$regex': req.params.id, '$options': 'i' } }, - ] }).exec((err, brew)=>{ - return res.json(brew); - }); -}); +router.get('/admin/lookup/:id', mw.adminOnly, async (req, res, next) => { + try { + const brew = await HomebrewModel.findOne({ + $or: [ + { editId: { $regex: req.params.id, $options: 'i' } }, + { shareId: { $regex: req.params.id, $options: 'i' } }, + ] + }).exec(); + + if (!brew) { + // No document found + return res.status(404).json({ error: 'Document not found' }); + } + + return res.json(brew); + } catch (error) { + console.error(error); + return res.status(500).json({ error: 'Internal Server Error' }); + } +}); + /* Find 50 brews that aren't compressed yet */ router.get('/admin/finduncompressed', mw.adminOnly, (req, res)=>{ uncompressedBrewQuery.exec((err, objs)=>{ @@ -91,13 +104,36 @@ router.put('/admin/compress/:id', (req, res)=>{ }); }); -router.get('/admin/stats', mw.adminOnly, (req, res)=>{ - HomebrewModel.count({}, (err, count)=>{ - return res.json({ - totalBrews : count - }); - }); +router.get('/admin/stats', mw.adminOnly, async (req, res) => { + try { + const totalBrewsCount = await HomebrewModel.countDocuments({}); + const publishedBrewsCount = await HomebrewModel.countDocuments({ published: true }); + + return res.json({ + totalBrews: totalBrewsCount, + totalPublishedBrews: publishedBrewsCount + }); + } catch (error) { + console.error(error); + return res.status(500).json({ error: 'Internal Server Error' }); + } }); + + + +/* +router.get('/admin/stats', mw.adminOnly, async (req, res) => { + try { + const count = await HomebrewModel.countDocuments({}); + return res.json({ + totalBrews: count + }); + } catch (error) { + console.error(error); + return res.status(500).json({ error: 'Internal Server Error' }); + } +}); +*/ router.get('/admin', mw.adminOnly, (req, res)=>{ templateFn('admin', { From 067a7cd507e84cf0b71d5cf486f2609913fd7b08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Losada=20Hern=C3=A1ndez?= Date: Wed, 24 Jan 2024 22:20:01 +0100 Subject: [PATCH 2/5] admin fixes, the rest( i think) --- server/admin.api.js | 75 ++++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 31 deletions(-) diff --git a/server/admin.api.js b/server/admin.api.js index 0884ca03f..f00b94416 100644 --- a/server/admin.api.js +++ b/server/admin.api.js @@ -40,19 +40,25 @@ const uncompressedBrewQuery = HomebrewModel.find({ 'text' : { '$exists': true } }).lean().limit(10000).select('_id'); -router.get('/admin/cleanup', mw.adminOnly, (req, res)=>{ - junkBrewQuery.exec((err, objs)=>{ - if(err) return res.status(500).send(err); - return res.json({ count: objs.length }); - }); + +router.get('/admin/cleanup', mw.adminOnly, (req, res) => { + junkBrewQuery.exec() + .then((objs) => res.json({ count: objs.length })) + .catch((error) => { + console.error(error); + res.status(500).json({ error: 'Internal Server Error' }); + }); }); /* Removes all empty brews that are older than 3 days and that are shorter than a tweet */ -router.post('/admin/cleanup', mw.adminOnly, (req, res)=>{ - junkBrewQuery.remove().exec((err, objs)=>{ - if(err) return res.status(500).send(err); - return res.json({ count: objs.length }); - }); +router.post('/admin/cleanup', mw.adminOnly, (req, res) => { + junkBrewQuery.remove().exec() + .then((result) => res.json({ count: result.n })) // 'n' represents the number of documents removed + .catch((error) => { + console.error(error); + res.status(500).json({ error: 'Internal Server Error' }); + }); }); + /* Searches for matching edit or share id, also attempts to partial match */ @@ -78,32 +84,39 @@ router.get('/admin/lookup/:id', mw.adminOnly, async (req, res, next) => { }); /* Find 50 brews that aren't compressed yet */ -router.get('/admin/finduncompressed', mw.adminOnly, (req, res)=>{ - uncompressedBrewQuery.exec((err, objs)=>{ - if(err) return res.status(500).send(err); - objs = objs.map((obj)=>{return obj._id;}); - return res.json({ count: objs.length, ids: objs }); - }); +router.get('/admin/finduncompressed', mw.adminOnly, async (req, res) => { + try { + const objs = await uncompressedBrewQuery.exec(); + const ids = objs.map((obj) => obj._id); + return res.json({ count: objs.length, ids: ids }); + } catch (error) { + console.error(error); + return res.status(500).json({ error: 'Internal Server Error' }); + } }); + /* Compresses the "text" field of a brew to binary */ -router.put('/admin/compress/:id', (req, res)=>{ - HomebrewModel.get({ _id: req.params.id }) - .then((brew)=>{ - brew.textBin = zlib.deflateRawSync(brew.text); // Compress brew text to binary before saving - brew.text = undefined; // Delete the non-binary text field since it's not needed anymore - - brew.save((err, obj)=>{ - if(err) throw err; - return res.status(200).send(obj); - }); - }) - .catch((err)=>{ - console.log(err); - return res.status(500).send('Error while saving'); - }); +router.put('/admin/compress/:id', (req, res) => { + HomebrewModel.findOne({ _id: req.params.id }) + .then((brew) => { + if (!brew) { + return res.status(404).send('Brew not found'); + } + + brew.textBin = zlib.deflateRawSync(brew.text); + brew.text = undefined; + + return brew.save(); + }) + .then((obj) => res.status(200).send(obj)) + .catch((err) => { + console.error(err); + res.status(500).send('Error while saving'); + }); }); + router.get('/admin/stats', mw.adminOnly, async (req, res) => { try { const totalBrewsCount = await HomebrewModel.countDocuments({}); From 8b6517eb8d4cbc98c1d8bfa3f19c5a305210c576 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Losada=20Hern=C3=A1ndez?= Date: Wed, 24 Jan 2024 22:49:14 +0100 Subject: [PATCH 3/5] more callback errors, and cloning queries --- server/admin.api.js | 71 ++++++++++++++++++++------------------------- 1 file changed, 32 insertions(+), 39 deletions(-) diff --git a/server/admin.api.js b/server/admin.api.js index f00b94416..97d299957 100644 --- a/server/admin.api.js +++ b/server/admin.api.js @@ -29,39 +29,43 @@ const mw = { /* Search for brews that are older than 3 days and that are shorter than a tweet */ const junkBrewQuery = HomebrewModel.find({ - '$where' : 'this.text.length < 140', - createdAt : { - $lt : Moment().subtract(30, 'days').toDate() - } + text: { $exists: true, $not: { $type: 10 } }, // $type 10 corresponds to null/undefined + createdAt: { + $lt: Moment().subtract(30, 'days').toDate() + } }).limit(100).maxTime(60000); + /* Search for brews that aren't compressed (missing the compressed text field) */ const uncompressedBrewQuery = HomebrewModel.find({ 'text' : { '$exists': true } }).lean().limit(10000).select('_id'); - router.get('/admin/cleanup', mw.adminOnly, (req, res) => { - junkBrewQuery.exec() + // Search for brews that are older than 3 days and shorter than 140 characters + const query = junkBrewQuery.clone(); + + query.exec() .then((objs) => res.json({ count: objs.length })) .catch((error) => { console.error(error); res.status(500).json({ error: 'Internal Server Error' }); }); }); -/* Removes all empty brews that are older than 3 days and that are shorter than a tweet */ + router.post('/admin/cleanup', mw.adminOnly, (req, res) => { - junkBrewQuery.remove().exec() - .then((result) => res.json({ count: result.n })) // 'n' represents the number of documents removed + // Remove all brews that are older than 3 days and shorter than 140 characters + const query = junkBrewQuery.clone(); + + query.remove().exec() + .then((result) => res.json({ count: result.n })) .catch((error) => { console.error(error); res.status(500).json({ error: 'Internal Server Error' }); }); }); - /* Searches for matching edit or share id, also attempts to partial match */ - router.get('/admin/lookup/:id', mw.adminOnly, async (req, res, next) => { try { const brew = await HomebrewModel.findOne({ @@ -84,16 +88,19 @@ router.get('/admin/lookup/:id', mw.adminOnly, async (req, res, next) => { }); /* Find 50 brews that aren't compressed yet */ -router.get('/admin/finduncompressed', mw.adminOnly, async (req, res) => { - try { - const objs = await uncompressedBrewQuery.exec(); - const ids = objs.map((obj) => obj._id); - return res.json({ count: objs.length, ids: ids }); - } catch (error) { - console.error(error); - return res.status(500).json({ error: 'Internal Server Error' }); - } -}); +router.get('/admin/finduncompressed', mw.adminOnly, (req, res) => { + const query = uncompressedBrewQuery.clone(); + + query.exec() + .then((objs) => { + const ids = objs.map((obj) => obj._id); + res.json({ count: ids.length, ids }); + }) + .catch((err) => { + console.error(err); + res.status(500).send(err.message || 'Internal Server Error'); + }); + }); /* Compresses the "text" field of a brew to binary */ @@ -104,8 +111,10 @@ router.put('/admin/compress/:id', (req, res) => { return res.status(404).send('Brew not found'); } - brew.textBin = zlib.deflateRawSync(brew.text); - brew.text = undefined; + if (brew.text) { + brew.textBin = zlib.deflateRawSync(brew.text); + brew.text = undefined; + } return brew.save(); }) @@ -131,22 +140,6 @@ router.get('/admin/stats', mw.adminOnly, async (req, res) => { return res.status(500).json({ error: 'Internal Server Error' }); } }); - - - -/* -router.get('/admin/stats', mw.adminOnly, async (req, res) => { - try { - const count = await HomebrewModel.countDocuments({}); - return res.json({ - totalBrews: count - }); - } catch (error) { - console.error(error); - return res.status(500).json({ error: 'Internal Server Error' }); - } -}); -*/ router.get('/admin', mw.adminOnly, (req, res)=>{ templateFn('admin', { From d2306b70a9d85be5f61193052853e233e14e1844 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Losada=20Hern=C3=A1ndez?= Date: Wed, 24 Jan 2024 23:24:51 +0100 Subject: [PATCH 4/5] quick fix based on review, textBin --- server/admin.api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/admin.api.js b/server/admin.api.js index 97d299957..3ab934c34 100644 --- a/server/admin.api.js +++ b/server/admin.api.js @@ -29,7 +29,7 @@ const mw = { /* Search for brews that are older than 3 days and that are shorter than a tweet */ const junkBrewQuery = HomebrewModel.find({ - text: { $exists: true, $not: { $type: 10 } }, // $type 10 corresponds to null/undefined + '$where' : 'this.textBin.length < 140', createdAt: { $lt: Moment().subtract(30, 'days').toDate() } From e5acbfed3a643d8009cc6b66bd40665be4ccd7a6 Mon Sep 17 00:00:00 2001 From: Trevor Buckner Date: Thu, 25 Jan 2024 16:27:10 -0500 Subject: [PATCH 5/5] Linting and small tweaks --- server/admin.api.js | 185 ++++++++++++++++++++++---------------------- 1 file changed, 91 insertions(+), 94 deletions(-) diff --git a/server/admin.api.js b/server/admin.api.js index 3ab934c34..5363ecc08 100644 --- a/server/admin.api.js +++ b/server/admin.api.js @@ -26,118 +26,115 @@ const mw = { } }; - -/* Search for brews that are older than 3 days and that are shorter than a tweet */ -const junkBrewQuery = HomebrewModel.find({ - '$where' : 'this.textBin.length < 140', - createdAt: { - $lt: Moment().subtract(30, 'days').toDate() - } -}).limit(100).maxTime(60000); - +const junkBrewPipeline = [ + { $match : { + updatedAt : { $lt: Moment().subtract(30, 'days').toDate() }, + lastViewed : { $lt: Moment().subtract(30, 'days').toDate() } + }}, + { $project: { textBinSize: { $binarySize: '$textBin' } } }, + { $match: { textBinSize: { $lt: 140 } } }, + { $limit: 100 } +]; /* Search for brews that aren't compressed (missing the compressed text field) */ const uncompressedBrewQuery = HomebrewModel.find({ 'text' : { '$exists': true } }).lean().limit(10000).select('_id'); -router.get('/admin/cleanup', mw.adminOnly, (req, res) => { - // Search for brews that are older than 3 days and shorter than 140 characters - const query = junkBrewQuery.clone(); - - query.exec() - .then((objs) => res.json({ count: objs.length })) - .catch((error) => { - console.error(error); - res.status(500).json({ error: 'Internal Server Error' }); - }); +// Search for up to 100 brews that have not been viewed or updated in 30 days and are shorter than 140 bytes +router.get('/admin/cleanup', mw.adminOnly, (req, res)=>{ + HomebrewModel.aggregate(junkBrewPipeline).option({ maxTimeMS: 60000 }) + .then((objs)=>res.json({ count: objs.length })) + .catch((error)=>{ + console.error(error); + res.status(500).json({ error: 'Internal Server Error' }); + }); }); - -router.post('/admin/cleanup', mw.adminOnly, (req, res) => { - // Remove all brews that are older than 3 days and shorter than 140 characters - const query = junkBrewQuery.clone(); - - query.remove().exec() - .then((result) => res.json({ count: result.n })) - .catch((error) => { - console.error(error); - res.status(500).json({ error: 'Internal Server Error' }); - }); + +// Delete up to 100 brews that have not been viewed or updated in 30 days and are shorter than 140 bytes +router.post('/admin/cleanup', mw.adminOnly, (req, res)=>{ + HomebrewModel.aggregate(junkBrewPipeline).option({ maxTimeMS: 60000 }) + .then((docs)=>{ + const ids = docs.map((doc)=>doc._id); + return HomebrewModel.deleteMany({ _id: { $in: ids } }); + }).then((result)=>{ + res.json({ count: result.deletedCount }); + }).catch((error)=>{ + console.error(error); + res.status(500).json({ error: 'Internal Server Error' }); + }); }); /* Searches for matching edit or share id, also attempts to partial match */ -router.get('/admin/lookup/:id', mw.adminOnly, async (req, res, next) => { - try { - const brew = await HomebrewModel.findOne({ - $or: [ - { editId: { $regex: req.params.id, $options: 'i' } }, - { shareId: { $regex: req.params.id, $options: 'i' } }, +router.get('/admin/lookup/:id', mw.adminOnly, async (req, res, next)=>{ + HomebrewModel.findOne({ + $or : [ + { editId: { $regex: req.params.id, $options: 'i' } }, + { shareId: { $regex: req.params.id, $options: 'i' } }, ] - }).exec(); - - if (!brew) { - // No document found - return res.status(404).json({ error: 'Document not found' }); - } - - return res.json(brew); - } catch (error) { - console.error(error); - return res.status(500).json({ error: 'Internal Server Error' }); - } -}); - -/* Find 50 brews that aren't compressed yet */ -router.get('/admin/finduncompressed', mw.adminOnly, (req, res) => { - const query = uncompressedBrewQuery.clone(); - - query.exec() - .then((objs) => { - const ids = objs.map((obj) => obj._id); - res.json({ count: ids.length, ids }); - }) - .catch((err) => { + }).exec() + .then((brew)=>{ + if(!brew) // No document found + return res.status(404).json({ error: 'Document not found' }); + else + return res.json(brew); + }) + .catch((err)=>{ console.error(err); - res.status(500).send(err.message || 'Internal Server Error'); - }); - }); - + return res.status(500).json({ error: 'Internal Server Error' }); + }); +}); + +/* Find 50 brews that aren't compressed yet */ +router.get('/admin/finduncompressed', mw.adminOnly, (req, res)=>{ + const query = uncompressedBrewQuery.clone(); + + query.exec() + .then((objs)=>{ + const ids = objs.map((obj)=>obj._id); + res.json({ count: ids.length, ids }); + }) + .catch((err)=>{ + console.error(err); + res.status(500).send(err.message || 'Internal Server Error'); + }); +}); + /* Compresses the "text" field of a brew to binary */ -router.put('/admin/compress/:id', (req, res) => { +router.put('/admin/compress/:id', (req, res)=>{ HomebrewModel.findOne({ _id: req.params.id }) - .then((brew) => { - if (!brew) { - return res.status(404).send('Brew not found'); - } - - if (brew.text) { - brew.textBin = zlib.deflateRawSync(brew.text); - brew.text = undefined; - } - - return brew.save(); - }) - .then((obj) => res.status(200).send(obj)) - .catch((err) => { - console.error(err); - res.status(500).send('Error while saving'); - }); + .then((brew)=>{ + if(!brew) + return res.status(404).send('Brew not found'); + + if(brew.text) { + brew.textBin = brew.textBin || zlib.deflateRawSync(brew.text); //Don't overwrite textBin if exists + brew.text = undefined; + } + + return brew.save(); + }) + .then((obj)=>res.status(200).send(obj)) + .catch((err)=>{ + console.error(err); + res.status(500).send('Error while saving'); + }); }); -router.get('/admin/stats', mw.adminOnly, async (req, res) => { +router.get('/admin/stats', mw.adminOnly, async (req, res)=>{ try { - const totalBrewsCount = await HomebrewModel.countDocuments({}); - const publishedBrewsCount = await HomebrewModel.countDocuments({ published: true }); - - return res.json({ - totalBrews: totalBrewsCount, - totalPublishedBrews: publishedBrewsCount - }); + const totalBrewsCount = await HomebrewModel.countDocuments({}); + const publishedBrewsCount = await HomebrewModel.countDocuments({ published: true }); + + return res.json({ + totalBrews : totalBrewsCount, + totalPublishedBrews : publishedBrewsCount + }); } catch (error) { - console.error(error); - return res.status(500).json({ error: 'Internal Server Error' }); + console.error(error); + return res.status(500).json({ error: 'Internal Server Error' }); } }); @@ -145,8 +142,8 @@ router.get('/admin', mw.adminOnly, (req, res)=>{ templateFn('admin', { url : req.originalUrl }) - .then((page)=>res.send(page)) - .catch((err)=>res.sendStatus(500)); + .then((page)=>res.send(page)) + .catch((err)=>res.sendStatus(500)); }); module.exports = router;