From 3ea3d273a573c4a1a80f4be0b4a40e912e319f96 Mon Sep 17 00:00:00 2001 From: Trevor Buckner Date: Sun, 5 Jan 2020 23:48:50 -0500 Subject: [PATCH 1/6] Add zlib compression to the "text" field. Now, when a file is saved, the original text field is blanked out and the "compressed" binary version is saved instead. Viewing files in Edit page unzips and views them just fine. Does not compress old files unless they are opened and resaved by someone. Have not tested on the "share" or "print" pages yet, but should work. --- server/homebrew.api.js | 4 ++++ server/homebrew.model.js | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/server/homebrew.api.js b/server/homebrew.api.js index c8c0e9122..acb57c68a 100644 --- a/server/homebrew.api.js +++ b/server/homebrew.api.js @@ -1,6 +1,7 @@ const _ = require('lodash'); const HomebrewModel = require('./homebrew.model.js').model; const router = require('express').Router(); +const zlib = require('zlib'); // const getTopBrews = (cb)=>{ // HomebrewModel.find().sort({ views: -1 }).limit(5).exec(function(err, brews) { @@ -47,6 +48,9 @@ router.put('/api/update/:id', (req, res)=>{ HomebrewModel.get({ editId: req.params.id }) .then((brew)=>{ brew = _.merge(brew, req.body); + brew.textBin = zlib.deflateSync(req.body.text); // Compress brew text to binary before saving + brew.text = ''; // Clear out the non-binary text field so its not saved twice + brew.updatedAt = new Date(); if(req.account) brew.authors = _.uniq(_.concat(brew.authors, req.account.username)); diff --git a/server/homebrew.model.js b/server/homebrew.model.js index 587ca37c1..b9cf00319 100644 --- a/server/homebrew.model.js +++ b/server/homebrew.model.js @@ -1,12 +1,14 @@ const mongoose = require('mongoose'); const shortid = require('shortid'); const _ = require('lodash'); +const zlib = require('zlib'); const HomebrewSchema = mongoose.Schema({ shareId : { type: String, default: shortid.generate, index: { unique: true } }, editId : { type: String, default: shortid.generate, index: { unique: true } }, title : { type: String, default: '' }, text : { type: String, default: '' }, + textBin : { type: Buffer }, description : { type: String, default: '' }, tags : { type: String, default: '' }, @@ -51,6 +53,10 @@ HomebrewSchema.statics.get = function(query){ return new Promise((resolve, reject)=>{ Homebrew.find(query, (err, brews)=>{ if(err || !brews.length) return reject('Can not find brew'); + if(!_.isUndefined(brews[0].textBin)) { // Uncompress zipped text field + unzipped = zlib.unzipSync(brews[0].textBin); + brews[0].text = unzipped.toString(); + } return resolve(brews[0]); }); }); From b289cb1003f86aef375aa80458abf4bba3579416 Mon Sep 17 00:00:00 2001 From: Trevor Buckner Date: Sun, 5 Jan 2020 23:59:42 -0500 Subject: [PATCH 2/6] lint --- server/homebrew.api.js | 2 +- server/homebrew.model.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/homebrew.api.js b/server/homebrew.api.js index acb57c68a..d7a74c42d 100644 --- a/server/homebrew.api.js +++ b/server/homebrew.api.js @@ -50,8 +50,8 @@ router.put('/api/update/:id', (req, res)=>{ brew = _.merge(brew, req.body); brew.textBin = zlib.deflateSync(req.body.text); // Compress brew text to binary before saving brew.text = ''; // Clear out the non-binary text field so its not saved twice - brew.updatedAt = new Date(); + if(req.account) brew.authors = _.uniq(_.concat(brew.authors, req.account.username)); brew.markModified('authors'); diff --git a/server/homebrew.model.js b/server/homebrew.model.js index b9cf00319..e4385205a 100644 --- a/server/homebrew.model.js +++ b/server/homebrew.model.js @@ -8,7 +8,7 @@ const HomebrewSchema = mongoose.Schema({ editId : { type: String, default: shortid.generate, index: { unique: true } }, title : { type: String, default: '' }, text : { type: String, default: '' }, - textBin : { type: Buffer }, + textBin : { type: Buffer }, description : { type: String, default: '' }, tags : { type: String, default: '' }, From 1ff3f96f6c8e128798f457c96b8a2baf6b7a10f6 Mon Sep 17 00:00:00 2001 From: Trevor Buckner Date: Fri, 10 Jan 2020 08:59:39 -0500 Subject: [PATCH 3/6] Make compression raw (no wrapper). Otherwise small files are actually larger after compression. --- server/homebrew.api.js | 2 +- server/homebrew.model.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/homebrew.api.js b/server/homebrew.api.js index d7a74c42d..346c6fe00 100644 --- a/server/homebrew.api.js +++ b/server/homebrew.api.js @@ -48,7 +48,7 @@ router.put('/api/update/:id', (req, res)=>{ HomebrewModel.get({ editId: req.params.id }) .then((brew)=>{ brew = _.merge(brew, req.body); - brew.textBin = zlib.deflateSync(req.body.text); // Compress brew text to binary before saving + brew.textBin = zlib.deflateRawSync(req.body.text); // Compress brew text to binary before saving brew.text = ''; // Clear out the non-binary text field so its not saved twice brew.updatedAt = new Date(); diff --git a/server/homebrew.model.js b/server/homebrew.model.js index e4385205a..e8d857982 100644 --- a/server/homebrew.model.js +++ b/server/homebrew.model.js @@ -54,7 +54,7 @@ HomebrewSchema.statics.get = function(query){ Homebrew.find(query, (err, brews)=>{ if(err || !brews.length) return reject('Can not find brew'); if(!_.isUndefined(brews[0].textBin)) { // Uncompress zipped text field - unzipped = zlib.unzipSync(brews[0].textBin); + unzipped = zlib.inflateRawSync(brews[0].textBin); brews[0].text = unzipped.toString(); } return resolve(brews[0]); From 8adc04a5652fb6b853a7efda7c70a81a9db6f59d Mon Sep 17 00:00:00 2001 From: Trevor Buckner Date: Fri, 10 Jan 2020 15:55:21 -0500 Subject: [PATCH 4/6] Make brand new files also save compressed --- server/homebrew.api.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/homebrew.api.js b/server/homebrew.api.js index 346c6fe00..6f137949c 100644 --- a/server/homebrew.api.js +++ b/server/homebrew.api.js @@ -32,9 +32,14 @@ router.post('/api', (req, res)=>{ req.body, { authors: authors } )); + if(!newHomebrew.title){ newHomebrew.title = getGoodBrewTitle(newHomebrew.text); } + + newHomebrew.textBin = zlib.deflateRawSync(newHomebrew.text); // Compress brew text to binary before saving + newHomebrew.text = ''; // Clear out the non-binary text field so its not saved twice + newHomebrew.save((err, obj)=>{ if(err){ console.error(err, err.toString(), err.stack); @@ -48,7 +53,7 @@ router.put('/api/update/:id', (req, res)=>{ HomebrewModel.get({ editId: req.params.id }) .then((brew)=>{ brew = _.merge(brew, req.body); - brew.textBin = zlib.deflateRawSync(req.body.text); // Compress brew text to binary before saving + brew.textBin = zlib.deflateRawSync(req.body.text); // Compress brew text to binary before saving brew.text = ''; // Clear out the non-binary text field so its not saved twice brew.updatedAt = new Date(); From bc81e096867ff13fd4c5ff598835bd12331aec67 Mon Sep 17 00:00:00 2001 From: Trevor Buckner Date: Thu, 23 Jan 2020 09:38:50 -0500 Subject: [PATCH 5/6] Admin script to compress old brews Also fully delete the text field after compressed version is saved instead of leaving just an empty field. --- client/admin/admin.jsx | 3 + client/admin/brewCompress/brewCompress.jsx | 77 +++++++++++++++++++++ client/admin/brewCompress/brewCompress.less | 10 +++ server/admin.api.js | 34 ++++++++- server/homebrew.api.js | 4 +- server/homebrew.model.js | 2 +- 6 files changed, 126 insertions(+), 4 deletions(-) create mode 100644 client/admin/brewCompress/brewCompress.jsx create mode 100644 client/admin/brewCompress/brewCompress.less diff --git a/client/admin/admin.jsx b/client/admin/admin.jsx index 966cb8b63..9708b458a 100644 --- a/client/admin/admin.jsx +++ b/client/admin/admin.jsx @@ -4,6 +4,7 @@ const createClass = require('create-react-class'); const BrewCleanup = require('./brewCleanup/brewCleanup.jsx'); const BrewLookup = require('./brewLookup/brewLookup.jsx'); +const BrewCompress = require ('./brewCompress/brewCompress.jsx'); const Stats = require('./stats/stats.jsx'); const Admin = createClass({ @@ -26,6 +27,8 @@ const Admin = createClass({
+
+ ; } diff --git a/client/admin/brewCompress/brewCompress.jsx b/client/admin/brewCompress/brewCompress.jsx new file mode 100644 index 000000000..e5b37415a --- /dev/null +++ b/client/admin/brewCompress/brewCompress.jsx @@ -0,0 +1,77 @@ +const React = require('react'); +const createClass = require('create-react-class'); +const cx = require('classnames'); + +const request = require('superagent'); + + +const BrewCompress = createClass({ + displayName : 'BrewCompress', + getDefaultProps(){ + return {}; + }, + getInitialState() { + return { + count : 0, + + pending : false, + primed : false, + err : null, + ids : null + }; + }, + prime(){ + this.setState({ pending: true }); + + request.get('/admin/finduncompressed') + .then((res)=>this.setState({ count: res.body.count, primed: true, ids: res.body.ids })) + .catch((err)=>this.setState({ error: err })) + .finally(()=>this.setState({ pending: false })); + }, + cleanup(){ + this.setState({ pending: true }); + this.state.ids.forEach((id) => { + console.log("trying to compress this one:"); + console.log(id); + request.put(`/admin/compress/${id}`) + .catch((err)=>this.setState({ error: err })) + .finally(()=>this.setState({ pending: false, primed: false })); + }); + }, + renderPrimed(){ + if(!this.state.primed) return; + + if(!this.state.count){ + return
No Matching Brews found.
; + } + return
+ + Found {this.state.count} Brews that could be compressed. +
; + }, + render(){ + return
+

Brew Compression

+

Compresses the text in brews to binary

+ + + {this.renderPrimed()} + + {this.state.error + &&
{this.state.error.toString()}
+ } +
; + } +}); + +module.exports = BrewCompress; \ No newline at end of file diff --git a/client/admin/brewCompress/brewCompress.less b/client/admin/brewCompress/brewCompress.less new file mode 100644 index 000000000..2a2bf42ea --- /dev/null +++ b/client/admin/brewCompress/brewCompress.less @@ -0,0 +1,10 @@ +.BrewCompress{ + .removeBox{ + margin-top: 20px; + button{ + background-color: @red; + margin-right: 10px; + } + } + +} \ No newline at end of file diff --git a/server/admin.api.js b/server/admin.api.js index a6486a618..90732fe09 100644 --- a/server/admin.api.js +++ b/server/admin.api.js @@ -3,6 +3,7 @@ const router = require('express').Router(); const Moment = require('moment'); const render = require('vitreum/steps/render'); const templateFn = require('../client/template.js'); +const zlib = require('zlib'); process.env.ADMIN_USER = process.env.ADMIN_USER || 'admin'; process.env.ADMIN_PASS = process.env.ADMIN_PASS || 'password3'; @@ -26,7 +27,7 @@ const mw = { }; -/* Removes all empty brews that are older than 3 days and that are shorter than a tweet */ +/* Search for brews that are older than 3 days and that are shorter than a tweet */ const junkBrewQuery = HomebrewModel.find({ '$where' : 'this.text.length < 140', createdAt : { @@ -34,6 +35,11 @@ const junkBrewQuery = HomebrewModel.find({ } }).limit(100).maxTime(60000); +/* Search for brews that aren't compressed (missing the compressed text field) */ +const uncompressedBrewQuery = HomebrewModel.find({ + 'textBin' : null +}).limit(50).distinct('_id'); + router.get('/admin/cleanup', mw.adminOnly, (req, res)=>{ junkBrewQuery.exec((err, objs)=>{ if(err) return res.status(500).send(err); @@ -58,6 +64,32 @@ router.get('/admin/lookup/:id', mw.adminOnly, (req, res, next)=>{ }); }); +/* Find 50 brews that aren't compressed yet */ +router.get('/admin/finduncompressed', mw.adminOnly, (req, res)=>{ + uncompressedBrewQuery.exec((err, objs)=>{ + if(err) return res.status(500).send(err); + return res.json({ count: objs.length, ids: objs}); + }); +}); + +/* Compresses the "text" field of a brew to binary */ +router.put('/admin/compress/:id', (req, res)=>{ + HomebrewModel.get({ _id: req.params.id }) + .then((brew)=>{ + brew.textBin = zlib.deflateRawSync(brew.text); // Compress brew text to binary before saving + brew.text = undefined; // Delete the non-binary text field since it's not needed anymore + + brew.save((err, obj)=>{ + if(err) throw err; + return res.status(200).send(obj); + }); + }) + .catch((err)=>{ + console.log(err); + return res.status(500).send('Error while saving'); + }); +}); + router.get('/admin/stats', mw.adminOnly, (req, res)=>{ HomebrewModel.count({}, (err, count)=>{ return res.json({ diff --git a/server/homebrew.api.js b/server/homebrew.api.js index 6f137949c..9966865b1 100644 --- a/server/homebrew.api.js +++ b/server/homebrew.api.js @@ -38,7 +38,7 @@ router.post('/api', (req, res)=>{ } newHomebrew.textBin = zlib.deflateRawSync(newHomebrew.text); // Compress brew text to binary before saving - newHomebrew.text = ''; // Clear out the non-binary text field so its not saved twice + newHomebrew.text = undefined; // Delete the non-binary text field since it's not needed anymore newHomebrew.save((err, obj)=>{ if(err){ @@ -54,7 +54,7 @@ router.put('/api/update/:id', (req, res)=>{ .then((brew)=>{ brew = _.merge(brew, req.body); brew.textBin = zlib.deflateRawSync(req.body.text); // Compress brew text to binary before saving - brew.text = ''; // Clear out the non-binary text field so its not saved twice + brew.text = undefined; // Clear out the non-binary text field so its not saved twice brew.updatedAt = new Date(); if(req.account) brew.authors = _.uniq(_.concat(brew.authors, req.account.username)); diff --git a/server/homebrew.model.js b/server/homebrew.model.js index e8d857982..25c7a38e1 100644 --- a/server/homebrew.model.js +++ b/server/homebrew.model.js @@ -53,7 +53,7 @@ HomebrewSchema.statics.get = function(query){ return new Promise((resolve, reject)=>{ Homebrew.find(query, (err, brews)=>{ if(err || !brews.length) return reject('Can not find brew'); - if(!_.isUndefined(brews[0].textBin)) { // Uncompress zipped text field + if(!_.isNil(brews[0].textBin)) { // Uncompress zipped text field unzipped = zlib.inflateRawSync(brews[0].textBin); brews[0].text = unzipped.toString(); } From 2efb24d692cd50bc82ef00caa5541985df411162 Mon Sep 17 00:00:00 2001 From: Trevor Buckner Date: Thu, 23 Jan 2020 09:47:19 -0500 Subject: [PATCH 6/6] Lint --- client/admin/brewCompress/brewCompress.jsx | 4 +--- server/admin.api.js | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/client/admin/brewCompress/brewCompress.jsx b/client/admin/brewCompress/brewCompress.jsx index e5b37415a..a43fd52c2 100644 --- a/client/admin/brewCompress/brewCompress.jsx +++ b/client/admin/brewCompress/brewCompress.jsx @@ -30,9 +30,7 @@ const BrewCompress = createClass({ }, cleanup(){ this.setState({ pending: true }); - this.state.ids.forEach((id) => { - console.log("trying to compress this one:"); - console.log(id); + this.state.ids.forEach((id)=>{ request.put(`/admin/compress/${id}`) .catch((err)=>this.setState({ error: err })) .finally(()=>this.setState({ pending: false, primed: false })); diff --git a/server/admin.api.js b/server/admin.api.js index 90732fe09..99d42f5ec 100644 --- a/server/admin.api.js +++ b/server/admin.api.js @@ -68,7 +68,7 @@ router.get('/admin/lookup/:id', mw.adminOnly, (req, res, next)=>{ router.get('/admin/finduncompressed', mw.adminOnly, (req, res)=>{ uncompressedBrewQuery.exec((err, objs)=>{ if(err) return res.status(500).send(err); - return res.json({ count: objs.length, ids: objs}); + return res.json({ count: objs.length, ids: objs }); }); }); @@ -78,7 +78,7 @@ router.put('/admin/compress/:id', (req, res)=>{ .then((brew)=>{ brew.textBin = zlib.deflateRawSync(brew.text); // Compress brew text to binary before saving brew.text = undefined; // Delete the non-binary text field since it's not needed anymore - + brew.save((err, obj)=>{ if(err) throw err; return res.status(200).send(obj);