From 22ef3cbebcff3bc9956387860c28b38d71ce8b72 Mon Sep 17 00:00:00 2001 From: Trevor Buckner Date: Fri, 11 Jul 2025 16:55:30 +0000 Subject: [PATCH 1/6] Gzip brew object when sending for save update --- client/homebrew/pages/editPage/editPage.jsx | 9 ++++++++- package-lock.json | 6 ++++++ package.json | 1 + server/homebrew.api.js | 14 +++++++++++++- 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/client/homebrew/pages/editPage/editPage.jsx b/client/homebrew/pages/editPage/editPage.jsx index 48d4a0b13..b464e63b6 100644 --- a/client/homebrew/pages/editPage/editPage.jsx +++ b/client/homebrew/pages/editPage/editPage.jsx @@ -5,6 +5,7 @@ const _ = require('lodash'); const createClass = require('create-react-class'); import {makePatches, applyPatches, stringifyPatches, parsePatches} from '@sanity/diff-match-patch'; import { md5 } from 'hash-wasm'; +import { gzipSync, strToU8 } from 'fflate'; import request from '../../utils/request-middleware.js'; const { Meta } = require('vitreum/headtags'); @@ -260,7 +261,7 @@ const EditPage = createClass({ await versionHistoryGarbageCollection().catch(console.error); //Prepare content to send to server - const brew = { ...brewState }; + let brew = { ...brewState }; brew.text = brew.text.normalize('NFC'); this.savedBrew.text = this.savedBrew.text.normalize('NFC'); brew.pageCount = ((brew.renderer=='legacy' ? brew.text.match(/\\page/g) : brew.text.match(/^\\page$/gm)) || []).length + 1; @@ -269,10 +270,16 @@ const EditPage = createClass({ //brew.text = undefined; - Temporary parallel path brew.textBin = undefined; + brew = JSON.stringify(brew); + brew = strToU8(brew); + brew = gzipSync(brew); + const transfer = this.state.saveGoogle == _.isNil(this.state.brew.googleId); const params = `${transfer ? `?${this.state.saveGoogle ? 'saveToGoogle' : 'removeFromGoogle'}=true` : ''}`; const res = await request .put(`/api/update/${brew.editId}${params}`) + .set('Content-Encoding', 'gzip') + .set('Content-Type', 'application/json') .send(brew) .catch((err)=>{ console.log('Error Updating Local Brew'); diff --git a/package-lock.json b/package-lock.json index 8fa722ac7..c6305a529 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30,6 +30,7 @@ "express": "^5.1.0", "express-async-handler": "^1.2.0", "express-static-gzip": "3.0.0", + "fflate": "^0.8.2", "fs-extra": "11.3.0", "hash-wasm": "^4.12.0", "idb-keyval": "^6.2.2", @@ -6650,6 +6651,11 @@ "node": "^12.20 || >= 14.13" } }, + "node_modules/fflate": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.2.tgz", + "integrity": "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==" + }, "node_modules/file-entry-cache": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", diff --git a/package.json b/package.json index 71e892058..cb6ff04a4 100644 --- a/package.json +++ b/package.json @@ -103,6 +103,7 @@ "express": "^5.1.0", "express-async-handler": "^1.2.0", "express-static-gzip": "3.0.0", + "fflate": "^0.8.2", "fs-extra": "11.3.0", "hash-wasm": "^4.12.0", "idb-keyval": "^6.2.2", diff --git a/server/homebrew.api.js b/server/homebrew.api.js index b39f3575f..fb9c78ac8 100644 --- a/server/homebrew.api.js +++ b/server/homebrew.api.js @@ -24,6 +24,18 @@ const isStaticTheme = (renderer, themeName)=>{ return Themes[renderer]?.[themeName] !== undefined; }; +const uncompressBrew = (input, encoding)=> { + try { + const jsonStr = encoding === 'gzip' + ? zlib.gunzipSync(input).toString('utf-8') + : input.toString('utf-8'); + + return JSON.parse(jsonStr); + } catch (err) { + throw new Error('Failed to parse JSON: ' + err.message); + } +} + // const getTopBrews = (cb) => { // HomebrewModel.find().sort({ views: -1 }).limit(5).exec(function(err, brews) { // cb(brews); @@ -337,7 +349,7 @@ const api = { }, updateBrew : async (req, res)=>{ // Initialize brew from request and body, destructure query params, and set the initial value for the after-save method - const brewFromClient = api.excludePropsFromUpdate(req.body); + const brewFromClient = api.excludePropsFromUpdate(uncompressBrew(req.body, req.headers['content-encoding'])); const brewFromServer = req.brew; splitTextStyleAndMetadata(brewFromServer); From fc475b2a7eb788867500b1b23ff5750cbb6ea380 Mon Sep 17 00:00:00 2001 From: Trevor Buckner Date: Sun, 13 Jul 2025 00:52:06 -0400 Subject: [PATCH 2/6] Allow babel to transpile fflate --- scripts/project.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/project.json b/scripts/project.json index 340de077c..7385b674a 100644 --- a/scripts/project.json +++ b/scripts/project.json @@ -28,6 +28,7 @@ "codemirror/addon/hint/show-hint.js", "moment", "superagent", - "@sanity/diff-match-patch" + "@sanity/diff-match-patch", + "fflate" ] } From d3a9d813c99878b8af05ea67da8a4e3512f96b55 Mon Sep 17 00:00:00 2001 From: Trevor Buckner Date: Sun, 13 Jul 2025 00:54:51 -0400 Subject: [PATCH 3/6] Log brew compression size just for testing purposes --- client/homebrew/pages/editPage/editPage.jsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/client/homebrew/pages/editPage/editPage.jsx b/client/homebrew/pages/editPage/editPage.jsx index b464e63b6..de7be7c6d 100644 --- a/client/homebrew/pages/editPage/editPage.jsx +++ b/client/homebrew/pages/editPage/editPage.jsx @@ -270,9 +270,9 @@ const EditPage = createClass({ //brew.text = undefined; - Temporary parallel path brew.textBin = undefined; - brew = JSON.stringify(brew); - brew = strToU8(brew); - brew = gzipSync(brew); + const compressedBrew = gzipSync(strToU8(JSON.stringify(brew))); + console.log('uncompressed size:', (JSON.stringify(brew).length / 1024).toFixed(2), 'KB'); + console.log('compressed size', (compressedBrew.length / 1024).toFixed(2), 'KB'); const transfer = this.state.saveGoogle == _.isNil(this.state.brew.googleId); const params = `${transfer ? `?${this.state.saveGoogle ? 'saveToGoogle' : 'removeFromGoogle'}=true` : ''}`; @@ -280,7 +280,7 @@ const EditPage = createClass({ .put(`/api/update/${brew.editId}${params}`) .set('Content-Encoding', 'gzip') .set('Content-Type', 'application/json') - .send(brew) + .send(compressedBrew) .catch((err)=>{ console.log('Error Updating Local Brew'); this.setState({ error: err }); From 5edea7d0f434d719c7d017fbbfa57430362488a7 Mon Sep 17 00:00:00 2001 From: Trevor Buckner Date: Sun, 13 Jul 2025 00:55:16 -0400 Subject: [PATCH 4/6] Turns out body-parser automatically inflates gzip. Can remove. --- server/homebrew.api.js | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/server/homebrew.api.js b/server/homebrew.api.js index fb9c78ac8..b39f3575f 100644 --- a/server/homebrew.api.js +++ b/server/homebrew.api.js @@ -24,18 +24,6 @@ const isStaticTheme = (renderer, themeName)=>{ return Themes[renderer]?.[themeName] !== undefined; }; -const uncompressBrew = (input, encoding)=> { - try { - const jsonStr = encoding === 'gzip' - ? zlib.gunzipSync(input).toString('utf-8') - : input.toString('utf-8'); - - return JSON.parse(jsonStr); - } catch (err) { - throw new Error('Failed to parse JSON: ' + err.message); - } -} - // const getTopBrews = (cb) => { // HomebrewModel.find().sort({ views: -1 }).limit(5).exec(function(err, brews) { // cb(brews); @@ -349,7 +337,7 @@ const api = { }, updateBrew : async (req, res)=>{ // Initialize brew from request and body, destructure query params, and set the initial value for the after-save method - const brewFromClient = api.excludePropsFromUpdate(uncompressBrew(req.body, req.headers['content-encoding'])); + const brewFromClient = api.excludePropsFromUpdate(req.body); const brewFromServer = req.brew; splitTextStyleAndMetadata(brewFromServer); From 90ee08de42f417f41252ce43f1822e6d6a39e719 Mon Sep 17 00:00:00 2001 From: "G.Ambatte" Date: Mon, 14 Jul 2025 11:06:35 +1200 Subject: [PATCH 5/6] Add text property to test object --- server/homebrew.api.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/homebrew.api.spec.js b/server/homebrew.api.spec.js index cb953f7e5..e6528bb9c 100644 --- a/server/homebrew.api.spec.js +++ b/server/homebrew.api.spec.js @@ -1056,7 +1056,7 @@ brew`); describe('updateBrew', ()=>{ it('should return error on version mismatch', async ()=>{ const brewFromClient = { version: 1 }; - const brewFromServer = { version: 1000 }; + const brewFromServer = { version: 1000, text: '' }; const req = { brew : brewFromServer, From 8432a6e367b5121fb793535b1b5f214a7ce40a5b Mon Sep 17 00:00:00 2001 From: Trevor Buckner Date: Sun, 13 Jul 2025 19:38:08 -0400 Subject: [PATCH 6/6] cleanup --- client/homebrew/pages/editPage/editPage.jsx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/client/homebrew/pages/editPage/editPage.jsx b/client/homebrew/pages/editPage/editPage.jsx index de7be7c6d..9ba54ed0a 100644 --- a/client/homebrew/pages/editPage/editPage.jsx +++ b/client/homebrew/pages/editPage/editPage.jsx @@ -261,7 +261,7 @@ const EditPage = createClass({ await versionHistoryGarbageCollection().catch(console.error); //Prepare content to send to server - let brew = { ...brewState }; + const brew = { ...brewState }; brew.text = brew.text.normalize('NFC'); this.savedBrew.text = this.savedBrew.text.normalize('NFC'); brew.pageCount = ((brew.renderer=='legacy' ? brew.text.match(/\\page/g) : brew.text.match(/^\\page$/gm)) || []).length + 1; @@ -271,8 +271,6 @@ const EditPage = createClass({ brew.textBin = undefined; const compressedBrew = gzipSync(strToU8(JSON.stringify(brew))); - console.log('uncompressed size:', (JSON.stringify(brew).length / 1024).toFixed(2), 'KB'); - console.log('compressed size', (compressedBrew.length / 1024).toFixed(2), 'KB'); const transfer = this.state.saveGoogle == _.isNil(this.state.brew.googleId); const params = `${transfer ? `?${this.state.saveGoogle ? 'saveToGoogle' : 'removeFromGoogle'}=true` : ''}`;