Adds routes to crawl google users for unsubbed brews

patch contents on failed patches clogging logs with pages and pages of text

client/homebrew/pages/editPage/editPage.jsx

@@ -266,7 +266,7 @@ const EditPage = createClass({
 		brew.text           = brew.text.normalize('NFC');
 		this.savedBrew.text = this.savedBrew.text.normalize('NFC');
 		brew.pageCount      = ((brew.renderer=='legacy' ? brew.text.match(/\\page/g) : brew.text.match(/^\\page$/gm)) || []).length + 1;
-		brew.patches        = stringifyPatches(makePatches(this.savedBrew.text, brew.text));
+		brew.patches        = stringifyPatches(makePatches(encodeURI(this.savedBrew.text), encodeURI(brew.text)));
 		brew.hash           = await md5(this.savedBrew.text);
 		//brew.text           = undefined; - Temporary parallel path
 		brew.textBin        = undefined;

scripts/buildHomebrew.js

@@ -161,7 +161,7 @@ fs.emptyDirSync('./build');
 		livereload('./build');     // Install the Chrome extension LiveReload to automatically refresh the browser
 		watchFile('./server.js', { // Restart server when change detected to this file or any nested directory from here
 			ignore : ['./build', './client', './themes'],  // Ignore folders that are not running server code / avoids unneeded restarts
-			ext    : 'js json'                             // Extensions to watch (only .js/.json by default)
+			ext    : 'js'                             // Extensions to watch (only .js/.json by default)
 			//watch : ['./server', './themes'],            // Watch additional folders if needed
 		});
 	}

server/app.js

@@ -147,6 +147,109 @@ app.get('/', (req, res, next)=>{
 	return next();
 });
 
+app.get('/analyze', async (req, res, next) => {
+  const accounts = JSON.parse(fs.readFileSync('accounts.json', 'utf8'));
+  
+	let totalBrewsStubbed = accounts.reduce((sum, account) => {
+		if (account.brewsStubbed) {
+			return sum + account.brewsStubbed;
+		}
+		return sum;
+	}, 0);
+
+	let totalAccountsProcessed = accounts.filter(account => account.fullyProcessed).length;
+	let totalAccountsWithInvalidCredentials = accounts.filter(account => account.invalidCredentials).length;
+
+	console.log(`Total Brews Stubbed: ${totalBrewsStubbed}`);
+	console.log(`Total Accounts Processed: ${totalAccountsProcessed}`);
+	console.log(`Total Accounts with Invalid Credentials: ${totalAccountsWithInvalidCredentials}`);
+});
+
+app.get('/destroy', async (req, res, next) => {
+  const accounts = JSON.parse(fs.readFileSync('accounts.json', 'utf8'));
+  let updated = false;
+
+	function sleep(ms) {
+		return new Promise(resolve => setTimeout(resolve, ms));
+	}
+
+  for (let i = 40000; i < accounts.length; i++) {
+    if (accounts[i].fullyProcessed || accounts[i].invalidCredentials) continue;
+
+    const originalAccount = { ...accounts[i] };
+    const account = accounts[i];
+
+    console.log(`Processing account: ${account.username}`);
+
+    let googleBrews;
+		let auth;
+    try {
+      auth = await GoogleActions.authCheck(account, res);
+      googleBrews = await GoogleActions.listGoogleBrews(auth);
+    } catch (err) {
+      console.error(`Auth error for ${account.username}`);
+      accounts[i] = { ...originalAccount, invalidCredentials: true };
+      updated = true;
+      continue;
+    }
+
+    if (!auth) {
+      accounts[i] = { ...originalAccount, missingAuth: true };
+      updated = true;
+      continue;
+    }
+
+    console.log('Google Brews:', googleBrews.length);
+    if (googleBrews.length === 0) {
+      accounts[i] = { ...originalAccount, fullyProcessed: true, brewsStubbed: 0 };
+      updated = true;
+      continue;
+    }
+
+    const fields = ['googleId', 'title', 'editId', 'shareId'];
+    let brews = await HomebrewModel.getByUser(account.username, true, fields).catch(err => console.error(err));
+    const stubbedEditIds = new Set(brews.map(b => b.editId));
+    googleBrews = googleBrews.filter(b => !stubbedEditIds.has(b.editId));
+		console.log('Unstubbed Google Brews:', googleBrews.length);
+
+    const results = await Promise.all(
+      googleBrews.map(async (brew) => {
+        let brewFromServer = await GoogleActions.getGoogleBrew(auth, brew.googleId, brew.editId, 'edit');
+        splitTextStyleAndMetadata(brewFromServer);
+        brewFromServer.authors = [account.username];
+        api.excludeStubProps(brewFromServer);
+        console.log(`Trying to Stub: ${brewFromServer.title} (${brewFromServer.shareId})`);
+        let saved = await new HomebrewModel(brewFromServer).save().catch(err => console.error(err));
+        if (saved) {
+          console.log(`Saved Stub: ${saved.title} (${saved.shareId})`);
+          return true;
+        }
+        return false;
+      })
+    );
+
+    const stubCount = results.filter(Boolean).length;
+    console.log('Brews stubbed:', stubCount);
+
+    accounts[i] = {
+      ...originalAccount,
+      brewsStubbed: stubCount,
+      fullyProcessed: stubCount === googleBrews.length
+    };
+
+    updated = true;
+
+		sleep(1000);
+  }
+
+  if (updated) {
+    fs.writeFileSync('accounts.json', JSON.stringify(accounts, null, 2), 'utf8');
+    console.log('accounts.json updated');
+  }
+
+  res.send('One account processed');
+});
+
 //Home page Legacy
 app.get('/legacy', (req, res, next)=>{
 	req.brew = {
@@ -383,6 +486,7 @@ app.get('/edit/:id', asyncHandler(getBrew('edit')), asyncHandler(async(req, res,
 		title       : req.brew.title || 'Untitled Brew',
 		description : req.brew.description || 'No description.',
 		image       : req.brew.thumbnail || defaultMetaTags.image,
+		locale      : req.brew.lang,
 		type        : 'article'
 	};
 

server/db.js

@@ -8,7 +8,8 @@
 import Mongoose from 'mongoose';
 
 const getMongoDBURL = (config)=>{
-	return config.get('mongodb_uri') ||
+	console.log('mongodb uri', config.get('MONGODB_URI'));
+	return config.get('MONGODB_URI') ||
            config.get('mongolab_uri') ||
 		   'mongodb://127.0.0.1/homebrewery';  // changed from mongodb://localhost/homebrewery to accommodate versions 16+ of node.
 };

server/homebrew.api.js

@@ -52,13 +52,13 @@ const api = {
 		// ID Validation Checks
 		// Homebrewery ID
 		// Typically 12 characters, but the DB shows a range of 7 to 14 characters
-		if(!id.match(/^[A-Za-z0-9_-]{7,14}$/)){
+		if(!id.match(/^[a-zA-Z0-9-_]{7,14}$/)){
 			throw { name: 'ID Error', message: 'Invalid ID', status: 404, HBErrorCode: '11', brewId: id };
 		}
 		// Google ID
 		// Typically 33 characters, old format is 44 - always starts with a 1
 		// Managed by Google, may change outside of our control, so any length between 33 and 44 is acceptable
-		if(googleId && !googleId.match(/^1(?:[A-Za-z0-9+\/]{32,43})$/)){
+		if(googleId && !googleId.match(/^1(?:[a-zA-Z0-9-_]{32,43})$/)){
 			throw { name: 'Google ID Error', message: 'Invalid ID', status: 404, HBErrorCode: '12', brewId: id };
 		}
 
@@ -375,14 +375,14 @@ const api = {
 		try {
 			const patches = parsePatch(brewFromClient.patches);
 			// Patch to a throwaway variable while parallelizing - we're more concerned with error/no error.
-			const patchedResult = applyPatches(patches, brewFromServer.text, { allowExceedingIndices: true })[0];
+			const patchedResult = decodeURI(applyPatches(patches, encodeURI(brewFromServer.text))[0]);
 			if(patchedResult != brewFromClient.text)
 				throw("Patches did not apply cleanly, text mismatch detected");
 			// brew.text = applyPatches(patches, brewFromServer.text)[0];
 		} catch (err) {
 			//debugTextMismatch(brewFromClient.text, brewFromServer.text, `edit/${brewFromClient.editId}`);
 			console.error('Failed to apply patches:', {
-				patches : brewFromClient.patches,
+				//patches : brewFromClient.patches,
 				brewId  : brewFromClient.editId || 'unknown',
 				error   : err
 			});
@@ -537,4 +537,4 @@ router.delete('/api/:id', checkClientVersion, asyncHandler(api.deleteBrew));
 router.get('/api/remove/:id', checkClientVersion, asyncHandler(api.deleteBrew));
 router.get('/api/theme/:renderer/:id', asyncHandler(api.getThemeBundle));
 
-export default api;
+export default api;