mirror of
https://github.com/naturalcrit/homebrewery.git
synced 2026-01-25 18:23:01 +00:00
Compare commits
4 Commits
crawlGoogl
...
relocateDi
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
253dbb358b | ||
|
|
719edd82c5 | ||
|
|
63d957fdc6 | ||
|
|
7751c0e37b |
@@ -21,7 +21,7 @@ const Account = require('../../navbar/account.navitem.jsx');
|
|||||||
const RecentNavItem = require('../../navbar/recent.navitem.jsx').both;
|
const RecentNavItem = require('../../navbar/recent.navitem.jsx').both;
|
||||||
const VaultNavItem = require('../../navbar/vault.navitem.jsx');
|
const VaultNavItem = require('../../navbar/vault.navitem.jsx');
|
||||||
|
|
||||||
const SplitPane = require('naturalcrit/splitPane/splitPane.jsx');
|
const SplitPane = require('client/components/splitPane/splitPane.jsx');
|
||||||
const Editor = require('../../editor/editor.jsx');
|
const Editor = require('../../editor/editor.jsx');
|
||||||
const BrewRenderer = require('../../brewRenderer/brewRenderer.jsx');
|
const BrewRenderer = require('../../brewRenderer/brewRenderer.jsx');
|
||||||
|
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ const AccountNavItem = require('../../navbar/account.navitem.jsx');
|
|||||||
const ErrorNavItem = require('../../navbar/error-navitem.jsx');
|
const ErrorNavItem = require('../../navbar/error-navitem.jsx');
|
||||||
const { fetchThemeBundle } = require('../../../../shared/helpers.js');
|
const { fetchThemeBundle } = require('../../../../shared/helpers.js');
|
||||||
|
|
||||||
const SplitPane = require('naturalcrit/splitPane/splitPane.jsx');
|
const SplitPane = require('client/components/splitPane/splitPane.jsx');
|
||||||
const Editor = require('../../editor/editor.jsx');
|
const Editor = require('../../editor/editor.jsx');
|
||||||
const BrewRenderer = require('../../brewRenderer/brewRenderer.jsx');
|
const BrewRenderer = require('../../brewRenderer/brewRenderer.jsx');
|
||||||
|
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ const ErrorNavItem = require('../../navbar/error-navitem.jsx');
|
|||||||
const RecentNavItem = require('../../navbar/recent.navitem.jsx').both;
|
const RecentNavItem = require('../../navbar/recent.navitem.jsx').both;
|
||||||
const HelpNavItem = require('../../navbar/help.navitem.jsx');
|
const HelpNavItem = require('../../navbar/help.navitem.jsx');
|
||||||
|
|
||||||
const SplitPane = require('naturalcrit/splitPane/splitPane.jsx');
|
const SplitPane = require('client/components/splitPane/splitPane.jsx');
|
||||||
const Editor = require('../../editor/editor.jsx');
|
const Editor = require('../../editor/editor.jsx');
|
||||||
const BrewRenderer = require('../../brewRenderer/brewRenderer.jsx');
|
const BrewRenderer = require('../../brewRenderer/brewRenderer.jsx');
|
||||||
|
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ const Account = require('../../navbar/account.navitem.jsx');
|
|||||||
const NewBrew = require('../../navbar/newbrew.navitem.jsx');
|
const NewBrew = require('../../navbar/newbrew.navitem.jsx');
|
||||||
const HelpNavItem = require('../../navbar/help.navitem.jsx');
|
const HelpNavItem = require('../../navbar/help.navitem.jsx');
|
||||||
const BrewItem = require('../basePages/listPage/brewItem/brewItem.jsx');
|
const BrewItem = require('../basePages/listPage/brewItem/brewItem.jsx');
|
||||||
const SplitPane = require('../../../../shared/naturalcrit/splitPane/splitPane.jsx');
|
const SplitPane = require('client/components/splitPane/splitPane.jsx');
|
||||||
const ErrorIndex = require('../errorPage/errors/errorIndex.js');
|
const ErrorIndex = require('../errorPage/errors/errorIndex.js');
|
||||||
|
|
||||||
import request from '../../utils/request-middleware.js';
|
import request from '../../utils/request-middleware.js';
|
||||||
|
|||||||
1101
package-lock.json
generated
1101
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -142,13 +142,13 @@
|
|||||||
"eslint-plugin-jest": "^29.0.1",
|
"eslint-plugin-jest": "^29.0.1",
|
||||||
"eslint-plugin-react": "^7.37.5",
|
"eslint-plugin-react": "^7.37.5",
|
||||||
"globals": "^16.3.0",
|
"globals": "^16.3.0",
|
||||||
"jest": "^30.0.4",
|
"jest": "^30.0.5",
|
||||||
"jest-expect-message": "^1.1.3",
|
"jest-expect-message": "^1.1.3",
|
||||||
"jsdom-global": "^3.0.2",
|
"jsdom-global": "^3.0.2",
|
||||||
"postcss-less": "^6.0.0",
|
"postcss-less": "^6.0.0",
|
||||||
"stylelint": "^16.21.1",
|
"stylelint": "^16.22.0",
|
||||||
"stylelint-config-recess-order": "^7.1.0",
|
"stylelint-config-recess-order": "^7.1.0",
|
||||||
"stylelint-config-recommended": "^16.0.0",
|
"stylelint-config-recommended": "^16.0.0",
|
||||||
"supertest": "^7.1.3"
|
"supertest": "^7.1.4"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -161,7 +161,7 @@ fs.emptyDirSync('./build');
|
|||||||
livereload('./build'); // Install the Chrome extension LiveReload to automatically refresh the browser
|
livereload('./build'); // Install the Chrome extension LiveReload to automatically refresh the browser
|
||||||
watchFile('./server.js', { // Restart server when change detected to this file or any nested directory from here
|
watchFile('./server.js', { // Restart server when change detected to this file or any nested directory from here
|
||||||
ignore : ['./build', './client', './themes'], // Ignore folders that are not running server code / avoids unneeded restarts
|
ignore : ['./build', './client', './themes'], // Ignore folders that are not running server code / avoids unneeded restarts
|
||||||
ext : 'js' // Extensions to watch (only .js/.json by default)
|
ext : 'js json' // Extensions to watch (only .js/.json by default)
|
||||||
//watch : ['./server', './themes'], // Watch additional folders if needed
|
//watch : ['./server', './themes'], // Watch additional folders if needed
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
103
server/app.js
103
server/app.js
@@ -147,109 +147,6 @@ app.get('/', (req, res, next)=>{
|
|||||||
return next();
|
return next();
|
||||||
});
|
});
|
||||||
|
|
||||||
app.get('/analyze', async (req, res, next) => {
|
|
||||||
const accounts = JSON.parse(fs.readFileSync('accounts.json', 'utf8'));
|
|
||||||
|
|
||||||
let totalBrewsStubbed = accounts.reduce((sum, account) => {
|
|
||||||
if (account.brewsStubbed) {
|
|
||||||
return sum + account.brewsStubbed;
|
|
||||||
}
|
|
||||||
return sum;
|
|
||||||
}, 0);
|
|
||||||
|
|
||||||
let totalAccountsProcessed = accounts.filter(account => account.fullyProcessed).length;
|
|
||||||
let totalAccountsWithInvalidCredentials = accounts.filter(account => account.invalidCredentials).length;
|
|
||||||
|
|
||||||
console.log(`Total Brews Stubbed: ${totalBrewsStubbed}`);
|
|
||||||
console.log(`Total Accounts Processed: ${totalAccountsProcessed}`);
|
|
||||||
console.log(`Total Accounts with Invalid Credentials: ${totalAccountsWithInvalidCredentials}`);
|
|
||||||
});
|
|
||||||
|
|
||||||
app.get('/destroy', async (req, res, next) => {
|
|
||||||
const accounts = JSON.parse(fs.readFileSync('accounts.json', 'utf8'));
|
|
||||||
let updated = false;
|
|
||||||
|
|
||||||
function sleep(ms) {
|
|
||||||
return new Promise(resolve => setTimeout(resolve, ms));
|
|
||||||
}
|
|
||||||
|
|
||||||
for (let i = 40000; i < accounts.length; i++) {
|
|
||||||
if (accounts[i].fullyProcessed || accounts[i].invalidCredentials) continue;
|
|
||||||
|
|
||||||
const originalAccount = { ...accounts[i] };
|
|
||||||
const account = accounts[i];
|
|
||||||
|
|
||||||
console.log(`Processing account: ${account.username}`);
|
|
||||||
|
|
||||||
let googleBrews;
|
|
||||||
let auth;
|
|
||||||
try {
|
|
||||||
auth = await GoogleActions.authCheck(account, res);
|
|
||||||
googleBrews = await GoogleActions.listGoogleBrews(auth);
|
|
||||||
} catch (err) {
|
|
||||||
console.error(`Auth error for ${account.username}`);
|
|
||||||
accounts[i] = { ...originalAccount, invalidCredentials: true };
|
|
||||||
updated = true;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!auth) {
|
|
||||||
accounts[i] = { ...originalAccount, missingAuth: true };
|
|
||||||
updated = true;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log('Google Brews:', googleBrews.length);
|
|
||||||
if (googleBrews.length === 0) {
|
|
||||||
accounts[i] = { ...originalAccount, fullyProcessed: true, brewsStubbed: 0 };
|
|
||||||
updated = true;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
const fields = ['googleId', 'title', 'editId', 'shareId'];
|
|
||||||
let brews = await HomebrewModel.getByUser(account.username, true, fields).catch(err => console.error(err));
|
|
||||||
const stubbedEditIds = new Set(brews.map(b => b.editId));
|
|
||||||
googleBrews = googleBrews.filter(b => !stubbedEditIds.has(b.editId));
|
|
||||||
console.log('Unstubbed Google Brews:', googleBrews.length);
|
|
||||||
|
|
||||||
const results = await Promise.all(
|
|
||||||
googleBrews.map(async (brew) => {
|
|
||||||
let brewFromServer = await GoogleActions.getGoogleBrew(auth, brew.googleId, brew.editId, 'edit');
|
|
||||||
splitTextStyleAndMetadata(brewFromServer);
|
|
||||||
brewFromServer.authors = [account.username];
|
|
||||||
api.excludeStubProps(brewFromServer);
|
|
||||||
console.log(`Trying to Stub: ${brewFromServer.title} (${brewFromServer.shareId})`);
|
|
||||||
let saved = await new HomebrewModel(brewFromServer).save().catch(err => console.error(err));
|
|
||||||
if (saved) {
|
|
||||||
console.log(`Saved Stub: ${saved.title} (${saved.shareId})`);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
})
|
|
||||||
);
|
|
||||||
|
|
||||||
const stubCount = results.filter(Boolean).length;
|
|
||||||
console.log('Brews stubbed:', stubCount);
|
|
||||||
|
|
||||||
accounts[i] = {
|
|
||||||
...originalAccount,
|
|
||||||
brewsStubbed: stubCount,
|
|
||||||
fullyProcessed: stubCount === googleBrews.length
|
|
||||||
};
|
|
||||||
|
|
||||||
updated = true;
|
|
||||||
|
|
||||||
sleep(1000);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (updated) {
|
|
||||||
fs.writeFileSync('accounts.json', JSON.stringify(accounts, null, 2), 'utf8');
|
|
||||||
console.log('accounts.json updated');
|
|
||||||
}
|
|
||||||
|
|
||||||
res.send('One account processed');
|
|
||||||
});
|
|
||||||
|
|
||||||
//Home page Legacy
|
//Home page Legacy
|
||||||
app.get('/legacy', (req, res, next)=>{
|
app.get('/legacy', (req, res, next)=>{
|
||||||
req.brew = {
|
req.brew = {
|
||||||
|
|||||||
@@ -8,8 +8,7 @@
|
|||||||
import Mongoose from 'mongoose';
|
import Mongoose from 'mongoose';
|
||||||
|
|
||||||
const getMongoDBURL = (config)=>{
|
const getMongoDBURL = (config)=>{
|
||||||
console.log('mongodb uri', config.get('MONGODB_URI'));
|
return config.get('mongodb_uri') ||
|
||||||
return config.get('MONGODB_URI') ||
|
|
||||||
config.get('mongolab_uri') ||
|
config.get('mongolab_uri') ||
|
||||||
'mongodb://127.0.0.1/homebrewery'; // changed from mongodb://localhost/homebrewery to accommodate versions 16+ of node.
|
'mongodb://127.0.0.1/homebrewery'; // changed from mongodb://localhost/homebrewery to accommodate versions 16+ of node.
|
||||||
};
|
};
|
||||||
|
|||||||
Reference in New Issue
Block a user