From e88253f36435426f28fff57cd92dd51d7a69e996 Mon Sep 17 00:00:00 2001
From: Trevor Buckner <calculuschild@gmail.com>
Date: Mon, 20 Apr 2020 13:46:18 -0400
Subject: [PATCH] Added robots.txt, nofollow in metatags

Also noindex on /print/ pages
---
 client/homebrew/pages/editPage/editPage.jsx   |  2 +-
 client/homebrew/pages/homePage/homePage.jsx   |  2 +-
 client/homebrew/pages/printPage/printPage.jsx |  2 ++
 client/homebrew/pages/sharePage/sharePage.jsx |  2 +-
 robots.txt                                    |  5 +++++
 server.js                                     | 13 +++++++++----
 6 files changed, 19 insertions(+), 7 deletions(-)
 create mode 100644 robots.txt

diff --git a/client/homebrew/pages/editPage/editPage.jsx b/client/homebrew/pages/editPage/editPage.jsx
index e1c31b516..2bfd44c3e 100644
--- a/client/homebrew/pages/editPage/editPage.jsx
+++ b/client/homebrew/pages/editPage/editPage.jsx
@@ -203,7 +203,7 @@ const EditPage = createClass({
 
 	render : function(){
 		return <div className='editPage page'>
-			<Meta name="robots" content="noindex" />
+			<Meta name='robots' content='noindex, nofollow' />
 			{this.renderNavbar()}
 
 			<div className='content'>
diff --git a/client/homebrew/pages/homePage/homePage.jsx b/client/homebrew/pages/homePage/homePage.jsx
index 6c2a2ee0a..596c3d27b 100644
--- a/client/homebrew/pages/homePage/homePage.jsx
+++ b/client/homebrew/pages/homePage/homePage.jsx
@@ -73,7 +73,7 @@ const HomePage = createClass({
 
 	render : function(){
 		return <div className='homePage page'>
-			<Meta name="google-site-verification" content="NwnAQSSJZzAT7N-p5MY6ydQ7Njm67dtbu73ZSyE5Fy4" />
+			<Meta name='google-site-verification' content='NwnAQSSJZzAT7N-p5MY6ydQ7Njm67dtbu73ZSyE5Fy4' />
 			{this.renderNavbar()}
 
 			<div className='content'>
diff --git a/client/homebrew/pages/printPage/printPage.jsx b/client/homebrew/pages/printPage/printPage.jsx
index 87f901558..b20791b36 100644
--- a/client/homebrew/pages/printPage/printPage.jsx
+++ b/client/homebrew/pages/printPage/printPage.jsx
@@ -2,6 +2,7 @@ const React = require('react');
 const createClass = require('create-react-class');
 const _     = require('lodash');
 const cx    = require('classnames');
+const Meta = require('vitreum/meta');
 const Markdown = require('naturalcrit/markdown.js');
 
 const PrintPage = createClass({
@@ -42,6 +43,7 @@ const PrintPage = createClass({
 
 	render : function(){
 		return <div>
+			<Meta name='robots' content='noindex, nofollow' />
 			{this.renderPages()}
 		</div>;
 	}
diff --git a/client/homebrew/pages/sharePage/sharePage.jsx b/client/homebrew/pages/sharePage/sharePage.jsx
index 81d7c0861..43be64f09 100644
--- a/client/homebrew/pages/sharePage/sharePage.jsx
+++ b/client/homebrew/pages/sharePage/sharePage.jsx
@@ -46,7 +46,7 @@ const SharePage = createClass({
 
 	render : function(){
 		return <div className='sharePage page'>
-			<Meta name="robots" content="noindex" />
+			<Meta name='robots' content='noindex, nofollow' />
 			<Navbar>
 				<Nav.section>
 					<Nav.item className='brewTitle'>{this.props.brew.title}</Nav.item>
diff --git a/robots.txt b/robots.txt
new file mode 100644
index 000000000..384640a98
--- /dev/null
+++ b/robots.txt
@@ -0,0 +1,5 @@
+# Disallow crawling brew edit and share pages
+User-agent: *
+Disallow: /edit/
+Disallow: /share/
+Disallow: /print/
diff --git a/server.js b/server.js
index e573c7dc2..60f6d46bb 100644
--- a/server.js
+++ b/server.js
@@ -44,9 +44,14 @@ const HomebrewModel = require('./server/homebrew.model.js').model;
 const welcomeText = require('fs').readFileSync('./client/homebrew/pages/homePage/welcome_msg.md', 'utf8');
 const changelogText = require('fs').readFileSync('./changelog.md', 'utf8');
 
+String.prototype.replaceAll = function(s, r){return this.split(s).join(r);};
+
+//Robots.txt
+app.get('/robots.txt', (req, res)=>{
+	return res.sendFile(`${__dirname}/robots.txt`);
+});
 
 //Source page
-String.prototype.replaceAll = function(s, r){return this.split(s).join(r);};
 app.get('/source/:id', (req, res)=>{
 	HomebrewModel.get({ shareId: req.params.id })
 		.then((brew)=>{
@@ -59,7 +64,7 @@ app.get('/source/:id', (req, res)=>{
 		});
 });
 
-
+//User Page
 app.get('/user/:username', (req, res, next)=>{
 	const fullAccess = req.account && (req.account.username == req.params.username);
 	HomebrewModel.getByUser(req.params.username, fullAccess)
@@ -72,7 +77,7 @@ app.get('/user/:username', (req, res, next)=>{
 		});
 });
 
-
+//Edit Page
 app.get('/edit/:id', (req, res, next)=>{
 	HomebrewModel.get({ editId: req.params.id })
 		.then((brew)=>{
@@ -115,7 +120,7 @@ app.get('/print/:id', (req, res, next)=>{
 });
 
 
-//Render Page
+//Render the page
 const render = require('vitreum/steps/render');
 const templateFn = require('./client/template.js');
 app.use((req, res)=>{