From 8e1848fd6d56ca097f904eae702c212621798dc7 Mon Sep 17 00:00:00 2001 From: Leo Singer Date: Wed, 20 Sep 2023 18:19:34 -0400 Subject: [PATCH 1/4] Add robots.txt and sitemap.xml See https://github.com/remix-run/remix/discussions/2912#discussioncomment-7039395 --- app/routes/_seo+/robots[.]txt.ts | 9 +++++++++ app/routes/_seo+/sitemap[.]xml.ts | 12 ++++++++++++ package-lock.json | 16 ++++++++++++++-- package.json | 1 + 4 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 app/routes/_seo+/robots[.]txt.ts create mode 100644 app/routes/_seo+/sitemap[.]xml.ts diff --git a/app/routes/_seo+/robots[.]txt.ts b/app/routes/_seo+/robots[.]txt.ts new file mode 100644 index 00000000..2abd727d --- /dev/null +++ b/app/routes/_seo+/robots[.]txt.ts @@ -0,0 +1,9 @@ +import { generateRobotsTxt } from '@nasa-gcn/remix-seo' +import { type DataFunctionArgs } from '@remix-run/node' + +export function loader({ request }: DataFunctionArgs) { + const origin = new URL(request.url).origin + return generateRobotsTxt([ + { type: 'sitemap', value: `${origin}/sitemap.xml` }, + ]) +} diff --git a/app/routes/_seo+/sitemap[.]xml.ts b/app/routes/_seo+/sitemap[.]xml.ts new file mode 100644 index 00000000..98d61b38 --- /dev/null +++ b/app/routes/_seo+/sitemap[.]xml.ts @@ -0,0 +1,12 @@ +import { generateSitemap } from '@nasa-gcn/remix-seo' +import { routes } from '@remix-run/dev/server-build' +import { type LoaderFunctionArgs } from '@remix-run/node' + +export function loader({ request }: LoaderFunctionArgs) { + return generateSitemap(request, routes, { + siteUrl: new URL(request.url).origin, + headers: { + 'Cache-Control': `public, max-age=${60 * 5}`, + }, + }) +} diff --git a/package-lock.json b/package-lock.json index 1ab884f6..3f4cdb9d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@conform-to/zod": "^0.9.0", "@epic-web/remember": "^1.0.2", "@epic-web/totp": "^1.1.1", + "@nasa-gcn/remix-seo": "^2.0.0", "@paralleldrive/cuid2": "^2.2.2", "@prisma/client": "^5.3.1", "@radix-ui/react-checkbox": "^1.0.4", @@ -2180,6 +2181,18 @@ "node": ">=18" } }, + "node_modules/@nasa-gcn/remix-seo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@nasa-gcn/remix-seo/-/remix-seo-2.0.0.tgz", + "integrity": "sha512-jawoxrjMMbFGgj20d61KblrQNkSFcW2yP7vWWQn2a+eK2J8uYYbw99j2GD7A4XbpQUxy7dg0yvRVsYYJ5uZnLQ==", + "dependencies": { + "lodash": "^4.17.21" + }, + "peerDependencies": { + "@remix-run/react": "^1.0.0 || ^2.0.0", + "@remix-run/server-runtime": "^1.0.0 || ^2.0.0" + } + }, "node_modules/@nicolo-ribaudo/eslint-scope-5-internals": { "version": "5.1.1-v1", "resolved": "https://registry.npmjs.org/@nicolo-ribaudo/eslint-scope-5-internals/-/eslint-scope-5-internals-5.1.1-v1.tgz", @@ -11999,8 +12012,7 @@ "node_modules/lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", - "dev": true + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "node_modules/lodash.camelcase": { "version": "4.3.0", diff --git a/package.json b/package.json index af79ba8b..c7a53691 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "@conform-to/zod": "^0.9.0", "@epic-web/remember": "^1.0.2", "@epic-web/totp": "^1.1.1", + "@nasa-gcn/remix-seo": "^2.0.0", "@paralleldrive/cuid2": "^2.2.2", "@prisma/client": "^5.3.1", "@radix-ui/react-checkbox": "^1.0.4", From 30171c6f532b080a68854d2171a11ff3c19eef16 Mon Sep 17 00:00:00 2001 From: "Kent C. Dodds" Date: Thu, 21 Sep 2023 13:37:49 -0600 Subject: [PATCH 2/4] tidy things up a bit --- app/routes/_seo+/robots[.]txt.ts | 4 ++-- app/routes/_seo+/sitemap[.]xml.ts | 8 +++++--- app/routes/admin+/cache.tsx | 5 +++++ app/routes/settings+/profile.change-email.tsx | 5 ++++- app/routes/settings+/profile.connections.tsx | 7 +++++-- app/routes/settings+/profile.index.tsx | 5 +++++ app/routes/settings+/profile.password.tsx | 5 ++++- app/routes/settings+/profile.password_.create.tsx | 5 ++++- app/routes/settings+/profile.photo.tsx | 5 ++++- app/routes/settings+/profile.tsx | 9 +++++++-- app/routes/settings+/profile.two-factor.disable.tsx | 5 ++++- app/routes/settings+/profile.two-factor.index.tsx | 5 +++++ app/routes/settings+/profile.two-factor.tsx | 5 ++++- app/routes/settings+/profile.two-factor.verify.tsx | 5 ++++- public/robots.txt | 2 -- 15 files changed, 62 insertions(+), 18 deletions(-) delete mode 100644 public/robots.txt diff --git a/app/routes/_seo+/robots[.]txt.ts b/app/routes/_seo+/robots[.]txt.ts index 2abd727d..f252528f 100644 --- a/app/routes/_seo+/robots[.]txt.ts +++ b/app/routes/_seo+/robots[.]txt.ts @@ -1,9 +1,9 @@ import { generateRobotsTxt } from '@nasa-gcn/remix-seo' import { type DataFunctionArgs } from '@remix-run/node' +import { getDomainUrl } from '#app/utils/misc.tsx' export function loader({ request }: DataFunctionArgs) { - const origin = new URL(request.url).origin return generateRobotsTxt([ - { type: 'sitemap', value: `${origin}/sitemap.xml` }, + { type: 'sitemap', value: `${getDomainUrl(request)}/sitemap.xml` }, ]) } diff --git a/app/routes/_seo+/sitemap[.]xml.ts b/app/routes/_seo+/sitemap[.]xml.ts index 98d61b38..d490b0fa 100644 --- a/app/routes/_seo+/sitemap[.]xml.ts +++ b/app/routes/_seo+/sitemap[.]xml.ts @@ -1,10 +1,12 @@ import { generateSitemap } from '@nasa-gcn/remix-seo' +// @ts-expect-error - this does work, though it's not exactly a public API import { routes } from '@remix-run/dev/server-build' -import { type LoaderFunctionArgs } from '@remix-run/node' +import { type DataFunctionArgs } from '@remix-run/node' +import { getDomainUrl } from '#app/utils/misc.tsx' -export function loader({ request }: LoaderFunctionArgs) { +export function loader({ request }: DataFunctionArgs) { return generateSitemap(request, routes, { - siteUrl: new URL(request.url).origin, + siteUrl: getDomainUrl(request), headers: { 'Cache-Control': `public, max-age=${60 * 5}`, }, diff --git a/app/routes/admin+/cache.tsx b/app/routes/admin+/cache.tsx index 403dc63a..7e836c9b 100644 --- a/app/routes/admin+/cache.tsx +++ b/app/routes/admin+/cache.tsx @@ -1,3 +1,4 @@ +import { type SEOHandle } from '@nasa-gcn/remix-seo' import { json, redirect, type DataFunctionArgs } from '@remix-run/node' import { Form, @@ -28,6 +29,10 @@ import { } from '#app/utils/misc.tsx' import { requireUserWithRole } from '#app/utils/permissions.ts' +export const handle: SEOHandle = { + getSitemapEntries: () => null, +} + export async function loader({ request }: DataFunctionArgs) { await requireUserWithRole(request, 'admin') const searchParams = new URL(request.url).searchParams diff --git a/app/routes/settings+/profile.change-email.tsx b/app/routes/settings+/profile.change-email.tsx index 2a94ff99..c1684975 100644 --- a/app/routes/settings+/profile.change-email.tsx +++ b/app/routes/settings+/profile.change-email.tsx @@ -1,5 +1,6 @@ import { conform, useForm } from '@conform-to/react' import { getFieldsetConstraint, parse } from '@conform-to/zod' +import { type SEOHandle } from '@nasa-gcn/remix-seo' import * as E from '@react-email/components' import { json, redirect, type DataFunctionArgs } from '@remix-run/node' import { Form, useActionData, useLoaderData } from '@remix-run/react' @@ -19,9 +20,11 @@ import { invariant, useIsPending } from '#app/utils/misc.tsx' import { redirectWithToast } from '#app/utils/toast.server.ts' import { EmailSchema } from '#app/utils/user-validation.ts' import { verifySessionStorage } from '#app/utils/verification.server.ts' +import { type BreadcrumbHandle } from './profile.tsx' -export const handle = { +export const handle: BreadcrumbHandle & SEOHandle = { breadcrumb: Change Email, + getSitemapEntries: () => null, } const newEmailAddressSessionKey = 'new-email-address' diff --git a/app/routes/settings+/profile.connections.tsx b/app/routes/settings+/profile.connections.tsx index d09565d9..5ad946eb 100644 --- a/app/routes/settings+/profile.connections.tsx +++ b/app/routes/settings+/profile.connections.tsx @@ -1,3 +1,4 @@ +import { type SEOHandle } from '@nasa-gcn/remix-seo' import { json, type DataFunctionArgs, @@ -17,7 +18,7 @@ import { requireUserId } from '#app/utils/auth.server.ts' import { resolveConnectionData } from '#app/utils/connections.server.ts' import { ProviderConnectionForm, - ProviderName, + type ProviderName, ProviderNameSchema, providerIcons, providerNames, @@ -25,9 +26,11 @@ import { import { prisma } from '#app/utils/db.server.ts' import { invariantResponse } from '#app/utils/misc.tsx' import { createToastHeaders } from '#app/utils/toast.server.ts' +import { type BreadcrumbHandle } from './profile.tsx' -export const handle = { +export const handle: BreadcrumbHandle & SEOHandle = { breadcrumb: Connections, + getSitemapEntries: () => null, } async function userCanDeleteConnections(userId: string) { diff --git a/app/routes/settings+/profile.index.tsx b/app/routes/settings+/profile.index.tsx index 92734338..d97b8999 100644 --- a/app/routes/settings+/profile.index.tsx +++ b/app/routes/settings+/profile.index.tsx @@ -1,5 +1,6 @@ import { conform, useForm } from '@conform-to/react' import { getFieldsetConstraint, parse } from '@conform-to/zod' +import { type SEOHandle } from '@nasa-gcn/remix-seo' import { json, redirect, type DataFunctionArgs } from '@remix-run/node' import { Link, useFetcher, useLoaderData } from '@remix-run/react' import { z } from 'zod' @@ -18,6 +19,10 @@ import { sessionStorage } from '#app/utils/session.server.ts' import { NameSchema, UsernameSchema } from '#app/utils/user-validation.ts' import { twoFAVerificationType } from './profile.two-factor.tsx' +export const handle: SEOHandle = { + getSitemapEntries: () => null, +} + const ProfileFormSchema = z.object({ name: NameSchema.optional(), username: UsernameSchema, diff --git a/app/routes/settings+/profile.password.tsx b/app/routes/settings+/profile.password.tsx index 5dcf6cf9..de82a77d 100644 --- a/app/routes/settings+/profile.password.tsx +++ b/app/routes/settings+/profile.password.tsx @@ -1,5 +1,6 @@ import { conform, useForm } from '@conform-to/react' import { getFieldsetConstraint, parse } from '@conform-to/zod' +import { type SEOHandle } from '@nasa-gcn/remix-seo' import { json, redirect, type DataFunctionArgs } from '@remix-run/node' import { Form, Link, useActionData } from '@remix-run/react' import { z } from 'zod' @@ -16,9 +17,11 @@ import { prisma } from '#app/utils/db.server.ts' import { useIsPending } from '#app/utils/misc.tsx' import { redirectWithToast } from '#app/utils/toast.server.ts' import { PasswordSchema } from '#app/utils/user-validation.ts' +import { type BreadcrumbHandle } from './profile.tsx' -export const handle = { +export const handle: BreadcrumbHandle & SEOHandle = { breadcrumb: Password, + getSitemapEntries: () => null, } const ChangePasswordForm = z diff --git a/app/routes/settings+/profile.password_.create.tsx b/app/routes/settings+/profile.password_.create.tsx index e5bc636e..befb2cee 100644 --- a/app/routes/settings+/profile.password_.create.tsx +++ b/app/routes/settings+/profile.password_.create.tsx @@ -1,5 +1,6 @@ import { conform, useForm } from '@conform-to/react' import { getFieldsetConstraint, parse } from '@conform-to/zod' +import { type SEOHandle } from '@nasa-gcn/remix-seo' import { json, redirect, type DataFunctionArgs } from '@remix-run/node' import { Form, Link, useActionData } from '@remix-run/react' import { z } from 'zod' @@ -11,9 +12,11 @@ import { getPasswordHash, requireUserId } from '#app/utils/auth.server.ts' import { prisma } from '#app/utils/db.server.ts' import { useIsPending } from '#app/utils/misc.tsx' import { PasswordSchema } from '#app/utils/user-validation.ts' +import { type BreadcrumbHandle } from './profile.tsx' -export const handle = { +export const handle: BreadcrumbHandle & SEOHandle = { breadcrumb: Password, + getSitemapEntries: () => null, } const CreatePasswordForm = z diff --git a/app/routes/settings+/profile.photo.tsx b/app/routes/settings+/profile.photo.tsx index 5fc95aa6..9b6337f7 100644 --- a/app/routes/settings+/profile.photo.tsx +++ b/app/routes/settings+/profile.photo.tsx @@ -1,5 +1,6 @@ import { conform, useForm } from '@conform-to/react' import { getFieldsetConstraint, parse } from '@conform-to/zod' +import { type SEOHandle } from '@nasa-gcn/remix-seo' import { json, redirect, @@ -23,9 +24,11 @@ import { useDoubleCheck, useIsPending, } from '#app/utils/misc.tsx' +import { type BreadcrumbHandle } from './profile.tsx' -export const handle = { +export const handle: BreadcrumbHandle & SEOHandle = { breadcrumb: Photo, + getSitemapEntries: () => null, } const MAX_SIZE = 1024 * 1024 * 3 // 3MB diff --git a/app/routes/settings+/profile.tsx b/app/routes/settings+/profile.tsx index 09f7df2d..fc4102e7 100644 --- a/app/routes/settings+/profile.tsx +++ b/app/routes/settings+/profile.tsx @@ -1,3 +1,4 @@ +import { type SEOHandle } from '@nasa-gcn/remix-seo' import { json, type DataFunctionArgs } from '@remix-run/node' import { Link, Outlet, useMatches } from '@remix-run/react' import { z } from 'zod' @@ -8,8 +9,12 @@ import { prisma } from '#app/utils/db.server.ts' import { cn, invariantResponse } from '#app/utils/misc.tsx' import { useUser } from '#app/utils/user.ts' -export const handle = { +export const BreadcrumbHandle = z.object({ breadcrumb: z.any() }) +export type BreadcrumbHandle = z.infer + +export const handle: BreadcrumbHandle & SEOHandle = { breadcrumb: Edit Profile, + getSitemapEntries: () => null, } export async function loader({ request }: DataFunctionArgs) { @@ -23,7 +28,7 @@ export async function loader({ request }: DataFunctionArgs) { } const BreadcrumbHandleMatch = z.object({ - handle: z.object({ breadcrumb: z.any() }), + handle: BreadcrumbHandle, }) export default function EditUserProfile() { diff --git a/app/routes/settings+/profile.two-factor.disable.tsx b/app/routes/settings+/profile.two-factor.disable.tsx index 1b627f86..b6e5d255 100644 --- a/app/routes/settings+/profile.two-factor.disable.tsx +++ b/app/routes/settings+/profile.two-factor.disable.tsx @@ -1,3 +1,4 @@ +import { type SEOHandle } from '@nasa-gcn/remix-seo' import { json, type DataFunctionArgs } from '@remix-run/node' import { useFetcher } from '@remix-run/react' import { Icon } from '#app/components/ui/icon.tsx' @@ -7,10 +8,12 @@ import { requireUserId } from '#app/utils/auth.server.ts' import { prisma } from '#app/utils/db.server.ts' import { useDoubleCheck } from '#app/utils/misc.tsx' import { redirectWithToast } from '#app/utils/toast.server.ts' +import { type BreadcrumbHandle } from './profile.tsx' import { twoFAVerificationType } from './profile.two-factor.tsx' -export const handle = { +export const handle: BreadcrumbHandle & SEOHandle = { breadcrumb: Disable, + getSitemapEntries: () => null, } export async function loader({ request }: DataFunctionArgs) { diff --git a/app/routes/settings+/profile.two-factor.index.tsx b/app/routes/settings+/profile.two-factor.index.tsx index 22c92259..bb0cd155 100644 --- a/app/routes/settings+/profile.two-factor.index.tsx +++ b/app/routes/settings+/profile.two-factor.index.tsx @@ -1,4 +1,5 @@ import { generateTOTP } from '@epic-web/totp' +import { SEOHandle } from '@nasa-gcn/remix-seo' import { json, redirect, type DataFunctionArgs } from '@remix-run/node' import { Link, useFetcher, useLoaderData } from '@remix-run/react' import { Icon } from '#app/components/ui/icon.tsx' @@ -8,6 +9,10 @@ import { prisma } from '#app/utils/db.server.ts' import { twoFAVerificationType } from './profile.two-factor.tsx' import { twoFAVerifyVerificationType } from './profile.two-factor.verify.tsx' +export const handle: SEOHandle = { + getSitemapEntries: () => null, +} + export async function loader({ request }: DataFunctionArgs) { const userId = await requireUserId(request) const verification = await prisma.verification.findUnique({ diff --git a/app/routes/settings+/profile.two-factor.tsx b/app/routes/settings+/profile.two-factor.tsx index ae3eb614..7f5dfd19 100644 --- a/app/routes/settings+/profile.two-factor.tsx +++ b/app/routes/settings+/profile.two-factor.tsx @@ -1,9 +1,12 @@ +import { type SEOHandle } from '@nasa-gcn/remix-seo' import { Outlet } from '@remix-run/react' import { Icon } from '#app/components/ui/icon.tsx' import { type VerificationTypes } from '#app/routes/_auth+/verify.tsx' +import { type BreadcrumbHandle } from './profile.tsx' -export const handle = { +export const handle: BreadcrumbHandle & SEOHandle = { breadcrumb: 2FA, + getSitemapEntries: () => null, } export const twoFAVerificationType = '2fa' satisfies VerificationTypes diff --git a/app/routes/settings+/profile.two-factor.verify.tsx b/app/routes/settings+/profile.two-factor.verify.tsx index 71a4f484..04829828 100644 --- a/app/routes/settings+/profile.two-factor.verify.tsx +++ b/app/routes/settings+/profile.two-factor.verify.tsx @@ -1,6 +1,7 @@ import { conform, useForm } from '@conform-to/react' import { getFieldsetConstraint, parse } from '@conform-to/zod' import { getTOTPAuthUri } from '@epic-web/totp' +import { type SEOHandle } from '@nasa-gcn/remix-seo' import { json, redirect, type DataFunctionArgs } from '@remix-run/node' import { Form, @@ -18,10 +19,12 @@ import { requireUserId } from '#app/utils/auth.server.ts' import { prisma } from '#app/utils/db.server.ts' import { getDomainUrl, useIsPending } from '#app/utils/misc.tsx' import { redirectWithToast } from '#app/utils/toast.server.ts' +import { type BreadcrumbHandle } from './profile.tsx' import { twoFAVerificationType } from './profile.two-factor.tsx' -export const handle = { +export const handle: BreadcrumbHandle & SEOHandle = { breadcrumb: Verify, + getSitemapEntries: () => null, } const VerifySchema = z.object({ diff --git a/public/robots.txt b/public/robots.txt deleted file mode 100644 index c2a49f4f..00000000 --- a/public/robots.txt +++ /dev/null @@ -1,2 +0,0 @@ -User-agent: * -Allow: / From 7ec7263cde18545426fa2197765167510ad50d9a Mon Sep 17 00:00:00 2001 From: "Kent C. Dodds" Date: Thu, 21 Sep 2023 13:42:45 -0600 Subject: [PATCH 3/4] add docs --- docs/seo.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 docs/seo.md diff --git a/docs/seo.md b/docs/seo.md new file mode 100644 index 00000000..341009f9 --- /dev/null +++ b/docs/seo.md @@ -0,0 +1,41 @@ +# SEO + +Remix has built-in support for setting up `meta` tags on a per-route basis which +you can read about +[in the Remix Metadata docs](https://remix.run/docs/en/main/route/meta). + +The Epic Stack also has built-in support for `/robots.txt` and `/sitemap.xml` +via [resource routes](https://remix.run/docs/en/main/guides/resource-routes) +using [`@nasa-gcn/remix-seo`](https://github.com/nasa-gcn/remix-seo). By +default, all routes are included in the `sitemap.xml` file, but you can +configure which routes are included using the `handle` export in the route. Only +public-facing pages should be included in the `sitemap.xml` file. + +Here are two quick examples of how to customize the sitemap on a per-route basis +from the `@nasa-gcn/remix-seo` docs: + +```tsx +// routes/blog/$blogslug.tsx + +export const handle: SEOHandle = { + getSitemapEntries: async request => { + const blogs = await db.blog.findMany() + return blogs.map(blog => { + return { route: `/blog/${blog.slug}`, priority: 0.7 } + }) + }, +} +``` + +```tsx +// in your routes/url-that-doesnt-need-sitemap +import { SEOHandle } from '@nasa-gcn/remix-seo' + +export let loader: LoaderFunction = ({ request }) => { + /**/ +} + +export const handle: SEOHandle = { + getSitemapEntries: () => null, +} +``` From 42869370ffa8d24ccbc0f3bd7829bd4a3ec5b69f Mon Sep 17 00:00:00 2001 From: "Kent C. Dodds" Date: Thu, 21 Sep 2023 13:43:11 -0600 Subject: [PATCH 4/4] remove unnecessary example --- docs/examples.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/examples.md b/docs/examples.md index f3313a65..6f6892d3 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -11,10 +11,6 @@ This page links to examples of how to implement some things with the Epic Stack. by [@kentcdodds](https://github.com/kentcdodds): Using client hints to avoid content layout shift with `prefers-reduced-motion` and framer motion animations. -- [Sitemaps](https://github.com/kentcdodds/epic-stack-with-sitemap) by - [@kentcdodds](https://github.com/kentcdodds): Automatically generating a - sitemap and a nice way to handle dynamic routes and customize the sitemap on a - per-route basis. - [Cross-site Request Forgery Protection (CSRF)](https://github.com/kentcdodds/epic-stack-with-csrf) by [@kentcdodds](https://github.com/kentcdodds): An example of the Epic Stack with CSRF protection on forms.