feat: add crawling settings (#959)

Co-authored-by: Meier Lukas <meierschlumpf@gmail.com>
2026-02-28 01:10:54 +01:00 · 2024-09-06 22:51:18 +02:00
parent d20384dfe0
commit 19cd41a8e5
11 changed files with 227 additions and 62 deletions
--- a/apps/nextjs/src/app/[locale]/layout.tsx
+++ b/apps/nextjs/src/app/[locale]/layout.tsx
@@ -12,6 +12,7 @@ import { ModalProvider } from "@homarr/modals";
 import { Notifications } from "@homarr/notifications";

 import { Analytics } from "~/components/layout/analytics";
+import { SearchEngineOptimization } from "~/components/layout/search-engine-optimization";
 import { JotaiProvider } from "./_client-providers/jotai";
 import { CustomMantineProvider } from "./_client-providers/mantine";
 import { NextInternationalProvider } from "./_client-providers/next-international";
@@ -70,6 +71,7 @@ export default async function Layout(props: { children: React.ReactNode; params:
    <html lang="en" data-mantine-color-scheme={colorScheme} suppressHydrationWarning>
      <head>
        <Analytics />
+        <SearchEngineOptimization />
      </head>
      <body className={["font-sans", fontSans.variable].join(" ")}>
        <StackedProvider>
--- a/apps/nextjs/src/app/[locale]/manage/about/page.tsx
+++ b/apps/nextjs/src/app/[locale]/manage/about/page.tsx
@@ -1,3 +1,4 @@
+import { headers } from "next/headers";
 import Image from "next/image";
 import {
  Accordion,
@@ -17,16 +18,15 @@ import {
  Title,
 } from "@mantine/core";
 import { IconLanguage, IconLibrary, IconUsers } from "@tabler/icons-react";
-import { setStaticParamsLocale } from "next-international/server";

-import { getScopedI18n, getStaticParams } from "@homarr/translation/server";
+import { getScopedI18n } from "@homarr/translation/server";

 import { homarrLogoPath } from "~/components/layout/logo/homarr-logo";
 import { DynamicBreadcrumb } from "~/components/navigation/dynamic-breadcrumb";
 import { createMetaTitle } from "~/metadata";
 import { getPackageAttributesAsync } from "~/versions/package-reader";
-import contributorsData from "../../../../../../../static-data/contributors.json";
-import translatorsData from "../../../../../../../static-data/translators.json";
+import type githubContributorsJson from "../../../../../../../static-data/contributors.json";
+import type crowdinContributorsJson from "../../../../../../../static-data/translators.json";
 import classes from "./about.module.css";

 export async function generateMetadata() {
@@ -37,16 +37,26 @@ export async function generateMetadata() {
  };
 }

-interface PageProps {
-  params: {
-    locale: string;
-  };
-}
+const getHost = () => {
+  if (process.env.HOSTNAME) {
+    return `${process.env.HOSTNAME}:3000`;
+  }

-export default async function AboutPage({ params: { locale } }: PageProps) {
-  setStaticParamsLocale(locale);
+  return headers().get("host");
+};
+
+export default async function AboutPage() {
+  const baseServerUrl = `http://${getHost()}`;
  const t = await getScopedI18n("management.page.about");
  const attributes = await getPackageAttributesAsync();
+  const githubContributors = (await fetch(`${baseServerUrl}/api/about/contributors/github`).then((res) =>
+    res.json(),
+  )) as typeof githubContributorsJson;
+
+  const crowdinContributors = (await fetch(`${baseServerUrl}/api/about/contributors/crowdin`).then((res) =>
+    res.json(),
+  )) as typeof crowdinContributorsJson;
+
  return (
    <div>
      <DynamicBreadcrumb />
@@ -70,14 +80,14 @@ export default async function AboutPage({ params: { locale } }: PageProps) {
              <Text>{t("accordion.contributors.title")}</Text>
              <Text size="sm" c="dimmed">
                {t("accordion.contributors.subtitle", {
-                  count: contributorsData.length,
+                  count: githubContributors.length,
                })}
              </Text>
            </Stack>
          </AccordionControl>
          <AccordionPanel>
            <Flex wrap="wrap" gap="xs">
-              {contributorsData.map((contributor) => (
+              {githubContributors.map((contributor) => (
                <GenericContributorLinkCard
                  key={contributor.login}
                  link={`https://github.com/${contributor.login}`}
@@ -94,14 +104,14 @@ export default async function AboutPage({ params: { locale } }: PageProps) {
              <Text>{t("accordion.translators.title")}</Text>
              <Text size="sm" c="dimmed">
                {t("accordion.translators.subtitle", {
-                  count: translatorsData.length,
+                  count: crowdinContributors.length,
                })}
              </Text>
            </Stack>
          </AccordionControl>
          <AccordionPanel>
            <Flex wrap="wrap" gap="xs">
-              {translatorsData.map((translator) => (
+              {crowdinContributors.map((translator) => (
                <GenericContributorLinkCard
                  key={translator.username}
                  link={`https://crowdin.com/profile/${translator.username}`}
@@ -164,9 +174,3 @@ const GenericContributorLinkCard = ({ name, image, link }: GenericContributorLin
    </AspectRatio>
  );
 };
-
-export function generateStaticParams() {
-  return getStaticParams();
-}
-
-export const dynamic = "force-static";
--- a/apps/nextjs/src/app/[locale]/manage/settings/_components/analytics.settings.tsx
+++ b/apps/nextjs/src/app/[locale]/manage/settings/_components/analytics.settings.tsx
@@ -1,16 +1,14 @@
 "use client";

-import type { ReactNode } from "react";
 import React from "react";
-import type { MantineSpacing } from "@mantine/core";
-import { Card, Group, LoadingOverlay, Stack, Switch, Text, Title, UnstyledButton } from "@mantine/core";
+import { Card, LoadingOverlay, Stack, Title } from "@mantine/core";

 import { clientApi } from "@homarr/api/client";
-import type { UseFormReturnType } from "@homarr/form";
 import { useForm } from "@homarr/form";
 import type { defaultServerSettings } from "@homarr/server-settings";
 import { useScopedI18n } from "@homarr/translation/client";

+import { SwitchSetting } from "~/app/[locale]/manage/settings/_components/setting-switch";
 import { revalidatePathActionAsync } from "~/app/revalidatePathAction";

 interface AnalyticsSettingsProps {
@@ -62,6 +60,7 @@ export const AnalyticsSettings = ({ initialData }: AnalyticsSettingsProps) => {
            ms="xl"
            title={t("integrationData.title")}
            text={t("integrationData.text")}
+            disabled={!form.values.enableGeneral}
          />
          <SwitchSetting
            form={form}
@@ -69,6 +68,7 @@ export const AnalyticsSettings = ({ initialData }: AnalyticsSettingsProps) => {
            ms="xl"
            title={t("widgetData.title")}
            text={t("widgetData.text")}
+            disabled={!form.values.enableGeneral}
          />
          <SwitchSetting
            form={form}
@@ -76,45 +76,10 @@ export const AnalyticsSettings = ({ initialData }: AnalyticsSettingsProps) => {
            ms="xl"
            title={t("usersData.title")}
            text={t("usersData.text")}
+            disabled={!form.values.enableGeneral}
          />
        </Stack>
      </Card>
    </>
  );
 };
-
-const SwitchSetting = ({
-  form,
-  ms,
-  title,
-  text,
-  formKey,
-}: {
-  form: UseFormReturnType<typeof defaultServerSettings.analytics>;
-  formKey: keyof typeof defaultServerSettings.analytics;
-  ms?: MantineSpacing;
-  title: string;
-  text: ReactNode;
-}) => {
-  const disabled = formKey !== "enableGeneral" && !form.values.enableGeneral;
-  const handleClick = React.useCallback(() => {
-    if (disabled) {
-      return;
-    }
-    form.setFieldValue(formKey, !form.values[formKey]);
-  }, [form, formKey, disabled]);
-
-  return (
-    <Group ms={ms} justify="space-between" gap="lg" align="center" wrap="nowrap">
-      <UnstyledButton style={{ flexGrow: 1 }} onClick={handleClick}>
-        <Stack gap={0}>
-          <Text fw="bold">{title}</Text>
-          <Text c="gray.5" fz={{ base: "xs", md: "sm" }}>
-            {text}
-          </Text>
-        </Stack>
-      </UnstyledButton>
-      <Switch disabled={disabled} onClick={handleClick} checked={form.values[formKey] && !disabled} />
-    </Group>
-  );
-};
--- a/apps/nextjs/src/app/[locale]/manage/settings/_components/crawling-and-indexing.settings.tsx
+++ b/apps/nextjs/src/app/[locale]/manage/settings/_components/crawling-and-indexing.settings.tsx
@@ -0,0 +1,68 @@
+"use client";
+
+import React from "react";
+import { Card, LoadingOverlay, Stack, Text, Title } from "@mantine/core";
+
+import { clientApi } from "@homarr/api/client";
+import { useForm } from "@homarr/form";
+import type { defaultServerSettings } from "@homarr/server-settings";
+import { useScopedI18n } from "@homarr/translation/client";
+
+import { SwitchSetting } from "~/app/[locale]/manage/settings/_components/setting-switch";
+import { revalidatePathActionAsync } from "~/app/revalidatePathAction";
+
+interface CrawlingAndIndexingSettingsProps {
+  initialData: typeof defaultServerSettings.crawlingAndIndexing;
+}
+
+export const CrawlingAndIndexingSettings = ({ initialData }: CrawlingAndIndexingSettingsProps) => {
+  const t = useScopedI18n("management.page.settings.section.crawlingAndIndexing");
+  const form = useForm({
+    initialValues: initialData,
+    onValuesChange: (updatedValues, _) => {
+      if (!form.isValid()) {
+        return;
+      }
+
+      void mutateAsync({
+        settingsKey: "crawlingAndIndexing",
+        value: updatedValues,
+      });
+    },
+  });
+
+  const { mutateAsync, isPending } = clientApi.serverSettings.saveSettings.useMutation({
+    onSettled: async () => {
+      await revalidatePathActionAsync("/manage/settings");
+    },
+  });
+
+  return (
+    <>
+      <Title order={2}>{t("title")}</Title>
+
+      <Card pos="relative" withBorder>
+        <Text c={"dimmed"} mb={"lg"}>
+          {t("warning")}
+        </Text>
+        <LoadingOverlay visible={isPending} zIndex={1000} overlayProps={{ radius: "sm", blur: 2 }} />
+        <Stack>
+          <SwitchSetting form={form} formKey="noIndex" title={t("noIndex.title")} text={t("noIndex.text")} />
+          <SwitchSetting form={form} formKey="noFollow" title={t("noFollow.title")} text={t("noFollow.text")} />
+          <SwitchSetting
+            form={form}
+            formKey="noTranslate"
+            title={t("noTranslate.title")}
+            text={t("noTranslate.text")}
+          />
+          <SwitchSetting
+            form={form}
+            formKey="noSiteLinksSearchBox"
+            title={t("noSiteLinksSearchBox.title")}
+            text={t("noSiteLinksSearchBox.text")}
+          />
+        </Stack>
+      </Card>
+    </>
+  );
+};
--- a/apps/nextjs/src/app/[locale]/manage/settings/_components/setting-switch.tsx
+++ b/apps/nextjs/src/app/[locale]/manage/settings/_components/setting-switch.tsx
@@ -0,0 +1,46 @@
+import type { ReactNode } from "react";
+import React from "react";
+import type { MantineSpacing } from "@mantine/core";
+import { Group, Stack, Switch, Text, UnstyledButton } from "@mantine/core";
+
+import type { UseFormReturnType } from "@homarr/form";
+
+export const SwitchSetting = <TFormValue extends Record<string, boolean>>({
+  form,
+  ms,
+  title,
+  text,
+  formKey,
+  disabled,
+}: {
+  form: Omit<UseFormReturnType<TFormValue, () => TFormValue>, "setFieldValue"> & {
+    setFieldValue: (key: keyof TFormValue, value: (previous: boolean) => boolean) => void;
+  };
+  formKey: keyof TFormValue;
+  ms?: MantineSpacing;
+  title: string;
+  text: ReactNode;
+  disabled?: boolean;
+}) => {
+  const handleClick = React.useCallback(() => {
+    if (disabled) {
+      return;
+    }
+
+    form.setFieldValue(formKey, (previous) => !previous);
+  }, [form, formKey, disabled]);
+
+  return (
+    <Group ms={ms} justify="space-between" gap="lg" align="center" wrap="nowrap">
+      <UnstyledButton style={{ flexGrow: 1 }} onClick={handleClick}>
+        <Stack gap={0}>
+          <Text fw="bold">{title}</Text>
+          <Text c="gray.5" fz={{ base: "xs", md: "sm" }}>
+            {text}
+          </Text>
+        </Stack>
+      </UnstyledButton>
+      <Switch disabled={disabled} onClick={handleClick} checked={form.values[formKey] && !disabled} />
+    </Group>
+  );
+};
--- a/apps/nextjs/src/app/[locale]/manage/settings/page.tsx
+++ b/apps/nextjs/src/app/[locale]/manage/settings/page.tsx
@@ -3,6 +3,7 @@ import { Stack, Title } from "@mantine/core";
 import { api } from "@homarr/api/server";
 import { getScopedI18n } from "@homarr/translation/server";

+import { CrawlingAndIndexingSettings } from "~/app/[locale]/manage/settings/_components/crawling-and-indexing.settings";
 import { DynamicBreadcrumb } from "~/components/navigation/dynamic-breadcrumb";
 import { AnalyticsSettings } from "./_components/analytics.settings";

@@ -24,6 +25,7 @@ export default async function SettingsPage() {
      <Stack>
        <Title order={1}>{t("title")}</Title>
        <AnalyticsSettings initialData={serverSettings.analytics} />
+        <CrawlingAndIndexingSettings initialData={serverSettings.crawlingAndIndexing} />
      </Stack>
    </>
  );
--- a/apps/nextjs/src/app/api/about/contributors/crowdin/route.ts
+++ b/apps/nextjs/src/app/api/about/contributors/crowdin/route.ts
@@ -0,0 +1,9 @@
+import { NextResponse } from "next/server";
+
+import crowdinContributors from "../../../../../../../../static-data/translators.json";
+
+export const GET = () => {
+  return NextResponse.json(crowdinContributors);
+};
+
+export const dynamic = "force-static";
--- a/apps/nextjs/src/app/api/about/contributors/github/route.ts
+++ b/apps/nextjs/src/app/api/about/contributors/github/route.ts
@@ -0,0 +1,9 @@
+import { NextResponse } from "next/server";
+
+import githubContributors from "../../../../../../../../static-data/contributors.json";
+
+export const GET = () => {
+  return NextResponse.json(githubContributors);
+};
+
+export const dynamic = "force-static";
--- a/apps/nextjs/src/components/layout/search-engine-optimization.tsx
+++ b/apps/nextjs/src/components/layout/search-engine-optimization.tsx
@@ -0,0 +1,33 @@
+import SuperJSON from "superjson";
+
+import { db, eq } from "@homarr/db";
+import { serverSettings } from "@homarr/db/schema/sqlite";
+import type { defaultServerSettings } from "@homarr/server-settings";
+
+export const SearchEngineOptimization = async () => {
+  const crawlingAndIndexingSetting = await db.query.serverSettings.findFirst({
+    where: eq(serverSettings.settingKey, "crawlingAndIndexing"),
+  });
+
+  if (!crawlingAndIndexingSetting) {
+    return null;
+  }
+
+  const value = SuperJSON.parse<(typeof defaultServerSettings)["crawlingAndIndexing"]>(
+    crawlingAndIndexingSetting.value,
+  );
+
+  const robotsAttributes = [...(value.noIndex ? ["noindex"] : []), ...(value.noIndex ? ["nofollow"] : [])];
+
+  const googleAttributes = [
+    ...(value.noSiteLinksSearchBox ? ["nositelinkssearchbox"] : []),
+    ...(value.noTranslate ? ["notranslate"] : []),
+  ];
+
+  return (
+    <>
+      <meta name="robots" content={robotsAttributes.join(",")} />
+      <meta name="google" content={googleAttributes.join(",")} />
+    </>
+  );
+};
--- a/packages/server-settings/src/index.ts
+++ b/packages/server-settings/src/index.ts
@@ -1,4 +1,4 @@
-export const defaultServerSettingsKeys = ["analytics"] as const;
+export const defaultServerSettingsKeys = ["analytics", "crawlingAndIndexing"] as const;

 export type ServerSettingsRecord = {
  [key in (typeof defaultServerSettingsKeys)[number]]: Record<string, unknown>;
@@ -11,6 +11,12 @@ export const defaultServerSettings = {
    enableIntegrationData: false,
    enableUserData: false,
  },
+  crawlingAndIndexing: {
+    noIndex: true,
+    noFollow: true,
+    noTranslate: true,
+    noSiteLinksSearchBox: false,
+  },
 } satisfies ServerSettingsRecord;

 export type ServerSettings = typeof defaultServerSettings;
--- a/packages/translation/src/lang/en.ts
+++ b/packages/translation/src/lang/en.ts
@@ -1622,6 +1622,27 @@ export default {
              text: "Send the amount of users and whether you've activated SSO",
            },
          },
+          crawlingAndIndexing: {
+            title: "Crawling and Indexing",
+            warning:
+              "Enabling or disabling any settings here will severely impact how search engines will index & crawl your page. Any setting is a request and it is up to the crawler to apply these settings. Any modification may take up to multiple days or weeks to apply. Some settings may be search engine specific.",
+            noIndex: {
+              title: "No index",
+              text: "Do not index the website on search engines and don't show it in any search results",
+            },
+            noFollow: {
+              title: "No follow",
+              text: "Do not follow any links while indexing. Disabling this will lead to crawlers attempting to follow all links on Homarr.",
+            },
+            noTranslate: {
+              title: "No translate",
+              text: "When the site language is likely not that the user is likely to want to read, Google will show a translation link in the search results",
+            },
+            noSiteLinksSearchBox: {
+              title: "No site links search box",
+              text: "Google will build a search box with the crawled links along with other direct links. Enabling this will ask Google to disable that box.",
+            },
+          },
        },
      },
      tool: {