From 993532bc1fe0f67d84e16a99ee916f7fff9b0935 Mon Sep 17 00:00:00 2001 From: Kinetix Date: Tue, 28 Jan 2025 15:57:45 -0800 Subject: [PATCH 1/3] Adding robots.txt override via admin control panel This is a requested low priority feature in #418 - I created the changes to follow similarly to how the Instance Description is handled. --- locales/index.d.ts | 8 ++++++++ .../backend/migration/1738098171990-robotsTxt.js | 16 ++++++++++++++++ .../src/core/entities/MetaEntityService.ts | 1 + packages/backend/src/models/Meta.ts | 5 +++++ packages/backend/src/models/json-schema/meta.ts | 4 ++++ .../src/server/api/endpoints/admin/meta.ts | 5 +++++ .../server/api/endpoints/admin/update-meta.ts | 5 +++++ .../src/server/web/ClientServerService.ts | 9 ++++++++- packages/frontend/src/pages/admin/settings.vue | 7 +++++++ sharkey-locales/en-US.yml | 3 +++ 10 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 packages/backend/migration/1738098171990-robotsTxt.js diff --git a/locales/index.d.ts b/locales/index.d.ts index 3a3b94b89d..70eba52ea0 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -11626,6 +11626,14 @@ export interface Locale extends ILocale { * Scheduled Notes */ "scheduledNotes": string; + /** + * Custom robots.txt + */ + "robotsTxt": string; + /** + * Adding entries here will override the default robots.txt packaged with Sharkey. Maximum 2048 characters. + */ + "robotsTxtDescription": string; } declare const locales: { [lang: string]: Locale; diff --git a/packages/backend/migration/1738098171990-robotsTxt.js b/packages/backend/migration/1738098171990-robotsTxt.js new file mode 100644 index 0000000000..947f21cc46 --- /dev/null +++ b/packages/backend/migration/1738098171990-robotsTxt.js @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: marie and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class RobotsTxt1738098171990 { + name = 'RobotsTxt1738098171990' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" ADD "robotsTxt" character varying(2048)`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "robotsTxt"`); + } +} diff --git a/packages/backend/src/core/entities/MetaEntityService.ts b/packages/backend/src/core/entities/MetaEntityService.ts index 7d7b4cbd81..857e8f5a7b 100644 --- a/packages/backend/src/core/entities/MetaEntityService.ts +++ b/packages/backend/src/core/entities/MetaEntityService.ts @@ -95,6 +95,7 @@ export class MetaEntityService { mcaptchaInstanceUrl: instance.mcaptchaInstanceUrl, enableRecaptcha: instance.enableRecaptcha, enableAchievements: instance.enableAchievements, + robotsTxt: instance.robotsTxt, recaptchaSiteKey: instance.recaptchaSiteKey, enableTurnstile: instance.enableTurnstile, turnstileSiteKey: instance.turnstileSiteKey, diff --git a/packages/backend/src/models/Meta.ts b/packages/backend/src/models/Meta.ts index 3fc3f273dd..a224117676 100644 --- a/packages/backend/src/models/Meta.ts +++ b/packages/backend/src/models/Meta.ts @@ -599,6 +599,11 @@ export class MiMeta { }) public enableAchievements: boolean; + @Column('varchar', { + length: 2048, nullable: true, + }) + public robotsTxt: string | null; + @Column('jsonb', { default: { }, }) diff --git a/packages/backend/src/models/json-schema/meta.ts b/packages/backend/src/models/json-schema/meta.ts index 5179e5d51c..29fdb4f6be 100644 --- a/packages/backend/src/models/json-schema/meta.ts +++ b/packages/backend/src/models/json-schema/meta.ts @@ -139,6 +139,10 @@ export const packedMetaLiteSchema = { type: 'boolean', optional: false, nullable: true, }, + robotsTxt: { + type: 'string', + optional: false, nullable: true, + }, enableTestcaptcha: { type: 'boolean', optional: false, nullable: false, diff --git a/packages/backend/src/server/api/endpoints/admin/meta.ts b/packages/backend/src/server/api/endpoints/admin/meta.ts index 6495e3b7da..436dcf27cb 100644 --- a/packages/backend/src/server/api/endpoints/admin/meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/meta.ts @@ -391,6 +391,10 @@ export const meta = { type: 'boolean', optional: false, nullable: false, }, + robotsTxt: { + type: 'string', + optional: false, nullable: true, + }, enableIdenticonGeneration: { type: 'boolean', optional: false, nullable: false, @@ -708,6 +712,7 @@ export default class extends Endpoint { // eslint- enableStatsForFederatedInstances: instance.enableStatsForFederatedInstances, enableServerMachineStats: instance.enableServerMachineStats, enableAchievements: instance.enableAchievements, + robotsTxt: instance.robotsTxt, enableIdenticonGeneration: instance.enableIdenticonGeneration, bannedEmailDomains: instance.bannedEmailDomains, policies: { ...DEFAULT_POLICIES, ...instance.policies }, diff --git a/packages/backend/src/server/api/endpoints/admin/update-meta.ts b/packages/backend/src/server/api/endpoints/admin/update-meta.ts index 72f428d85f..b3733d3d39 100644 --- a/packages/backend/src/server/api/endpoints/admin/update-meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/update-meta.ts @@ -149,6 +149,7 @@ export const paramDef = { enableStatsForFederatedInstances: { type: 'boolean' }, enableServerMachineStats: { type: 'boolean' }, enableAchievements: { type: 'boolean' }, + robotsTxt: { type: 'string', nullable: true }, enableIdenticonGeneration: { type: 'boolean' }, serverRules: { type: 'array', items: { type: 'string' } }, bannedEmailDomains: { type: 'array', items: { type: 'string' } }, @@ -636,6 +637,10 @@ export default class extends Endpoint { // eslint- set.enableAchievements = ps.enableAchievements; } + if (ps.robotsTxt !== undefined) { + set.robotsTxt = ps.robotsTxt; + } + if (ps.enableIdenticonGeneration !== undefined) { set.enableIdenticonGeneration = ps.enableIdenticonGeneration; } diff --git a/packages/backend/src/server/web/ClientServerService.ts b/packages/backend/src/server/web/ClientServerService.ts index e59314bf55..e93900b358 100644 --- a/packages/backend/src/server/web/ClientServerService.ts +++ b/packages/backend/src/server/web/ClientServerService.ts @@ -488,7 +488,14 @@ export class ClientServerService { }); fastify.get('/robots.txt', async (request, reply) => { - return await reply.sendFile('/robots.txt', staticAssets); + if (this.meta.robotsTxt) { + let content = ''; + content += this.meta.robotsTxt; + reply.header('Content-Type', 'text/plain'); + return await reply.send(content); + } else { + return await reply.sendFile('/robots.txt', staticAssets); + } }); // OpenSearch XML diff --git a/packages/frontend/src/pages/admin/settings.vue b/packages/frontend/src/pages/admin/settings.vue index 68f211de5c..cd05b43be8 100644 --- a/packages/frontend/src/pages/admin/settings.vue +++ b/packages/frontend/src/pages/admin/settings.vue @@ -159,6 +159,11 @@ SPDX-License-Identifier: AGPL-3.0-only + + + + + @@ -369,10 +374,12 @@ const serviceWorkerForm = useForm({ const otherForm = useForm({ enableAchievements: meta.enableAchievements, enableBotTrending: meta.enableBotTrending, + robotsTxt: meta.robotsTxt, }, async (state) => { await os.apiWithDialog('admin/update-meta', { enableAchievements: state.enableAchievements, enableBotTrending: state.enableBotTrending, + robotsTxt: state.robotsTxt, }); fetchInstance(true); }); diff --git a/sharkey-locales/en-US.yml b/sharkey-locales/en-US.yml index 6b3c099411..e0430b10ea 100644 --- a/sharkey-locales/en-US.yml +++ b/sharkey-locales/en-US.yml @@ -434,3 +434,6 @@ scheduledNotes: "Scheduled Notes" _permissions: "read:notes-schedule": "View your list of scheduled notes" "write:notes-schedule": "Compose or delete scheduled notes" + +robotsTxt: "Custom robots.txt" +robotsTxtDescription: "Adding entries here will override the default robots.txt packaged with Sharkey. Maximum 2048 characters." From 4d91baaa13a8c4e8b9df221ce710d7005102c5a3 Mon Sep 17 00:00:00 2001 From: Kinetix Date: Fri, 31 Jan 2025 10:22:13 -0800 Subject: [PATCH 2/3] Adjust as per suggestions in !877 --- ...738098171990-robotsTxt.js => 1738346484187-robotsTxt.js} | 6 +++--- packages/backend/src/server/web/ClientServerService.ts | 4 +--- 2 files changed, 4 insertions(+), 6 deletions(-) rename packages/backend/migration/{1738098171990-robotsTxt.js => 1738346484187-robotsTxt.js} (75%) diff --git a/packages/backend/migration/1738098171990-robotsTxt.js b/packages/backend/migration/1738346484187-robotsTxt.js similarity index 75% rename from packages/backend/migration/1738098171990-robotsTxt.js rename to packages/backend/migration/1738346484187-robotsTxt.js index 947f21cc46..00ea1fb030 100644 --- a/packages/backend/migration/1738098171990-robotsTxt.js +++ b/packages/backend/migration/1738346484187-robotsTxt.js @@ -3,11 +3,11 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -export class RobotsTxt1738098171990 { - name = 'RobotsTxt1738098171990' +export class RobotsTxt1738346484187 { + name = 'RobotsTxt1738346484187' async up(queryRunner) { - await queryRunner.query(`ALTER TABLE "meta" ADD "robotsTxt" character varying(2048)`); + await queryRunner.query(`ALTER TABLE "meta" ADD "robotsTxt" text`); } async down(queryRunner) { diff --git a/packages/backend/src/server/web/ClientServerService.ts b/packages/backend/src/server/web/ClientServerService.ts index e93900b358..ae923ada1f 100644 --- a/packages/backend/src/server/web/ClientServerService.ts +++ b/packages/backend/src/server/web/ClientServerService.ts @@ -489,10 +489,8 @@ export class ClientServerService { fastify.get('/robots.txt', async (request, reply) => { if (this.meta.robotsTxt) { - let content = ''; - content += this.meta.robotsTxt; reply.header('Content-Type', 'text/plain'); - return await reply.send(content); + return await reply.send(this.meta.robotsTxt); } else { return await reply.sendFile('/robots.txt', staticAssets); } From 826b793c31005fa2dfa938048279ed7fe89ee0a6 Mon Sep 17 00:00:00 2001 From: Kinetix Date: Fri, 31 Jan 2025 10:26:05 -0800 Subject: [PATCH 3/3] Fix the description text in the control panel. --- sharkey-locales/en-US.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sharkey-locales/en-US.yml b/sharkey-locales/en-US.yml index e0430b10ea..8219ccd046 100644 --- a/sharkey-locales/en-US.yml +++ b/sharkey-locales/en-US.yml @@ -436,4 +436,4 @@ _permissions: "write:notes-schedule": "Compose or delete scheduled notes" robotsTxt: "Custom robots.txt" -robotsTxtDescription: "Adding entries here will override the default robots.txt packaged with Sharkey. Maximum 2048 characters." +robotsTxtDescription: "Adding entries here will override the default robots.txt packaged with Sharkey."