From 3b69669edd9db6a5aeb21f3302875f941e14f369 Mon Sep 17 00:00:00 2001 From: dakkar Date: Thu, 2 Jan 2025 13:14:16 +0000 Subject: [PATCH 01/21] use template variables for MR template I often copy my commit messages into the MR description, this automates that process. --- .gitlab/merge_request_templates/default.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab/merge_request_templates/default.md b/.gitlab/merge_request_templates/default.md index e6977def70..389b2c8cbe 100644 --- a/.gitlab/merge_request_templates/default.md +++ b/.gitlab/merge_request_templates/default.md @@ -3,10 +3,12 @@ # **What does this MR do?** +%{all_commits} + # **Contribution Guidelines** By submitting this merge request, you agree to follow our [Contribution Guidelines](https://activitypub.software/TransFem-org/Sharkey/-/blob/develop/CONTRIBUTING.md) - [ ] I agree to follow this project's Contribution Guidelines - [ ] I have made sure to test this merge request - + From 01a5300be88665ae414c15e8e0667c93799d8f0e Mon Sep 17 00:00:00 2001 From: dakkar Date: Sat, 18 Jan 2025 12:51:38 +0000 Subject: [PATCH 02/21] handle more complex ruby from/to html - fixes #605 this is not exactly great, but it should be "good enough" note that the new `group` function should not escape in the wild, as we don't document it and only use it internally I tried using `$[scale foo bar]` instead of `$[group foo bar]`, but that would be rendered as `foo bar` when sent over the network to non-misskey instances, and we don't want that --- packages/backend/src/core/MfmService.ts | 67 +++++++++++++++++++ packages/backend/test/unit/MfmService.ts | 13 ++++ .../frontend/src/components/global/MkMfm.ts | 4 ++ 3 files changed, 84 insertions(+) diff --git a/packages/backend/src/core/MfmService.ts b/packages/backend/src/core/MfmService.ts index 42676d6f98..48995672c5 100644 --- a/packages/backend/src/core/MfmService.ts +++ b/packages/backend/src/core/MfmService.ts @@ -230,6 +230,67 @@ export class MfmService { break; } + case 'rp': break + case 'rt': { + appendChildren(node.childNodes); + break; + } + case 'ruby': { + if (node.childNodes) { + /* + we get: + ``` + + some text ( annotation ) + more text more annotation + + ``` + + and we want to produce: + ``` + $[ruby $[group some text] annotation] + $[ruby $[group more text] more annotation] + ``` + + that `group` is a hack, because when the `ruby` render + sees just text inside the `$[ruby]`, it splits on + whitespace, considers the first "word" to be the main + content, and the rest the annotation + + with that `group`, we force it to consider the whole + group as the main content + + (note that the `rp` are to be ignored, they only exist + for browsers who don't understand ruby) + */ + let nonRtNodes=[]; + // scan children, ignore `rp`, split on `rt` + for (const child of node.childNodes) { + if (treeAdapter.isTextNode(child)) { + nonRtNodes.push(child); + continue; + } + if (!treeAdapter.isElementNode(child)) { + continue; + } + if (child.nodeName == 'rp') { + continue; + } + if (child.nodeName == 'rt') { + text += '$[ruby $[group '; + appendChildren(nonRtNodes); + text += '] '; + analyze(child); + text += '] '; + nonRtNodes=[]; + continue; + } + nonRtNodes.push(child); + } + } + break; + } + default: // includes inline elements { appendChildren(node.childNodes); @@ -348,6 +409,12 @@ export class MfmService { } } + case 'group': { // this is mostly a hack for `ruby` + const el = doc.createElement('span'); + appendChildren(node.children, el); + return el; + } + default: { return fnDefault(node); } diff --git a/packages/backend/test/unit/MfmService.ts b/packages/backend/test/unit/MfmService.ts index 8d5683329f..93ce0672dc 100644 --- a/packages/backend/test/unit/MfmService.ts +++ b/packages/backend/test/unit/MfmService.ts @@ -45,6 +45,12 @@ describe('MfmService', () => { const output = '

<p>Hello, world!</p>

'; assert.equal(mfmService.toHtml(mfm.parse(input)), output); }); + + test('ruby', () => { + const input = '$[ruby $[group *some* text] ignore me]'; + const output = '

some text(ignore me)

'; + assert.equal(mfmService.toHtml(mfm.parse(input)), output); + }); }); describe('fromHtml', () => { @@ -115,5 +121,12 @@ describe('MfmService', () => { test('hashtag', () => { assert.deepStrictEqual(mfmService.fromHtml('

a #a d

', ['#a']), 'a #a d'); }); + + test('ruby', () => { + assert.deepStrictEqual( + mfmService.fromHtml(' some text (ignore me) and more'), + '$[ruby $[group some text ] ignore me] $[ruby $[group and ] more]' + ); + }); }); }); diff --git a/packages/frontend/src/components/global/MkMfm.ts b/packages/frontend/src/components/global/MkMfm.ts index 1039572a06..aceed17189 100644 --- a/packages/frontend/src/components/global/MkMfm.ts +++ b/packages/frontend/src/components/global/MkMfm.ts @@ -358,6 +358,10 @@ export default function (props: MfmProps, { emit }: { emit: SetupContext Date: Sat, 18 Jan 2025 14:54:17 +0000 Subject: [PATCH 03/21] pick lints --- packages/backend/src/core/MfmService.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/backend/src/core/MfmService.ts b/packages/backend/src/core/MfmService.ts index 48995672c5..bc624daaee 100644 --- a/packages/backend/src/core/MfmService.ts +++ b/packages/backend/src/core/MfmService.ts @@ -230,7 +230,7 @@ export class MfmService { break; } - case 'rp': break + case 'rp': break; case 'rt': { appendChildren(node.childNodes); break; @@ -263,7 +263,7 @@ export class MfmService { (note that the `rp` are to be ignored, they only exist for browsers who don't understand ruby) */ - let nonRtNodes=[]; + let nonRtNodes = []; // scan children, ignore `rp`, split on `rt` for (const child of node.childNodes) { if (treeAdapter.isTextNode(child)) { @@ -273,16 +273,16 @@ export class MfmService { if (!treeAdapter.isElementNode(child)) { continue; } - if (child.nodeName == 'rp') { + if (child.nodeName === 'rp') { continue; } - if (child.nodeName == 'rt') { + if (child.nodeName === 'rt') { text += '$[ruby $[group '; appendChildren(nonRtNodes); text += '] '; analyze(child); text += '] '; - nonRtNodes=[]; + nonRtNodes = []; continue; } nonRtNodes.push(child); From 408e2f824a686defd3ea96cc5782b6b59779b7b9 Mon Sep 17 00:00:00 2001 From: dakkar Date: Sun, 19 Jan 2025 11:15:01 +0000 Subject: [PATCH 04/21] format ruby for masto api --- packages/backend/src/core/MfmService.ts | 68 +++++++++++++++++++++--- packages/backend/test/unit/MfmService.ts | 38 +++++++++++++ 2 files changed, 100 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/core/MfmService.ts b/packages/backend/src/core/MfmService.ts index bc624daaee..dc47e38562 100644 --- a/packages/backend/src/core/MfmService.ts +++ b/packages/backend/src/core/MfmService.ts @@ -409,7 +409,9 @@ export class MfmService { } } - case 'group': { // this is mostly a hack for `ruby` + // hack for ruby, should never be needed because we should + // never send this out to other instances + case 'group': { const el = doc.createElement('span'); appendChildren(node.children, el); return el; @@ -593,11 +595,65 @@ export class MfmService { }, async fn(node) { - const el = doc.createElement('span'); - el.textContent = '*'; - await appendChildren(node.children, el); - el.textContent += '*'; - return el; + switch (node.props.name) { + case 'group': { // hack for ruby + const el = doc.createElement('span'); + await appendChildren(node.children, el); + return el; + } + case 'ruby': { + if (node.children.length === 1) { + const child = node.children[0]; + const text = child.type === 'text' ? child.props.text : ''; + const rubyEl = doc.createElement('ruby'); + const rtEl = doc.createElement('rt'); + + const rpStartEl = doc.createElement('rp'); + rpStartEl.appendChild(doc.createTextNode('(')); + const rpEndEl = doc.createElement('rp'); + rpEndEl.appendChild(doc.createTextNode(')')); + + rubyEl.appendChild(doc.createTextNode(text.split(' ')[0])); + rtEl.appendChild(doc.createTextNode(text.split(' ')[1])); + rubyEl.appendChild(rpStartEl); + rubyEl.appendChild(rtEl); + rubyEl.appendChild(rpEndEl); + return rubyEl; + } else { + const rt = node.children.at(-1); + + if (!rt) { + const el = doc.createElement('span'); + await appendChildren(node.children, el); + return el; + } + + const text = rt.type === 'text' ? rt.props.text : ''; + const rubyEl = doc.createElement('ruby'); + const rtEl = doc.createElement('rt'); + + const rpStartEl = doc.createElement('rp'); + rpStartEl.appendChild(doc.createTextNode('(')); + const rpEndEl = doc.createElement('rp'); + rpEndEl.appendChild(doc.createTextNode(')')); + + await appendChildren(node.children.slice(0, node.children.length - 1), rubyEl); + rtEl.appendChild(doc.createTextNode(text.trim())); + rubyEl.appendChild(rpStartEl); + rubyEl.appendChild(rtEl); + rubyEl.appendChild(rpEndEl); + return rubyEl; + } + } + + default: { + const el = doc.createElement('span'); + el.textContent = '*'; + await appendChildren(node.children, el); + el.textContent += '*'; + return el; + } + } }, blockCode(node) { diff --git a/packages/backend/test/unit/MfmService.ts b/packages/backend/test/unit/MfmService.ts index 93ce0672dc..5c3ffba422 100644 --- a/packages/backend/test/unit/MfmService.ts +++ b/packages/backend/test/unit/MfmService.ts @@ -47,12 +47,50 @@ describe('MfmService', () => { }); test('ruby', () => { + const input = '$[ruby some text ignore me]'; + const output = '

some(text)

'; + assert.equal(mfmService.toHtml(mfm.parse(input)), output); + }); + + test('ruby2', () => { + const input = '$[ruby *some text* ignore me]'; + const output = '

some text(ignore me)

'; + assert.equal(mfmService.toHtml(mfm.parse(input)), output); + }); + + test('ruby 3', () => { const input = '$[ruby $[group *some* text] ignore me]'; const output = '

some text(ignore me)

'; assert.equal(mfmService.toHtml(mfm.parse(input)), output); }); }); + describe('toMastoApiHtml', () => { + test('br', async () => { + const input = 'foo\nbar\nbaz'; + const output = '

foo
bar
baz

'; + assert.equal(await mfmService.toMastoApiHtml(mfm.parse(input)), output); + }); + + test('br alt', async () => { + const input = 'foo\r\nbar\rbaz'; + const output = '

foo
bar
baz

'; + assert.equal(await mfmService.toMastoApiHtml(mfm.parse(input)), output); + }); + + test('escape', async () => { + const input = '```\n

Hello, world!

\n```'; + const output = '

<p>Hello, world!</p>

'; + assert.equal(await mfmService.toMastoApiHtml(mfm.parse(input)), output); + }); + + test('ruby', async () => { + const input = '$[ruby $[group *some* text] ignore me]'; + const output = '

*some* text(ignore me)

'; + assert.equal(await mfmService.toMastoApiHtml(mfm.parse(input)), output); + }); + }); + describe('fromHtml', () => { test('p', () => { assert.deepStrictEqual(mfmService.fromHtml('

a

b

'), 'a\n\nb'); From 1080f19e99ffccdebb09c39035d93d9b561ef0e6 Mon Sep 17 00:00:00 2001 From: piuvas Date: Fri, 24 Jan 2025 18:37:18 -0300 Subject: [PATCH 05/21] generalize current language so we match more broadly on fallback --- packages/backend/src/server/web/boot.embed.js | 2 +- packages/backend/src/server/web/boot.js | 2 +- packages/frontend/src/_dev_boot_.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/server/web/boot.embed.js b/packages/backend/src/server/web/boot.embed.js index b07dce3ac4..1af1dc545b 100644 --- a/packages/backend/src/server/web/boot.embed.js +++ b/packages/backend/src/server/web/boot.embed.js @@ -48,7 +48,7 @@ if (supportedLangs.includes(navigator.language)) { lang = navigator.language; } else { - lang = supportedLangs.find(x => x.split('-')[0] === navigator.language); + lang = supportedLangs.find(x => x.split('-')[0] === navigator.language.split('-')[0]); // Fallback if (lang == null) lang = 'en-US'; diff --git a/packages/backend/src/server/web/boot.js b/packages/backend/src/server/web/boot.js index bf83340bde..54750e26e5 100644 --- a/packages/backend/src/server/web/boot.js +++ b/packages/backend/src/server/web/boot.js @@ -39,7 +39,7 @@ if (supportedLangs.includes(navigator.language)) { lang = navigator.language; } else { - lang = supportedLangs.find(x => x.split('-')[0] === navigator.language); + lang = supportedLangs.find(x => x.split('-')[0] === navigator.language.split('-')[0]); // Fallback if (lang == null) lang = 'en-US'; diff --git a/packages/frontend/src/_dev_boot_.ts b/packages/frontend/src/_dev_boot_.ts index f312765dcf..ebce7e735f 100644 --- a/packages/frontend/src/_dev_boot_.ts +++ b/packages/frontend/src/_dev_boot_.ts @@ -25,7 +25,7 @@ async function main() { if (supportedLangs.includes(navigator.language)) { lang = navigator.language; } else { - lang = supportedLangs.find(x => x.split('-')[0] === navigator.language); + lang = supportedLangs.find(x => x.split('-')[0] === navigator.language.split('-')[0]); // Fallback if (lang == null) lang = 'en-US'; From 83f2d93d30100f38df2ed34807ec5d8d80a19c4b Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sat, 1 Feb 2025 18:53:32 -0500 Subject: [PATCH 06/21] increase rate limit on federation/update-remote-user --- .../server/api/endpoints/federation/update-remote-user.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/server/api/endpoints/federation/update-remote-user.ts b/packages/backend/src/server/api/endpoints/federation/update-remote-user.ts index 3ec9522c44..5217f79065 100644 --- a/packages/backend/src/server/api/endpoints/federation/update-remote-user.ts +++ b/packages/backend/src/server/api/endpoints/federation/update-remote-user.ts @@ -13,10 +13,11 @@ export const meta = { requireCredential: false, - // 2 calls per second + // Up to 10 calls, then 4 / second. + // This allows for reliable automation. limit: { - duration: 1000, - max: 2, + max: 10, + dripRate: 250, }, } as const; From bd716ed8377ab160abf23f60ccc43affebc87882 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sat, 1 Feb 2025 23:56:41 -0500 Subject: [PATCH 07/21] increase the rate limit for `/api/i` endpoint, preventing some 429 errors if multiple tabs reload simultaneously --- packages/backend/src/server/api/endpoints/i.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/server/api/endpoints/i.ts b/packages/backend/src/server/api/endpoints/i.ts index 9347c9ca27..48a2e3b40a 100644 --- a/packages/backend/src/server/api/endpoints/i.ts +++ b/packages/backend/src/server/api/endpoints/i.ts @@ -31,10 +31,12 @@ export const meta = { }, }, - // 3 calls per second + // up to 20 calls, then 1 per second. + // This handles bursty traffic when all tabs reload as a group limit: { - duration: 1000, - max: 3, + max: 20, + dripSize: 1, + dripRate: 1000, }, } as const; From 9934e4d94874dee6400ff21454e1c07e12b6f807 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 26 Jan 2025 17:03:34 -0500 Subject: [PATCH 08/21] allow MkFollowButton to be disabled --- packages/frontend/src/components/MkFollowButton.vue | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/frontend/src/components/MkFollowButton.vue b/packages/frontend/src/components/MkFollowButton.vue index c7965aaac4..277d116845 100644 --- a/packages/frontend/src/components/MkFollowButton.vue +++ b/packages/frontend/src/components/MkFollowButton.vue @@ -7,7 +7,7 @@ SPDX-License-Identifier: AGPL-3.0-only