From 7c002ce56ef86f8a375275a78c0bda38d540c131 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 8 Dec 2024 11:33:57 -0500 Subject: [PATCH] move all Rate Limit type defs to rate-limit-utils.ts --- packages/backend/src/misc/rate-limit-utils.ts | 120 +++++++++++++++++- .../backend/src/server/FileServerService.ts | 4 +- .../backend/src/server/api/ApiCallService.ts | 19 ++- .../src/server/api/SkRateLimiterService.ts | 99 +-------------- packages/backend/src/server/api/endpoints.ts | 26 +--- .../test/unit/SigninWithPasskeyApiService.ts | 3 +- .../server/api/SkRateLimiterServiceTests.ts | 5 +- 7 files changed, 144 insertions(+), 132 deletions(-) diff --git a/packages/backend/src/misc/rate-limit-utils.ts b/packages/backend/src/misc/rate-limit-utils.ts index 0f38755502..00c0701ad6 100644 --- a/packages/backend/src/misc/rate-limit-utils.ts +++ b/packages/backend/src/misc/rate-limit-utils.ts @@ -4,7 +4,125 @@ */ import { FastifyReply } from 'fastify'; -import { LimitInfo } from '@/server/api/SkRateLimiterService.js'; + +export type RateLimit = BucketRateLimit | LegacyRateLimit; + +/** + * Rate limit based on "leaky bucket" logic. + * The bucket count increases with each call, and decreases gradually at a given rate. + * The subject is blocked until the bucket count drops below the limit. + */ +export interface BucketRateLimit { + /** + * Unique key identifying the particular resource (or resource group) being limited. + */ + key: string; + + /** + * Constant value identifying the type of rate limit. + */ + type: 'bucket'; + + /** + * Size of the bucket, in number of requests. + * The subject will be blocked when the number of calls exceeds this size. + */ + size: number; + + /** + * How often the bucket should "drip" and reduce the counter, measured in milliseconds. + * Defaults to 1000 (1 second). + */ + dripRate?: number; + + /** + * Amount to reduce the counter on each drip. + * Defaults to 1. + */ + dripSize?: number; +} + +/** + * Legacy rate limit based on a "request window" with a maximum number of requests within a given time box. + * These will be translated into a bucket with linear drip rate. + */ +export interface LegacyRateLimit { + /** + * Unique key identifying the particular resource (or resource group) being limited. + */ + key: string; + + /** + * Constant value identifying the type of rate limit. + * Must be excluded or explicitly set to undefined + */ + type?: undefined; + + /** + * Duration of the request window, in milliseconds. + * If present, then "max" must also be included. + */ + duration?: number; + + /** + * Maximum number of requests allowed in the request window. + * If present, then "duration" must also be included. + */ + max?: number; + + /** + * Optional minimum interval between consecutive requests. + * Will apply in addition to the primary rate limit. + */ + minInterval?: number; +} + +/** + * Metadata about the current status of a rate limiter + */ +export interface LimitInfo { + /** + * True if the limit has been reached, and the call should be blocked. + */ + blocked: boolean; + + /** + * Number of calls that can be made before the limit is triggered. + */ + remaining: number; + + /** + * Time in seconds until the next call can be made, or zero if the next call can be made immediately. + * Rounded up to the nearest second. + */ + resetSec: number; + + /** + * Time in milliseconds until the next call can be made, or zero if the next call can be made immediately. + * Rounded up to the nearest milliseconds. + */ + resetMs: number; + + /** + * Time in seconds until the limit has fully reset. + * Rounded up to the nearest second. + */ + fullResetSec: number; + + /** + * Time in milliseconds until the limit has fully reset. + * Rounded up to the nearest millisecond. + */ + fullResetMs: number; +} + +export function isLegacyRateLimit(limit: RateLimit): limit is LegacyRateLimit { + return limit.type === undefined; +} + +export function hasMinLimit(limit: LegacyRateLimit): limit is LegacyRateLimit & { minInterval: number } { + return !!limit.minInterval; +} export function sendRateLimitHeaders(reply: FastifyReply, info: LimitInfo): void { // Number of seconds until the limit has fully reset. diff --git a/packages/backend/src/server/FileServerService.ts b/packages/backend/src/server/FileServerService.ts index 9903113f43..3a03cd8c00 100644 --- a/packages/backend/src/server/FileServerService.ts +++ b/packages/backend/src/server/FileServerService.ts @@ -31,8 +31,8 @@ import { handleRequestRedirectToOmitSearch } from '@/misc/fastify-hook-handlers. import { getIpHash } from '@/misc/get-ip-hash.js'; import { AuthenticateService } from '@/server/api/AuthenticateService.js'; import { RoleService } from '@/core/RoleService.js'; -import { RateLimit, SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; -import { sendRateLimitHeaders } from '@/misc/rate-limit-utils.js'; +import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; +import { RateLimit, sendRateLimitHeaders } from '@/misc/rate-limit-utils.js'; import type { FastifyInstance, FastifyRequest, FastifyReply, FastifyPluginOptions } from 'fastify'; const _filename = fileURLToPath(import.meta.url); diff --git a/packages/backend/src/server/api/ApiCallService.ts b/packages/backend/src/server/api/ApiCallService.ts index 38d33c761d..6ad4bc8cb5 100644 --- a/packages/backend/src/server/api/ApiCallService.ts +++ b/packages/backend/src/server/api/ApiCallService.ts @@ -18,8 +18,8 @@ import { createTemp } from '@/misc/create-temp.js'; import { bindThis } from '@/decorators.js'; import { RoleService } from '@/core/RoleService.js'; import type { Config } from '@/config.js'; -import { sendRateLimitHeaders } from '@/misc/rate-limit-utils.js'; -import { LegacyRateLimit, SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; +import { RateLimit, sendRateLimitHeaders } from '@/misc/rate-limit-utils.js'; +import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; import { ApiError } from './error.js'; import { ApiLoggerService } from './ApiLoggerService.js'; import { AuthenticateService, AuthenticationError } from './AuthenticateService.js'; @@ -304,7 +304,7 @@ export class ApiCallService implements OnApplicationShutdown { } // For endpoints without a limit, the default is 10 calls per second - const endpointLimit: IEndpointMeta['limit'] = ep.meta.limit ?? { + const endpointLimit = ep.meta.limit ?? { duration: 1000, max: 10, }; @@ -320,18 +320,17 @@ export class ApiCallService implements OnApplicationShutdown { limitActor = getIpHash(request.ip); } - const limit = Object.assign({}, endpointLimit); - - if (limit.key == null) { - (limit as any).key = ep.name; - } - // TODO: 毎リクエスト計算するのもあれだしキャッシュしたい const factor = user ? (await this.roleService.getUserPolicies(user.id)).rateLimitFactor : 1; if (factor > 0) { + const limit = { + key: ep.name, + ...endpointLimit, + } as RateLimit; + // Rate limit - const info = await this.rateLimiterService.limit(limit as LegacyRateLimit, limitActor, factor); + const info = await this.rateLimiterService.limit(limit, limitActor, factor); sendRateLimitHeaders(reply, info); diff --git a/packages/backend/src/server/api/SkRateLimiterService.ts b/packages/backend/src/server/api/SkRateLimiterService.ts index b3c09d01c2..027d05310b 100644 --- a/packages/backend/src/server/api/SkRateLimiterService.ts +++ b/packages/backend/src/server/api/SkRateLimiterService.ts @@ -5,97 +5,12 @@ import { Inject, Injectable } from '@nestjs/common'; import Redis from 'ioredis'; -import type { IEndpointMeta } from '@/server/api/endpoints.js'; import { LoggerService } from '@/core/LoggerService.js'; import { TimeService } from '@/core/TimeService.js'; import { EnvService } from '@/core/EnvService.js'; import { DI } from '@/di-symbols.js'; import type Logger from '@/logger.js'; - -/** - * Metadata about the current status of a rate limiter - */ -export interface LimitInfo { - /** - * True if the limit has been reached, and the call should be blocked. - */ - blocked: boolean; - - /** - * Number of calls that can be made before the limit is triggered. - */ - remaining: number; - - /** - * Time in seconds until the next call can be made, or zero if the next call can be made immediately. - * Rounded up to the nearest second. - */ - resetSec: number; - - /** - * Time in milliseconds until the next call can be made, or zero if the next call can be made immediately. - * Rounded up to the nearest milliseconds. - */ - resetMs: number; - - /** - * Time in seconds until the limit has fully reset. - * Rounded up to the nearest second. - */ - fullResetSec: number; - - /** - * Time in milliseconds until the limit has fully reset. - * Rounded up to the nearest millisecond. - */ - fullResetMs: number; -} - -/** - * Rate limit based on "leaky bucket" logic. - * The bucket count increases with each call, and decreases gradually at a given rate. - * The subject is blocked until the bucket count drops below the limit. - */ -export interface RateLimit { - /** - * Unique key identifying the particular resource (or resource group) being limited. - */ - key: string; - - /** - * Constant value identifying the type of rate limit. - */ - type: 'bucket'; - - /** - * Size of the bucket, in number of requests. - * The subject will be blocked when the number of calls exceeds this size. - */ - size: number; - - /** - * How often the bucket should "drip" and reduce the counter, measured in milliseconds. - * Defaults to 1000 (1 second). - */ - dripRate?: number; - - /** - * Amount to reduce the counter on each drip. - * Defaults to 1. - */ - dripSize?: number; -} - -export type SupportedRateLimit = RateLimit | LegacyRateLimit; -export type LegacyRateLimit = IEndpointMeta['limit'] & { key: NonNullable, type?: undefined }; - -export function isLegacyRateLimit(limit: SupportedRateLimit): limit is LegacyRateLimit { - return limit.type === undefined; -} - -export function hasMinLimit(limit: LegacyRateLimit): limit is LegacyRateLimit & { minInterval: number } { - return !!limit.minInterval; -} +import { BucketRateLimit, LegacyRateLimit, LimitInfo, RateLimit, hasMinLimit, isLegacyRateLimit } from '@/misc/rate-limit-utils.js'; @Injectable() export class SkRateLimiterService { @@ -116,10 +31,10 @@ export class SkRateLimiterService { envService: EnvService, ) { this.logger = loggerService.getLogger('limiter'); - this.disabled = envService.env.NODE_ENV !== 'production'; + this.disabled = envService.env.NODE_ENV !== 'production'; // TODO disable in TEST *only* } - public async limit(limit: SupportedRateLimit, actor: string, factor = 1): Promise { + public async limit(limit: RateLimit, actor: string, factor = 1): Promise { if (this.disabled) { return { blocked: false, @@ -211,7 +126,7 @@ export class SkRateLimiterService { return limitInfo; } - private async limitBucket(limit: RateLimit, actor: string, factor: number): Promise { + private async limitBucket(limit: BucketRateLimit, actor: string, factor: number): Promise { if (limit.size < 1) throw new Error(`Invalid rate limit ${limit.key}: size is less than 1 (${limit.size})`); if (limit.dripRate != null && limit.dripRate < 1) throw new Error(`Invalid rate limit ${limit.key}: dripRate is less than 1 (${limit.dripRate})`); if (limit.dripSize != null && limit.dripSize < 1) throw new Error(`Invalid rate limit ${limit.key}: dripSize is less than 1 (${limit.dripSize})`); @@ -251,7 +166,7 @@ export class SkRateLimiterService { return limitInfo; } - private async getLimitCounter(limit: SupportedRateLimit, actor: string, subject: string): Promise { + private async getLimitCounter(limit: RateLimit, actor: string, subject: string): Promise { const key = createLimitKey(limit, actor, subject); const value = await this.redisClient.get(key); @@ -262,7 +177,7 @@ export class SkRateLimiterService { return JSON.parse(value); } - private async setLimitCounter(limit: SupportedRateLimit, actor: string, counter: LimitCounter, expiration: number, subject: string): Promise { + private async setLimitCounter(limit: RateLimit, actor: string, counter: LimitCounter, expiration: number, subject: string): Promise { const key = createLimitKey(limit, actor, subject); const value = JSON.stringify(counter); const expirationSec = Math.max(expiration, 1); @@ -270,7 +185,7 @@ export class SkRateLimiterService { } } -function createLimitKey(limit: SupportedRateLimit, actor: string, subject: string): string { +function createLimitKey(limit: RateLimit, actor: string, subject: string): string { return `rl_${actor}_${limit.key}_${subject}`; } diff --git a/packages/backend/src/server/api/endpoints.ts b/packages/backend/src/server/api/endpoints.ts index 14e002929a..a4193a64ec 100644 --- a/packages/backend/src/server/api/endpoints.ts +++ b/packages/backend/src/server/api/endpoints.ts @@ -5,6 +5,7 @@ import { permissions } from 'misskey-js'; import type { KeyOf, Schema } from '@/misc/json-schema.js'; +import type { RateLimit } from '@/misc/rate-limit-utils.js'; import * as ep___admin_abuseReport_notificationRecipient_list from '@/server/api/endpoints/admin/abuse-report/notification-recipient/list.js'; @@ -855,30 +856,7 @@ interface IEndpointMetaBase { * エンドポイントのリミテーションに関するやつ * 省略した場合はリミテーションは無いものとして解釈されます。 */ - readonly limit?: { - - /** - * 複数のエンドポイントでリミットを共有したい場合に指定するキー - */ - readonly key?: string; - - /** - * リミットを適用する期間(ms) - * このプロパティを設定する場合、max プロパティも設定する必要があります。 - */ - readonly duration?: number; - - /** - * durationで指定した期間内にいくつまでリクエストできるのか - * このプロパティを設定する場合、duration プロパティも設定する必要があります。 - */ - readonly max?: number; - - /** - * 最低でもどれくらいの間隔を開けてリクエストしなければならないか(ms) - */ - readonly minInterval?: number; - }; + readonly limit?: Readonly>; /** * ファイルの添付を必要とするか否か diff --git a/packages/backend/test/unit/SigninWithPasskeyApiService.ts b/packages/backend/test/unit/SigninWithPasskeyApiService.ts index f0630f2d2b..7df991c15c 100644 --- a/packages/backend/test/unit/SigninWithPasskeyApiService.ts +++ b/packages/backend/test/unit/SigninWithPasskeyApiService.ts @@ -20,7 +20,8 @@ import { SigninWithPasskeyApiService } from '@/server/api/SigninWithPasskeyApiSe import { WebAuthnService } from '@/core/WebAuthnService.js'; import { SigninService } from '@/server/api/SigninService.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; -import { LimitInfo, SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; +import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; +import { LimitInfo } from '@/misc/rate-limit-utils.js'; const moduleMocker = new ModuleMocker(global); diff --git a/packages/backend/test/unit/server/api/SkRateLimiterServiceTests.ts b/packages/backend/test/unit/server/api/SkRateLimiterServiceTests.ts index 2297c2bc03..215b83b53f 100644 --- a/packages/backend/test/unit/server/api/SkRateLimiterServiceTests.ts +++ b/packages/backend/test/unit/server/api/SkRateLimiterServiceTests.ts @@ -6,8 +6,9 @@ import { KEYWORD } from 'color-convert/conversions.js'; import { jest } from '@jest/globals'; import type Redis from 'ioredis'; -import { LegacyRateLimit, LimitCounter, RateLimit, SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; +import { LimitCounter, SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; import { LoggerService } from '@/core/LoggerService.js'; +import { BucketRateLimit, LegacyRateLimit } from '@/misc/rate-limit-utils.js'; /* eslint-disable @typescript-eslint/no-non-null-assertion */ /* eslint-disable @typescript-eslint/no-unnecessary-condition */ @@ -141,7 +142,7 @@ describe(SkRateLimiterService, () => { }); describe('with bucket limit', () => { - let limit: RateLimit = null!; + let limit: BucketRateLimit = null!; beforeEach(() => { limit = {