merge: Optionally log remote ActivityPub objects to database (!833)

View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/833

Approved-by: dakkar <dakkar@thenautilus.net>
Approved-by: Marie <github@yuugi.dev>
This commit is contained in:
Hazelnoot 2025-02-18 19:51:13 +00:00
commit c28b27b57f
28 changed files with 900 additions and 14 deletions

View file

@ -263,12 +263,17 @@ checkActivityPubGetSignature: false
# # default: false # # default: false
# disableQueryTruncation: false # disableQueryTruncation: false
# Log settings # Settings for the activity logger, which records inbound activities to the database.
# logging: # Disabled by default due to the large volume of data it saves.
# sql: #activityLogging:
# # Outputs query parameters during SQL execution to the log. # Log activities to the database (default: false)
# # default: false #enabled: false
# enableQueryParamLogging: false
# # Disable query truncation. If set to true, the full text of the query will be output to the log. # Save the activity before processing, then update later with the results.
# # default: false # This has the advantage of capturing activities that cause a hard-crash, but doubles the number of queries used.
# disableQueryTruncation: false # Default: false
#preSave: false
# How long to save each log entry before deleting it.
# Default: 2592000000 (1 week)
#maxAge: 2592000000

View file

@ -272,3 +272,18 @@ allowedPrivateNetworks: [
# # Disable query truncation. If set to true, the full text of the query will be output to the log. # # Disable query truncation. If set to true, the full text of the query will be output to the log.
# # default: false # # default: false
# disableQueryTruncation: false # disableQueryTruncation: false
# Settings for the activity logger, which records inbound activities to the database.
# Disabled by default due to the large volume of data it saves.
#activityLogging:
# Log activities to the database (default: false)
#enabled: false
# Save the activity before processing, then update later with the results.
# This has the advantage of capturing activities that cause a hard-crash, but doubles the number of queries used.
# Default: false
#preSave: false
# How long to save each log entry before deleting it.
# Default: 2592000000 (1 week)
#maxAge: 2592000000

View file

@ -345,3 +345,18 @@ checkActivityPubGetSignature: false
# # Disable query truncation. If set to true, the full text of the query will be output to the log. # # Disable query truncation. If set to true, the full text of the query will be output to the log.
# # default: false # # default: false
# disableQueryTruncation: false # disableQueryTruncation: false
# Settings for the activity logger, which records inbound activities to the database.
# Disabled by default due to the large volume of data it saves.
#activityLogging:
# Log activities to the database (default: false)
#enabled: false
# Save the activity before processing, then update later with the results.
# This has the advantage of capturing activities that cause a hard-crash, but doubles the number of queries used.
# Default: false
#preSave: false
# How long to save each log entry before deleting it.
# Default: 2592000000 (1 week)
#maxAge: 2592000000

View file

@ -383,3 +383,18 @@ checkActivityPubGetSignature: false
# # Disable query truncation. If set to true, the full text of the query will be output to the log. # # Disable query truncation. If set to true, the full text of the query will be output to the log.
# # default: false # # default: false
# disableQueryTruncation: false # disableQueryTruncation: false
# Settings for the activity logger, which records inbound activities to the database.
# Disabled by default due to the large volume of data it saves.
#activityLogging:
# Log activities to the database (default: false)
#enabled: false
# Save the activity before processing, then update later with the results.
# This has the advantage of capturing activities that cause a hard-crash, but doubles the number of queries used.
# Default: false
#preSave: false
# How long to save each log entry before deleting it.
# Default: 2592000000 (1 week)
#maxAge: 2592000000

View file

@ -0,0 +1,28 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class AddActivityLog1731565470048 {
name = 'AddActivityLog1731565470048'
async up(queryRunner) {
await queryRunner.query(`CREATE TABLE "activity_context" ("md5" text NOT NULL, "json" jsonb NOT NULL, CONSTRAINT "PK_activity_context" PRIMARY KEY ("md5"))`);
await queryRunner.query(`CREATE INDEX "IDK_activity_context_md5" ON "activity_context" ("md5") `);
await queryRunner.query(`CREATE TABLE "activity_log" ("id" character varying(32) NOT NULL, "at" TIMESTAMP WITH TIME ZONE NOT NULL, "key_id" text NOT NULL, "host" text NOT NULL, "verified" boolean NOT NULL, "accepted" boolean NOT NULL, "result" text NOT NULL, "activity" jsonb NOT NULL, "context_hash" text, "auth_user_id" character varying(32), CONSTRAINT "PK_activity_log" PRIMARY KEY ("id"))`);
await queryRunner.query(`CREATE INDEX "IDX_activity_log_at" ON "activity_log" ("at") `);
await queryRunner.query(`CREATE INDEX "IDX_activity_log_host" ON "activity_log" ("host") `);
await queryRunner.query(`ALTER TABLE "activity_log" ADD CONSTRAINT "FK_activity_log_context_hash" FOREIGN KEY ("context_hash") REFERENCES "activity_context"("md5") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "activity_log" ADD CONSTRAINT "FK_activity_log_auth_user_id" FOREIGN KEY ("auth_user_id") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "activity_log" DROP CONSTRAINT "FK_activity_log_auth_user_id"`);
await queryRunner.query(`ALTER TABLE "activity_log" DROP CONSTRAINT "FK_activity_log_context_hash"`);
await queryRunner.query(`DROP INDEX "public"."IDX_activity_log_host"`);
await queryRunner.query(`DROP INDEX "public"."IDX_activity_log_at"`);
await queryRunner.query(`DROP TABLE "activity_log"`);
await queryRunner.query(`DROP INDEX "public"."IDK_activity_context_md5"`);
await queryRunner.query(`DROP TABLE "activity_context"`);
}
}

View file

@ -0,0 +1,19 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class ActivityLogTiming1731909785724 {
name = 'ActivityLogTiming1731909785724'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "activity_log" ADD "duration" double precision NOT NULL DEFAULT '0'`);
await queryRunner.query(`ALTER TABLE "activity_log" ALTER COLUMN "result" DROP NOT NULL`);
}
async down(queryRunner) {
await queryRunner.query(`UPDATE "activity_log" SET "result" = 'not processed' WHERE "result" IS NULL`);
await queryRunner.query(`ALTER TABLE "activity_log" ALTER COLUMN "result" SET NOT NULL`);
await queryRunner.query(`ALTER TABLE "activity_log" DROP COLUMN "duration"`);
}
}

View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class RenameActivityLogIndexes1731910422761 {
name = 'RenameActivityLogIndexes1731910422761'
async up(queryRunner) {
await queryRunner.query(`DROP INDEX "public"."IDK_activity_context_md5"`);
}
async down(queryRunner) {
await queryRunner.query(`CREATE INDEX "IDK_activity_context_md5" ON "activity_context" ("md5") `);
}
}

View file

@ -0,0 +1,20 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class NullableActivityLogDuration1731935047347 {
name = 'NullableActivityLogDuration1731935047347'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "activity_log" ALTER COLUMN "duration" DROP NOT NULL`);
await queryRunner.query(`ALTER TABLE "activity_log" ALTER COLUMN "duration" DROP DEFAULT`);
await queryRunner.query(`UPDATE "activity_log" SET "duration" = NULL WHERE "duration" = 0`);
}
async down(queryRunner) {
await queryRunner.query(`UPDATE "activity_log" SET "duration" = 0 WHERE "duration" IS NULL`);
await queryRunner.query(`ALTER TABLE "activity_log" ALTER COLUMN "duration" SET DEFAULT '0'`);
await queryRunner.query(`ALTER TABLE "activity_log" ALTER COLUMN "duration" SET NOT NULL`);
}
}

View file

@ -0,0 +1,32 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class RenameActivityLogToApInboxLog1733756280460 {
name = 'RenameActivityLogToApInboxLog1733756280460'
async up(queryRunner) {
await queryRunner.query(`ALTER INDEX "IDX_activity_log_at" RENAME TO "IDX_ap_inbox_log_at"`);
await queryRunner.query(`ALTER INDEX "IDX_activity_log_host" RENAME TO "IDX_ap_inbox_log_host"`);
await queryRunner.query(`ALTER TABLE "activity_log" RENAME CONSTRAINT "PK_activity_log" TO "PK_ap_inbox_log"`);
await queryRunner.query(`ALTER TABLE "activity_log" RENAME CONSTRAINT "FK_activity_log_context_hash" TO "FK_ap_inbox_log_context_hash"`);
await queryRunner.query(`ALTER TABLE "activity_log" RENAME CONSTRAINT "FK_activity_log_auth_user_id" TO "FK_ap_inbox_log_auth_user_id"`);
await queryRunner.query(`ALTER TABLE "activity_log" RENAME TO "ap_inbox_log"`);
await queryRunner.query(`ALTER TABLE "activity_context" RENAME CONSTRAINT "PK_activity_context" TO "PK_ap_context"`);
await queryRunner.query(`ALTER TABLE "activity_context" RENAME TO "ap_context"`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "ap_context" RENAME TO "activity_context"`);
await queryRunner.query(`ALTER TABLE "activity_context" RENAME CONSTRAINT "PK_ap_context" TO "PK_activity_context"`);
await queryRunner.query(`ALTER TABLE "ap_inbox_log" RENAME TO "activity_log"`);
await queryRunner.query(`ALTER TABLE "activity_log" RENAME CONSTRAINT "FK_ap_inbox_log_auth_user_id" TO "FK_activity_log_auth_user_id"`);
await queryRunner.query(`ALTER TABLE "activity_log" RENAME CONSTRAINT "FK_ap_inbox_log_context_hash" TO "FK_activity_log_context_hash"`);
await queryRunner.query(`ALTER TABLE "activity_log" RENAME CONSTRAINT "PK_ap_inbox_log" TO "PK_activity_log"`);
await queryRunner.query(`ALTER INDEX "IDX_ap_inbox_log_host" RENAME TO "IDX_activity_log_host"`);
await queryRunner.query(`ALTER INDEX "IDX_ap_inbox_log_at" RENAME TO "IDX_activity_log_at"`);
}
}

View file

@ -0,0 +1,19 @@
export class CreateApFetchLog1738293576355 {
name = 'CreateApFetchLog1738293576355'
async up(queryRunner) {
await queryRunner.query(`CREATE TABLE "ap_fetch_log" ("id" character varying(32) NOT NULL, "at" TIMESTAMP WITH TIME ZONE NOT NULL, "duration" double precision, "host" text NOT NULL, "request_uri" text NOT NULL, "object_uri" text, "accepted" boolean, "result" text, "object" jsonb, "context_hash" text, CONSTRAINT "PK_ap_fetch_log" PRIMARY KEY ("id"))`);
await queryRunner.query(`CREATE INDEX "IDX_ap_fetch_log_at" ON "ap_fetch_log" ("at") `);
await queryRunner.query(`CREATE INDEX "IDX_ap_fetch_log_host" ON "ap_fetch_log" ("host") `);
await queryRunner.query(`CREATE INDEX "IDX_ap_fetch_log_object_uri" ON "ap_fetch_log" ("object_uri") `);
await queryRunner.query(`ALTER TABLE "ap_fetch_log" ADD CONSTRAINT "FK_ap_fetch_log_context_hash" FOREIGN KEY ("context_hash") REFERENCES "ap_context"("md5") ON DELETE CASCADE ON UPDATE NO ACTION`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "ap_fetch_log" DROP CONSTRAINT "FK_ap_fetch_log_context_hash"`);
await queryRunner.query(`DROP INDEX "public"."IDX_ap_fetch_log_object_uri"`);
await queryRunner.query(`DROP INDEX "public"."IDX_ap_fetch_log_host"`);
await queryRunner.query(`DROP INDEX "public"."IDX_ap_fetch_log_at"`);
await queryRunner.query(`DROP TABLE "ap_fetch_log"`);
}
}

View file

@ -13,6 +13,7 @@ import { ServerStatsService } from '@/daemons/ServerStatsService.js';
import { ServerService } from '@/server/ServerService.js'; import { ServerService } from '@/server/ServerService.js';
import { MainModule } from '@/MainModule.js'; import { MainModule } from '@/MainModule.js';
import { envOption } from '@/env.js'; import { envOption } from '@/env.js';
import { ApLogCleanupService } from '@/daemons/ApLogCleanupService.js';
export async function server() { export async function server() {
const app = await NestFactory.createApplicationContext(MainModule, { const app = await NestFactory.createApplicationContext(MainModule, {
@ -28,6 +29,7 @@ export async function server() {
if (!envOption.noDaemons) { if (!envOption.noDaemons) {
app.get(QueueStatsService).start(); app.get(QueueStatsService).start();
app.get(ServerStatsService).start(); app.get(ServerStatsService).start();
app.get(ApLogCleanupService).start();
} }
return app; return app;

View file

@ -129,6 +129,12 @@ type Source = {
enableQueryParamLogging? : boolean, enableQueryParamLogging? : boolean,
} }
} }
activityLogging?: {
enabled?: boolean;
preSave?: boolean;
maxAge?: number;
};
}; };
export type Config = { export type Config = {
@ -238,6 +244,12 @@ export type Config = {
pidFile: string; pidFile: string;
filePermissionBits?: string; filePermissionBits?: string;
activityLogging: {
enabled: boolean;
preSave: boolean;
maxAge: number;
};
}; };
export type FulltextSearchProvider = 'sqlLike' | 'sqlPgroonga' | 'meilisearch'; export type FulltextSearchProvider = 'sqlLike' | 'sqlPgroonga' | 'meilisearch';
@ -380,6 +392,11 @@ export function loadConfig(): Config {
pidFile: config.pidFile, pidFile: config.pidFile,
filePermissionBits: config.filePermissionBits, filePermissionBits: config.filePermissionBits,
logging: config.logging, logging: config.logging,
activityLogging: {
enabled: config.activityLogging?.enabled ?? false,
preSave: config.activityLogging?.preSave ?? false,
maxAge: config.activityLogging?.maxAge ?? (1000 * 60 * 60 * 24 * 30),
},
}; };
} }
@ -531,4 +548,5 @@ function applyEnvOverrides(config: Source) {
_apply_top(['import', ['downloadTimeout', 'maxFileSize']]); _apply_top(['import', ['downloadTimeout', 'maxFileSize']]);
_apply_top([['signToActivityPubGet', 'checkActivityPubGetSignature', 'setupPassword']]); _apply_top([['signToActivityPubGet', 'checkActivityPubGetSignature', 'setupPassword']]);
_apply_top(['logging', 'sql', ['disableQueryTruncation', 'enableQueryParamLogging']]); _apply_top(['logging', 'sql', ['disableQueryTruncation', 'enableQueryParamLogging']]);
_apply_top(['activityLogging', ['enabled', 'preSave', 'maxAge']]);
} }

View file

@ -0,0 +1,207 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { createHash } from 'crypto';
import { Inject, Injectable } from '@nestjs/common';
import { In, LessThan } from 'typeorm';
import { DI } from '@/di-symbols.js';
import { SkApFetchLog, SkApInboxLog, SkApContext } from '@/models/_.js';
import type { ApContextsRepository, ApFetchLogsRepository, ApInboxLogsRepository } from '@/models/_.js';
import type { Config } from '@/config.js';
import { JsonValue } from '@/misc/json-value.js';
import { UtilityService } from '@/core/UtilityService.js';
import { IdService } from '@/core/IdService.js';
import { IActivity, IObject } from './activitypub/type.js';
@Injectable()
export class ApLogService {
constructor(
@Inject(DI.config)
private readonly config: Config,
@Inject(DI.apContextsRepository)
private apContextsRepository: ApContextsRepository,
@Inject(DI.apInboxLogsRepository)
private readonly apInboxLogsRepository: ApInboxLogsRepository,
@Inject(DI.apFetchLogsRepository)
private readonly apFetchLogsRepository: ApFetchLogsRepository,
private readonly utilityService: UtilityService,
private readonly idService: IdService,
) {}
/**
* Creates an inbox log from an activity, and saves it if pre-save is enabled.
*/
public async createInboxLog(data: Partial<SkApInboxLog> & {
activity: IActivity,
keyId: string,
}): Promise<SkApInboxLog> {
const { object: activity, context, contextHash } = extractObjectContext(data.activity);
const host = this.utilityService.extractDbHost(data.keyId);
const log = new SkApInboxLog({
id: this.idService.gen(),
at: new Date(),
verified: false,
accepted: false,
host,
...data,
activity,
context,
contextHash,
});
if (this.config.activityLogging.preSave) {
await this.saveInboxLog(log);
}
return log;
}
/**
* Saves or finalizes an inbox log.
*/
public async saveInboxLog(log: SkApInboxLog): Promise<SkApInboxLog> {
if (log.context) {
await this.saveContext(log.context);
}
// Will be UPDATE with preSave, and INSERT without.
await this.apInboxLogsRepository.upsert(log, ['id']);
return log;
}
/**
* Creates a fetch log from an activity, and saves it if pre-save is enabled.
*/
public async createFetchLog(data: Partial<SkApFetchLog> & {
requestUri: string
host: string,
}): Promise<SkApFetchLog> {
const log = new SkApFetchLog({
id: this.idService.gen(),
at: new Date(),
accepted: false,
...data,
});
if (this.config.activityLogging.preSave) {
await this.saveFetchLog(log);
}
return log;
}
/**
* Saves or finalizes a fetch log.
*/
public async saveFetchLog(log: SkApFetchLog): Promise<SkApFetchLog> {
if (log.context) {
await this.saveContext(log.context);
}
// Will be UPDATE with preSave, and INSERT without.
await this.apFetchLogsRepository.upsert(log, ['id']);
return log;
}
private async saveContext(context: SkApContext): Promise<void> {
// https://stackoverflow.com/a/47064558
await this.apContextsRepository
.createQueryBuilder('activity_context')
.insert()
.into(SkApContext)
.values(context)
.orIgnore('md5')
.execute();
}
/**
* Deletes all logged copies of an object or objects
* @param objectUris URIs / AP IDs of the objects to delete
*/
public async deleteObjectLogs(objectUris: string | string[]): Promise<number> {
if (Array.isArray(objectUris)) {
const logsDeleted = await this.apFetchLogsRepository.delete({
objectUri: In(objectUris),
});
return logsDeleted.affected ?? 0;
} else {
const logsDeleted = await this.apFetchLogsRepository.delete({
objectUri: objectUris,
});
return logsDeleted.affected ?? 0;
}
}
/**
* Deletes all expired AP logs and garbage-collects the AP context cache.
* Returns the total number of deleted rows.
*/
public async deleteExpiredLogs(): Promise<number> {
// This is the date in UTC of the oldest log to KEEP
const oldestAllowed = new Date(Date.now() - this.config.activityLogging.maxAge);
// Delete all logs older than the threshold.
const inboxDeleted = await this.deleteExpiredInboxLogs(oldestAllowed);
const fetchDeleted = await this.deleteExpiredFetchLogs(oldestAllowed);
return inboxDeleted + fetchDeleted;
}
private async deleteExpiredInboxLogs(oldestAllowed: Date): Promise<number> {
const { affected } = await this.apInboxLogsRepository.delete({
at: LessThan(oldestAllowed),
});
return affected ?? 0;
}
private async deleteExpiredFetchLogs(oldestAllowed: Date): Promise<number> {
const { affected } = await this.apFetchLogsRepository.delete({
at: LessThan(oldestAllowed),
});
return affected ?? 0;
}
}
export function extractObjectContext<T extends IObject>(input: T) {
const object = Object.assign({}, input, { '@context': undefined }) as Omit<T, '@context'>;
const { context, contextHash } = parseContext(input['@context']);
return { object, context, contextHash };
}
export function parseContext(input: JsonValue | undefined): { contextHash: string | null, context: SkApContext | null } {
// Empty contexts are excluded for easier querying
if (input == null) {
return {
contextHash: null,
context: null,
};
}
const contextHash = createHash('md5').update(JSON.stringify(input)).digest('base64');
const context = new SkApContext({
md5: contextHash,
json: input,
});
return { contextHash, context };
}
export function calculateDurationSince(startTime: bigint): number {
// Calculate the processing time with correct rounding and decimals.
// 1. Truncate nanoseconds to microseconds
// 2. Scale to 1/10 millisecond ticks.
// 3. Round to nearest tick.
// 4. Sale to milliseconds
// Example: 123,456,789 ns -> 123,456 us -> 12,345.6 ticks -> 12,346 ticks -> 123.46 ms
const endTime = process.hrtime.bigint();
return Math.round(Number((endTime - startTime) / 1000n) / 10) / 100;
}

View file

@ -157,6 +157,7 @@ import { QueueService } from './QueueService.js';
import { LoggerService } from './LoggerService.js'; import { LoggerService } from './LoggerService.js';
import { SponsorsService } from './SponsorsService.js'; import { SponsorsService } from './SponsorsService.js';
import type { Provider } from '@nestjs/common'; import type { Provider } from '@nestjs/common';
import { ApLogService } from '@/core/ApLogService.js';
//#region 文字列ベースでのinjection用(循環参照対応のため) //#region 文字列ベースでのinjection用(循環参照対応のため)
const $LoggerService: Provider = { provide: 'LoggerService', useExisting: LoggerService }; const $LoggerService: Provider = { provide: 'LoggerService', useExisting: LoggerService };
@ -166,6 +167,7 @@ const $AccountMoveService: Provider = { provide: 'AccountMoveService', useExisti
const $AccountUpdateService: Provider = { provide: 'AccountUpdateService', useExisting: AccountUpdateService }; const $AccountUpdateService: Provider = { provide: 'AccountUpdateService', useExisting: AccountUpdateService };
const $AnnouncementService: Provider = { provide: 'AnnouncementService', useExisting: AnnouncementService }; const $AnnouncementService: Provider = { provide: 'AnnouncementService', useExisting: AnnouncementService };
const $AntennaService: Provider = { provide: 'AntennaService', useExisting: AntennaService }; const $AntennaService: Provider = { provide: 'AntennaService', useExisting: AntennaService };
const $ApLogService: Provider = { provide: 'ApLogService', useExisting: ApLogService };
const $AppLockService: Provider = { provide: 'AppLockService', useExisting: AppLockService }; const $AppLockService: Provider = { provide: 'AppLockService', useExisting: AppLockService };
const $AchievementService: Provider = { provide: 'AchievementService', useExisting: AchievementService }; const $AchievementService: Provider = { provide: 'AchievementService', useExisting: AchievementService };
const $AvatarDecorationService: Provider = { provide: 'AvatarDecorationService', useExisting: AvatarDecorationService }; const $AvatarDecorationService: Provider = { provide: 'AvatarDecorationService', useExisting: AvatarDecorationService };
@ -322,6 +324,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
AccountUpdateService, AccountUpdateService,
AnnouncementService, AnnouncementService,
AntennaService, AntennaService,
ApLogService,
AppLockService, AppLockService,
AchievementService, AchievementService,
AvatarDecorationService, AvatarDecorationService,
@ -474,6 +477,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
$AccountUpdateService, $AccountUpdateService,
$AnnouncementService, $AnnouncementService,
$AntennaService, $AntennaService,
$ApLogService,
$AppLockService, $AppLockService,
$AchievementService, $AchievementService,
$AvatarDecorationService, $AvatarDecorationService,
@ -627,6 +631,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
AccountUpdateService, AccountUpdateService,
AnnouncementService, AnnouncementService,
AntennaService, AntennaService,
ApLogService,
AppLockService, AppLockService,
AchievementService, AchievementService,
AvatarDecorationService, AvatarDecorationService,
@ -778,6 +783,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
$AccountUpdateService, $AccountUpdateService,
$AnnouncementService, $AnnouncementService,
$AntennaService, $AntennaService,
$ApLogService,
$AppLockService, $AppLockService,
$AchievementService, $AchievementService,
$AvatarDecorationService, $AvatarDecorationService,

View file

@ -24,9 +24,14 @@ import { SearchService } from '@/core/SearchService.js';
import { ModerationLogService } from '@/core/ModerationLogService.js'; import { ModerationLogService } from '@/core/ModerationLogService.js';
import { isQuote, isRenote } from '@/misc/is-renote.js'; import { isQuote, isRenote } from '@/misc/is-renote.js';
import { LatestNoteService } from '@/core/LatestNoteService.js'; import { LatestNoteService } from '@/core/LatestNoteService.js';
import { ApLogService } from '@/core/ApLogService.js';
import Logger from '@/logger.js';
import { LoggerService } from './LoggerService.js';
@Injectable() @Injectable()
export class NoteDeleteService { export class NoteDeleteService {
private readonly logger: Logger;
constructor( constructor(
@Inject(DI.config) @Inject(DI.config)
private config: Config, private config: Config,
@ -55,7 +60,11 @@ export class NoteDeleteService {
private perUserNotesChart: PerUserNotesChart, private perUserNotesChart: PerUserNotesChart,
private instanceChart: InstanceChart, private instanceChart: InstanceChart,
private latestNoteService: LatestNoteService, private latestNoteService: LatestNoteService,
) {} private readonly apLogService: ApLogService,
loggerService: LoggerService,
) {
this.logger = loggerService.getLogger('note-delete-service');
}
/** /**
* 稿 * 稿
@ -156,6 +165,11 @@ export class NoteDeleteService {
note: note, note: note,
}); });
} }
if (note.uri) {
this.apLogService.deleteObjectLogs(note.uri)
.catch(err => this.logger.error(err, `Failed to delete AP logs for note '${note.uri}'`));
}
} }
@bindThis @bindThis

View file

@ -7,7 +7,7 @@ import { Inject, Injectable } from '@nestjs/common';
import { IsNull, Not } from 'typeorm'; import { IsNull, Not } from 'typeorm';
import type { MiLocalUser, MiRemoteUser } from '@/models/User.js'; import type { MiLocalUser, MiRemoteUser } from '@/models/User.js';
import { InstanceActorService } from '@/core/InstanceActorService.js'; import { InstanceActorService } from '@/core/InstanceActorService.js';
import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository, FollowRequestsRepository, MiMeta } from '@/models/_.js'; import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository, FollowRequestsRepository, MiMeta, SkApFetchLog } from '@/models/_.js';
import type { Config } from '@/config.js'; import type { Config } from '@/config.js';
import { HttpRequestService } from '@/core/HttpRequestService.js'; import { HttpRequestService } from '@/core/HttpRequestService.js';
import { DI } from '@/di-symbols.js'; import { DI } from '@/di-symbols.js';
@ -17,7 +17,8 @@ import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js'; import type Logger from '@/logger.js';
import { fromTuple } from '@/misc/from-tuple.js'; import { fromTuple } from '@/misc/from-tuple.js';
import { IdentifiableError } from '@/misc/identifiable-error.js'; import { IdentifiableError } from '@/misc/identifiable-error.js';
import { isCollectionOrOrderedCollection } from './type.js'; import { ApLogService, calculateDurationSince, extractObjectContext } from '@/core/ApLogService.js';
import { getNullableApId, isCollectionOrOrderedCollection } from './type.js';
import { ApDbResolverService } from './ApDbResolverService.js'; import { ApDbResolverService } from './ApDbResolverService.js';
import { ApRendererService } from './ApRendererService.js'; import { ApRendererService } from './ApRendererService.js';
import { ApRequestService } from './ApRequestService.js'; import { ApRequestService } from './ApRequestService.js';
@ -43,6 +44,7 @@ export class Resolver {
private apRendererService: ApRendererService, private apRendererService: ApRendererService,
private apDbResolverService: ApDbResolverService, private apDbResolverService: ApDbResolverService,
private loggerService: LoggerService, private loggerService: LoggerService,
private readonly apLogService: ApLogService,
private recursionLimit = 256, private recursionLimit = 256,
) { ) {
this.history = new Set(); this.history = new Set();
@ -81,6 +83,44 @@ export class Resolver {
return value; return value;
} }
const host = this.utilityService.extractDbHost(value);
if (this.config.activityLogging.enabled && !this.utilityService.isSelfHost(host)) {
return await this._resolveLogged(value, host);
} else {
return await this._resolve(value, host);
}
}
private async _resolveLogged(requestUri: string, host: string): Promise<IObject> {
const startTime = process.hrtime.bigint();
const log = await this.apLogService.createFetchLog({
host: host,
requestUri,
});
try {
const result = await this._resolve(requestUri, host, log);
log.accepted = true;
log.result = 'ok';
return result;
} catch (err) {
log.accepted = false;
log.result = String(err);
throw err;
} finally {
log.duration = calculateDurationSince(startTime);
// Save or finalize asynchronously
this.apLogService.saveFetchLog(log)
.catch(err => this.logger.error('Failed to record AP object fetch:', err));
}
}
private async _resolve(value: string, host: string, log?: SkApFetchLog): Promise<IObject> {
if (value.includes('#')) { if (value.includes('#')) {
// URLs with fragment parts cannot be resolved correctly because // URLs with fragment parts cannot be resolved correctly because
// the fragment part does not get transmitted over HTTP(S). // the fragment part does not get transmitted over HTTP(S).
@ -98,7 +138,6 @@ export class Resolver {
this.history.add(value); this.history.add(value);
const host = this.utilityService.extractDbHost(value);
if (this.utilityService.isSelfHost(host)) { if (this.utilityService.isSelfHost(host)) {
return await this.resolveLocal(value); return await this.resolveLocal(value);
} }
@ -115,6 +154,20 @@ export class Resolver {
? await this.apRequestService.signedGet(value, this.user) as IObject ? await this.apRequestService.signedGet(value, this.user) as IObject
: await this.httpRequestService.getActivityJson(value)) as IObject; : await this.httpRequestService.getActivityJson(value)) as IObject;
if (log) {
const { object: objectOnly, context, contextHash } = extractObjectContext(object);
const objectUri = getNullableApId(object);
if (objectUri) {
log.objectUri = objectUri;
log.host = this.utilityService.extractDbHost(objectUri);
}
log.object = objectOnly;
log.context = context;
log.contextHash = contextHash;
}
if ( if (
Array.isArray(object['@context']) ? Array.isArray(object['@context']) ?
!(object['@context'] as unknown[]).includes('https://www.w3.org/ns/activitystreams') : !(object['@context'] as unknown[]).includes('https://www.w3.org/ns/activitystreams') :
@ -232,6 +285,7 @@ export class ApResolverService {
private apRendererService: ApRendererService, private apRendererService: ApRendererService,
private apDbResolverService: ApDbResolverService, private apDbResolverService: ApDbResolverService,
private loggerService: LoggerService, private loggerService: LoggerService,
private readonly apLogService: ApLogService,
) { ) {
} }
@ -252,6 +306,7 @@ export class ApResolverService {
this.apRendererService, this.apRendererService,
this.apDbResolverService, this.apDbResolverService,
this.loggerService, this.loggerService,
this.apLogService,
); );
} }
} }

View file

@ -0,0 +1,62 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Injectable, type OnApplicationShutdown } from '@nestjs/common';
import { bindThis } from '@/decorators.js';
import { LoggerService } from '@/core/LoggerService.js';
import Logger from '@/logger.js';
import { ApLogService } from '@/core/ApLogService.js';
// 10 minutes
export const scanInterval = 1000 * 60 * 10;
@Injectable()
export class ApLogCleanupService implements OnApplicationShutdown {
private readonly logger: Logger;
private scanTimer: NodeJS.Timeout | null = null;
constructor(
private readonly apLogService: ApLogService,
loggerService: LoggerService,
) {
this.logger = loggerService.getLogger('activity-log-cleanup');
}
@bindThis
public async start(): Promise<void> {
// Just in case start() gets called multiple times.
this.dispose();
// Prune at startup, in case the server was rebooted during the interval.
// noinspection ES6MissingAwait
this.tick();
// Prune on a regular interval for the lifetime of the server.
this.scanTimer = setInterval(this.tick, scanInterval);
}
@bindThis
private async tick(): Promise<void> {
try {
const affected = await this.apLogService.deleteExpiredLogs();
this.logger.info(`Activity Log cleanup complete; removed ${affected} expired logs.`);
} catch (err) {
this.logger.error('Activity Log cleanup failed:', err as Error);
}
}
@bindThis
public onApplicationShutdown(): void {
this.dispose();
}
@bindThis
public dispose(): void {
if (this.scanTimer) {
clearInterval(this.scanTimer);
this.scanTimer = null;
}
}
}

View file

@ -8,6 +8,7 @@ import { CoreModule } from '@/core/CoreModule.js';
import { GlobalModule } from '@/GlobalModule.js'; import { GlobalModule } from '@/GlobalModule.js';
import { QueueStatsService } from './QueueStatsService.js'; import { QueueStatsService } from './QueueStatsService.js';
import { ServerStatsService } from './ServerStatsService.js'; import { ServerStatsService } from './ServerStatsService.js';
import { ApLogCleanupService } from './ApLogCleanupService.js';
@Module({ @Module({
imports: [ imports: [
@ -17,10 +18,12 @@ import { ServerStatsService } from './ServerStatsService.js';
providers: [ providers: [
QueueStatsService, QueueStatsService,
ServerStatsService, ServerStatsService,
ApLogCleanupService,
], ],
exports: [ exports: [
QueueStatsService, QueueStatsService,
ServerStatsService, ServerStatsService,
ApLogCleanupService,
], ],
}) })
export class DaemonModule {} export class DaemonModule {}

View file

@ -22,6 +22,9 @@ export const DI = {
appsRepository: Symbol('appsRepository'), appsRepository: Symbol('appsRepository'),
avatarDecorationsRepository: Symbol('avatarDecorationsRepository'), avatarDecorationsRepository: Symbol('avatarDecorationsRepository'),
latestNotesRepository: Symbol('latestNotesRepository'), latestNotesRepository: Symbol('latestNotesRepository'),
apContextsRepository: Symbol('apContextsRepository'),
apFetchLogsRepository: Symbol('apFetchLogsRepository'),
apInboxLogsRepository: Symbol('apInboxLogsRepository'),
noteFavoritesRepository: Symbol('noteFavoritesRepository'), noteFavoritesRepository: Symbol('noteFavoritesRepository'),
noteThreadMutingsRepository: Symbol('noteThreadMutingsRepository'), noteThreadMutingsRepository: Symbol('noteThreadMutingsRepository'),
noteReactionsRepository: Symbol('noteReactionsRepository'), noteReactionsRepository: Symbol('noteReactionsRepository'),

View file

@ -80,7 +80,10 @@ import {
MiUserPublickey, MiUserPublickey,
MiUserSecurityKey, MiUserSecurityKey,
MiWebhook, MiWebhook,
NoteEdit NoteEdit,
SkApContext,
SkApFetchLog,
SkApInboxLog,
} from './_.js'; } from './_.js';
import type { DataSource } from 'typeorm'; import type { DataSource } from 'typeorm';
@ -126,6 +129,24 @@ const $latestNotesRepository: Provider = {
inject: [DI.db], inject: [DI.db],
}; };
const $apContextRepository: Provider = {
provide: DI.apContextsRepository,
useFactory: (db: DataSource) => db.getRepository(SkApContext).extend(miRepository as MiRepository<SkApContext>),
inject: [DI.db],
};
const $apFetchLogsRepository: Provider = {
provide: DI.apFetchLogsRepository,
useFactory: (db: DataSource) => db.getRepository(SkApFetchLog).extend(miRepository as MiRepository<SkApFetchLog>),
inject: [DI.db],
};
const $apInboxLogsRepository: Provider = {
provide: DI.apInboxLogsRepository,
useFactory: (db: DataSource) => db.getRepository(SkApInboxLog).extend(miRepository as MiRepository<SkApInboxLog>),
inject: [DI.db],
};
const $noteFavoritesRepository: Provider = { const $noteFavoritesRepository: Provider = {
provide: DI.noteFavoritesRepository, provide: DI.noteFavoritesRepository,
useFactory: (db: DataSource) => db.getRepository(MiNoteFavorite).extend(miRepository as MiRepository<MiNoteFavorite>), useFactory: (db: DataSource) => db.getRepository(MiNoteFavorite).extend(miRepository as MiRepository<MiNoteFavorite>),
@ -526,6 +547,9 @@ const $noteScheduleRepository: Provider = {
$appsRepository, $appsRepository,
$avatarDecorationsRepository, $avatarDecorationsRepository,
$latestNotesRepository, $latestNotesRepository,
$apContextRepository,
$apFetchLogsRepository,
$apInboxLogsRepository,
$noteFavoritesRepository, $noteFavoritesRepository,
$noteThreadMutingsRepository, $noteThreadMutingsRepository,
$noteReactionsRepository, $noteReactionsRepository,
@ -600,6 +624,9 @@ const $noteScheduleRepository: Provider = {
$appsRepository, $appsRepository,
$avatarDecorationsRepository, $avatarDecorationsRepository,
$latestNotesRepository, $latestNotesRepository,
$apContextRepository,
$apFetchLogsRepository,
$apInboxLogsRepository,
$noteFavoritesRepository, $noteFavoritesRepository,
$noteThreadMutingsRepository, $noteThreadMutingsRepository,
$noteReactionsRepository, $noteReactionsRepository,

View file

@ -0,0 +1,25 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Column, PrimaryColumn, Entity } from 'typeorm';
@Entity('ap_context')
export class SkApContext {
@PrimaryColumn('text', {
primaryKeyConstraintName: 'PK_ap_context',
})
public md5: string;
@Column('jsonb')
// https://github.com/typeorm/typeorm/issues/8559
// eslint-disable-next-line @typescript-eslint/no-explicit-any
public json: any;
constructor(data?: Partial<SkApContext>) {
if (data) {
Object.assign(this, data);
}
}
}

View file

@ -0,0 +1,89 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Column, Index, JoinColumn, ManyToOne, PrimaryColumn, Entity } from 'typeorm';
import { SkApContext } from '@/models/SkApContext.js';
import { id } from './util/id.js';
/**
* Records objects fetched via AP
*/
@Entity('ap_fetch_log')
export class SkApFetchLog {
@PrimaryColumn({
...id(),
primaryKeyConstraintName: 'PK_ap_fetch_log',
})
public id: string;
@Index('IDX_ap_fetch_log_at')
@Column('timestamptz')
public at: Date;
/**
* Processing duration in milliseconds
*/
@Column('double precision', { nullable: true })
public duration: number | null = null;
/**
* DB hostname extracted from responseUri, or requestUri if fetch is incomplete
*/
@Index('IDX_ap_fetch_log_host')
@Column('text')
public host: string;
/**
* Original requested URI
*/
@Column('text', {
name: 'request_uri',
})
public requestUri: string;
/**
* Canonical URI / object ID, taken from the final payload
*/
@Column('text', {
name: 'object_uri',
nullable: true,
})
@Index('IDX_ap_fetch_log_object_uri')
public objectUri: string | null = null;
@Column('boolean', { nullable: true })
public accepted: boolean | null = null;
@Column('text', { nullable: true })
public result: string | null = null;
@Column('jsonb', { nullable: true })
// https://github.com/typeorm/typeorm/issues/8559
// eslint-disable-next-line @typescript-eslint/no-explicit-any
public object: any | null = null;
@Column({
type: 'text',
name: 'context_hash',
nullable: true,
})
public contextHash: string | null;
@ManyToOne(() => SkApContext, {
onDelete: 'CASCADE',
nullable: true,
})
@JoinColumn({
name: 'context_hash',
foreignKeyConstraintName: 'FK_ap_fetch_log_context_hash',
})
public context: SkApContext | null;
constructor(data?: Partial<SkApFetchLog>) {
if (data) {
Object.assign(this, data);
}
}
}

View file

@ -0,0 +1,109 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Column, Entity, Index, JoinColumn, ManyToOne, PrimaryColumn } from 'typeorm';
import { SkApContext } from '@/models/SkApContext.js';
import { MiUser } from '@/models/_.js';
import { id } from './util/id.js';
/**
* Records activities received in the inbox
*/
@Entity('ap_inbox_log')
export class SkApInboxLog {
@PrimaryColumn({
...id(),
primaryKeyConstraintName: 'PK_ap_inbox_log',
})
public id: string;
@Index('IDX_ap_inbox_log_at')
@Column('timestamptz')
public at: Date;
/**
* Processing duration in milliseconds
*/
@Column('double precision', { nullable: true })
public duration: number | null = null;
/**
* Key ID that was used to sign this request.
* Untrusted unless verified is true.
*/
@Column({
type: 'text',
name: 'key_id',
})
public keyId: string;
/**
* Instance that the activity was sent from.
* Untrusted unless verified is true.
*/
@Index('IDX_ap_inbox_log_host')
@Column('text')
public host: string;
@Column('boolean')
public verified: boolean;
@Column('boolean')
public accepted: boolean;
@Column('text', { nullable: true })
public result: string | null = null;
@Column('jsonb')
// https://github.com/typeorm/typeorm/issues/8559
// eslint-disable-next-line @typescript-eslint/no-explicit-any
public activity: any;
@Column({
type: 'text',
name: 'context_hash',
nullable: true,
})
public contextHash: string | null;
@ManyToOne(() => SkApContext, {
onDelete: 'CASCADE',
nullable: true,
})
@JoinColumn({
name: 'context_hash',
foreignKeyConstraintName: 'FK_ap_inbox_log_context_hash',
})
public context: SkApContext | null;
/**
* ID of the user who signed this request.
*/
@Column({
...id(),
name: 'auth_user_id',
nullable: true,
})
public authUserId: string | null;
/**
* User who signed this request.
*/
@ManyToOne(() => MiUser, {
onDelete: 'CASCADE',
nullable: true,
})
@JoinColumn({
name: 'auth_user_id',
foreignKeyConstraintName: 'FK_ap_inbox_log_auth_user_id',
})
public authUser: MiUser | null;
constructor(data?: Partial<SkApInboxLog>) {
if (data) {
Object.assign(this, data);
}
}
}

View file

@ -82,6 +82,9 @@ import { NoteEdit } from '@/models/NoteEdit.js';
import { MiBubbleGameRecord } from '@/models/BubbleGameRecord.js'; import { MiBubbleGameRecord } from '@/models/BubbleGameRecord.js';
import { MiReversiGame } from '@/models/ReversiGame.js'; import { MiReversiGame } from '@/models/ReversiGame.js';
import { MiNoteSchedule } from '@/models/NoteSchedule.js'; import { MiNoteSchedule } from '@/models/NoteSchedule.js';
import { SkApInboxLog } from '@/models/SkApInboxLog.js';
import { SkApFetchLog } from '@/models/SkApFetchLog.js';
import { SkApContext } from '@/models/SkApContext.js';
import type { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity.js'; import type { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity.js';
export interface MiRepository<T extends ObjectLiteral> { export interface MiRepository<T extends ObjectLiteral> {
@ -129,6 +132,9 @@ export const miRepository = {
export { export {
SkLatestNote, SkLatestNote,
SkApContext,
SkApFetchLog,
SkApInboxLog,
MiAbuseUserReport, MiAbuseUserReport,
MiAbuseReportNotificationRecipient, MiAbuseReportNotificationRecipient,
MiAccessToken, MiAccessToken,
@ -229,6 +235,9 @@ export type HashtagsRepository = Repository<MiHashtag> & MiRepository<MiHashtag>
export type InstancesRepository = Repository<MiInstance> & MiRepository<MiInstance>; export type InstancesRepository = Repository<MiInstance> & MiRepository<MiInstance>;
export type MetasRepository = Repository<MiMeta> & MiRepository<MiMeta>; export type MetasRepository = Repository<MiMeta> & MiRepository<MiMeta>;
export type LatestNotesRepository = Repository<SkLatestNote> & MiRepository<SkLatestNote>; export type LatestNotesRepository = Repository<SkLatestNote> & MiRepository<SkLatestNote>;
export type ApContextsRepository = Repository<SkApContext> & MiRepository<SkApContext>;
export type ApFetchLogsRepository = Repository<SkApFetchLog> & MiRepository<SkApFetchLog>;
export type ApInboxLogsRepository = Repository<SkApInboxLog> & MiRepository<SkApInboxLog>;
export type ModerationLogsRepository = Repository<MiModerationLog> & MiRepository<MiModerationLog>; export type ModerationLogsRepository = Repository<MiModerationLog> & MiRepository<MiModerationLog>;
export type MutingsRepository = Repository<MiMuting> & MiRepository<MiMuting>; export type MutingsRepository = Repository<MiMuting> & MiRepository<MiMuting>;
export type RenoteMutingsRepository = Repository<MiRenoteMuting> & MiRepository<MiRenoteMuting>; export type RenoteMutingsRepository = Repository<MiRenoteMuting> & MiRepository<MiRenoteMuting>;

View file

@ -85,6 +85,9 @@ import { Config } from '@/config.js';
import MisskeyLogger from '@/logger.js'; import MisskeyLogger from '@/logger.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { SkLatestNote } from '@/models/LatestNote.js'; import { SkLatestNote } from '@/models/LatestNote.js';
import { SkApContext } from '@/models/SkApContext.js';
import { SkApFetchLog } from '@/models/SkApFetchLog.js';
import { SkApInboxLog } from '@/models/SkApInboxLog.js';
pg.types.setTypeParser(20, Number); pg.types.setTypeParser(20, Number);
@ -171,6 +174,9 @@ class MyCustomLogger implements Logger {
export const entities = [ export const entities = [
SkLatestNote, SkLatestNote,
SkApContext,
SkApFetchLog,
SkApInboxLog,
MiAnnouncement, MiAnnouncement,
MiAnnouncementRead, MiAnnouncementRead,
MiMeta, MiMeta,

View file

@ -15,6 +15,7 @@ import type { MiNoteReaction } from '@/models/NoteReaction.js';
import { EmailService } from '@/core/EmailService.js'; import { EmailService } from '@/core/EmailService.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { SearchService } from '@/core/SearchService.js'; import { SearchService } from '@/core/SearchService.js';
import { ApLogService } from '@/core/ApLogService.js';
import { ReactionService } from '@/core/ReactionService.js'; import { ReactionService } from '@/core/ReactionService.js';
import { QueueLoggerService } from '../QueueLoggerService.js'; import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq'; import type * as Bull from 'bullmq';
@ -45,6 +46,7 @@ export class DeleteAccountProcessorService {
private queueLoggerService: QueueLoggerService, private queueLoggerService: QueueLoggerService,
private searchService: SearchService, private searchService: SearchService,
private reactionService: ReactionService, private reactionService: ReactionService,
private readonly apLogService: ApLogService,
) { ) {
this.logger = this.queueLoggerService.logger.createSubLogger('delete-account'); this.logger = this.queueLoggerService.logger.createSubLogger('delete-account');
} }
@ -84,6 +86,13 @@ export class DeleteAccountProcessorService {
for (const note of notes) { for (const note of notes) {
await this.searchService.unindexNote(note); await this.searchService.unindexNote(note);
} }
// Delete note AP logs
const noteUris = notes.map(n => n.uri).filter(u => !!u) as string[];
if (noteUris.length > 0) {
await this.apLogService.deleteObjectLogs(noteUris)
.catch(err => this.logger.error(err, `Failed to delete AP logs for notes of user '${user.uri ?? user.id}'`));
}
} }
this.logger.succ('All of notes deleted'); this.logger.succ('All of notes deleted');
@ -149,6 +158,13 @@ export class DeleteAccountProcessorService {
this.logger.succ('All of files deleted'); this.logger.succ('All of files deleted');
} }
{ // Delete actor logs
if (user.uri) {
await this.apLogService.deleteObjectLogs(user.uri)
.catch(err => this.logger.error(err, `Failed to delete AP logs for user '${user.uri}'`));
}
}
{ // Send email notification { // Send email notification
const profile = await this.userProfilesRepository.findOneByOrFail({ userId: user.id }); const profile = await this.userProfilesRepository.findOneByOrFail({ userId: user.id });
if (profile.email && profile.emailVerified) { if (profile.email && profile.emailVerified) {

View file

@ -29,6 +29,9 @@ import { CollapsedQueue } from '@/misc/collapsed-queue.js';
import { MiNote } from '@/models/Note.js'; import { MiNote } from '@/models/Note.js';
import { MiMeta } from '@/models/Meta.js'; import { MiMeta } from '@/models/Meta.js';
import { DI } from '@/di-symbols.js'; import { DI } from '@/di-symbols.js';
import { SkApInboxLog } from '@/models/_.js';
import type { Config } from '@/config.js';
import { ApLogService, calculateDurationSince } from '@/core/ApLogService.js';
import { QueueLoggerService } from '../QueueLoggerService.js'; import { QueueLoggerService } from '../QueueLoggerService.js';
import type { InboxJobData } from '../types.js'; import type { InboxJobData } from '../types.js';
@ -46,6 +49,9 @@ export class InboxProcessorService implements OnApplicationShutdown {
@Inject(DI.meta) @Inject(DI.meta)
private meta: MiMeta, private meta: MiMeta,
@Inject(DI.config)
private config: Config,
private utilityService: UtilityService, private utilityService: UtilityService,
private apInboxService: ApInboxService, private apInboxService: ApInboxService,
private federatedInstanceService: FederatedInstanceService, private federatedInstanceService: FederatedInstanceService,
@ -57,6 +63,7 @@ export class InboxProcessorService implements OnApplicationShutdown {
private apRequestChart: ApRequestChart, private apRequestChart: ApRequestChart,
private federationChart: FederationChart, private federationChart: FederationChart,
private queueLoggerService: QueueLoggerService, private queueLoggerService: QueueLoggerService,
private readonly apLogService: ApLogService,
) { ) {
this.logger = this.queueLoggerService.logger.createSubLogger('inbox'); this.logger = this.queueLoggerService.logger.createSubLogger('inbox');
this.updateInstanceQueue = new CollapsedQueue(process.env.NODE_ENV !== 'test' ? 60 * 1000 * 5 : 0, this.collapseUpdateInstanceJobs, this.performUpdateInstance); this.updateInstanceQueue = new CollapsedQueue(process.env.NODE_ENV !== 'test' ? 60 * 1000 * 5 : 0, this.collapseUpdateInstanceJobs, this.performUpdateInstance);
@ -64,6 +71,41 @@ export class InboxProcessorService implements OnApplicationShutdown {
@bindThis @bindThis
public async process(job: Bull.Job<InboxJobData>): Promise<string> { public async process(job: Bull.Job<InboxJobData>): Promise<string> {
if (this.config.activityLogging.enabled) {
return await this._processLogged(job);
} else {
return await this._process(job);
}
}
private async _processLogged(job: Bull.Job<InboxJobData>): Promise<string> {
const startTime = process.hrtime.bigint();
const activity = job.data.activity;
const keyId = job.data.signature.keyId;
const log = await this.apLogService.createInboxLog({ activity, keyId });
try {
const result = await this._process(job, log);
log.accepted = result.startsWith('ok');
log.result = result;
return result;
} catch (err) {
log.accepted = false;
log.result = String(err);
throw err;
} finally {
log.duration = calculateDurationSince(startTime);
// Save or finalize asynchronously
this.apLogService.saveInboxLog(log)
.catch(err => this.logger.error('Failed to record AP activity:', err));
}
}
private async _process(job: Bull.Job<InboxJobData>, log?: SkApInboxLog): Promise<string> {
const signature = job.data.signature; // HTTP-signature const signature = job.data.signature; // HTTP-signature
let activity = job.data.activity; let activity = job.data.activity;
@ -197,6 +239,13 @@ export class InboxProcessorService implements OnApplicationShutdown {
delete activity.id; delete activity.id;
} }
// Record verified user in log
if (log) {
log.verified = true;
log.authUser = authUser.user;
log.authUserId = authUser.user.id;
}
this.apRequestChart.inbox(); this.apRequestChart.inbox();
this.federationChart.inbox(authUser.user.host); this.federationChart.inbox(authUser.user.host);

View file

@ -23,6 +23,7 @@ import type {
PollsRepository, PollsRepository,
UsersRepository, UsersRepository,
} from '@/models/_.js'; } from '@/models/_.js';
import { ApLogService } from '@/core/ApLogService.js';
type MockResponse = { type MockResponse = {
type: string; type: string;
@ -49,6 +50,7 @@ export class MockResolver extends Resolver {
{} as ApRendererService, {} as ApRendererService,
{} as ApDbResolverService, {} as ApDbResolverService,
loggerService, loggerService,
{} as ApLogService,
); );
} }