merge: Optionally log remote ActivityPub objects to database (!833)
View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/833 Approved-by: dakkar <dakkar@thenautilus.net> Approved-by: Marie <github@yuugi.dev>
This commit is contained in:
commit
c28b27b57f
28 changed files with 900 additions and 14 deletions
|
@ -263,12 +263,17 @@ checkActivityPubGetSignature: false
|
||||||
# # default: false
|
# # default: false
|
||||||
# disableQueryTruncation: false
|
# disableQueryTruncation: false
|
||||||
|
|
||||||
# Log settings
|
# Settings for the activity logger, which records inbound activities to the database.
|
||||||
# logging:
|
# Disabled by default due to the large volume of data it saves.
|
||||||
# sql:
|
#activityLogging:
|
||||||
# # Outputs query parameters during SQL execution to the log.
|
# Log activities to the database (default: false)
|
||||||
# # default: false
|
#enabled: false
|
||||||
# enableQueryParamLogging: false
|
|
||||||
# # Disable query truncation. If set to true, the full text of the query will be output to the log.
|
# Save the activity before processing, then update later with the results.
|
||||||
# # default: false
|
# This has the advantage of capturing activities that cause a hard-crash, but doubles the number of queries used.
|
||||||
# disableQueryTruncation: false
|
# Default: false
|
||||||
|
#preSave: false
|
||||||
|
|
||||||
|
# How long to save each log entry before deleting it.
|
||||||
|
# Default: 2592000000 (1 week)
|
||||||
|
#maxAge: 2592000000
|
||||||
|
|
|
@ -272,3 +272,18 @@ allowedPrivateNetworks: [
|
||||||
# # Disable query truncation. If set to true, the full text of the query will be output to the log.
|
# # Disable query truncation. If set to true, the full text of the query will be output to the log.
|
||||||
# # default: false
|
# # default: false
|
||||||
# disableQueryTruncation: false
|
# disableQueryTruncation: false
|
||||||
|
|
||||||
|
# Settings for the activity logger, which records inbound activities to the database.
|
||||||
|
# Disabled by default due to the large volume of data it saves.
|
||||||
|
#activityLogging:
|
||||||
|
# Log activities to the database (default: false)
|
||||||
|
#enabled: false
|
||||||
|
|
||||||
|
# Save the activity before processing, then update later with the results.
|
||||||
|
# This has the advantage of capturing activities that cause a hard-crash, but doubles the number of queries used.
|
||||||
|
# Default: false
|
||||||
|
#preSave: false
|
||||||
|
|
||||||
|
# How long to save each log entry before deleting it.
|
||||||
|
# Default: 2592000000 (1 week)
|
||||||
|
#maxAge: 2592000000
|
||||||
|
|
|
@ -345,3 +345,18 @@ checkActivityPubGetSignature: false
|
||||||
# # Disable query truncation. If set to true, the full text of the query will be output to the log.
|
# # Disable query truncation. If set to true, the full text of the query will be output to the log.
|
||||||
# # default: false
|
# # default: false
|
||||||
# disableQueryTruncation: false
|
# disableQueryTruncation: false
|
||||||
|
|
||||||
|
# Settings for the activity logger, which records inbound activities to the database.
|
||||||
|
# Disabled by default due to the large volume of data it saves.
|
||||||
|
#activityLogging:
|
||||||
|
# Log activities to the database (default: false)
|
||||||
|
#enabled: false
|
||||||
|
|
||||||
|
# Save the activity before processing, then update later with the results.
|
||||||
|
# This has the advantage of capturing activities that cause a hard-crash, but doubles the number of queries used.
|
||||||
|
# Default: false
|
||||||
|
#preSave: false
|
||||||
|
|
||||||
|
# How long to save each log entry before deleting it.
|
||||||
|
# Default: 2592000000 (1 week)
|
||||||
|
#maxAge: 2592000000
|
||||||
|
|
|
@ -383,3 +383,18 @@ checkActivityPubGetSignature: false
|
||||||
# # Disable query truncation. If set to true, the full text of the query will be output to the log.
|
# # Disable query truncation. If set to true, the full text of the query will be output to the log.
|
||||||
# # default: false
|
# # default: false
|
||||||
# disableQueryTruncation: false
|
# disableQueryTruncation: false
|
||||||
|
|
||||||
|
# Settings for the activity logger, which records inbound activities to the database.
|
||||||
|
# Disabled by default due to the large volume of data it saves.
|
||||||
|
#activityLogging:
|
||||||
|
# Log activities to the database (default: false)
|
||||||
|
#enabled: false
|
||||||
|
|
||||||
|
# Save the activity before processing, then update later with the results.
|
||||||
|
# This has the advantage of capturing activities that cause a hard-crash, but doubles the number of queries used.
|
||||||
|
# Default: false
|
||||||
|
#preSave: false
|
||||||
|
|
||||||
|
# How long to save each log entry before deleting it.
|
||||||
|
# Default: 2592000000 (1 week)
|
||||||
|
#maxAge: 2592000000
|
||||||
|
|
28
packages/backend/migration/1731565470048-add-activity-log.js
Normal file
28
packages/backend/migration/1731565470048-add-activity-log.js
Normal file
|
@ -0,0 +1,28 @@
|
||||||
|
/*
|
||||||
|
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
|
||||||
|
* SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
*/
|
||||||
|
|
||||||
|
export class AddActivityLog1731565470048 {
|
||||||
|
name = 'AddActivityLog1731565470048'
|
||||||
|
|
||||||
|
async up(queryRunner) {
|
||||||
|
await queryRunner.query(`CREATE TABLE "activity_context" ("md5" text NOT NULL, "json" jsonb NOT NULL, CONSTRAINT "PK_activity_context" PRIMARY KEY ("md5"))`);
|
||||||
|
await queryRunner.query(`CREATE INDEX "IDK_activity_context_md5" ON "activity_context" ("md5") `);
|
||||||
|
await queryRunner.query(`CREATE TABLE "activity_log" ("id" character varying(32) NOT NULL, "at" TIMESTAMP WITH TIME ZONE NOT NULL, "key_id" text NOT NULL, "host" text NOT NULL, "verified" boolean NOT NULL, "accepted" boolean NOT NULL, "result" text NOT NULL, "activity" jsonb NOT NULL, "context_hash" text, "auth_user_id" character varying(32), CONSTRAINT "PK_activity_log" PRIMARY KEY ("id"))`);
|
||||||
|
await queryRunner.query(`CREATE INDEX "IDX_activity_log_at" ON "activity_log" ("at") `);
|
||||||
|
await queryRunner.query(`CREATE INDEX "IDX_activity_log_host" ON "activity_log" ("host") `);
|
||||||
|
await queryRunner.query(`ALTER TABLE "activity_log" ADD CONSTRAINT "FK_activity_log_context_hash" FOREIGN KEY ("context_hash") REFERENCES "activity_context"("md5") ON DELETE CASCADE ON UPDATE NO ACTION`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "activity_log" ADD CONSTRAINT "FK_activity_log_auth_user_id" FOREIGN KEY ("auth_user_id") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
|
||||||
|
}
|
||||||
|
|
||||||
|
async down(queryRunner) {
|
||||||
|
await queryRunner.query(`ALTER TABLE "activity_log" DROP CONSTRAINT "FK_activity_log_auth_user_id"`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "activity_log" DROP CONSTRAINT "FK_activity_log_context_hash"`);
|
||||||
|
await queryRunner.query(`DROP INDEX "public"."IDX_activity_log_host"`);
|
||||||
|
await queryRunner.query(`DROP INDEX "public"."IDX_activity_log_at"`);
|
||||||
|
await queryRunner.query(`DROP TABLE "activity_log"`);
|
||||||
|
await queryRunner.query(`DROP INDEX "public"."IDK_activity_context_md5"`);
|
||||||
|
await queryRunner.query(`DROP TABLE "activity_context"`);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,19 @@
|
||||||
|
/*
|
||||||
|
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
|
||||||
|
* SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
*/
|
||||||
|
|
||||||
|
export class ActivityLogTiming1731909785724 {
|
||||||
|
name = 'ActivityLogTiming1731909785724'
|
||||||
|
|
||||||
|
async up(queryRunner) {
|
||||||
|
await queryRunner.query(`ALTER TABLE "activity_log" ADD "duration" double precision NOT NULL DEFAULT '0'`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "activity_log" ALTER COLUMN "result" DROP NOT NULL`);
|
||||||
|
}
|
||||||
|
|
||||||
|
async down(queryRunner) {
|
||||||
|
await queryRunner.query(`UPDATE "activity_log" SET "result" = 'not processed' WHERE "result" IS NULL`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "activity_log" ALTER COLUMN "result" SET NOT NULL`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "activity_log" DROP COLUMN "duration"`);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,16 @@
|
||||||
|
/*
|
||||||
|
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
|
||||||
|
* SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
*/
|
||||||
|
|
||||||
|
export class RenameActivityLogIndexes1731910422761 {
|
||||||
|
name = 'RenameActivityLogIndexes1731910422761'
|
||||||
|
|
||||||
|
async up(queryRunner) {
|
||||||
|
await queryRunner.query(`DROP INDEX "public"."IDK_activity_context_md5"`);
|
||||||
|
}
|
||||||
|
|
||||||
|
async down(queryRunner) {
|
||||||
|
await queryRunner.query(`CREATE INDEX "IDK_activity_context_md5" ON "activity_context" ("md5") `);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,20 @@
|
||||||
|
/*
|
||||||
|
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
|
||||||
|
* SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
*/
|
||||||
|
|
||||||
|
export class NullableActivityLogDuration1731935047347 {
|
||||||
|
name = 'NullableActivityLogDuration1731935047347'
|
||||||
|
|
||||||
|
async up(queryRunner) {
|
||||||
|
await queryRunner.query(`ALTER TABLE "activity_log" ALTER COLUMN "duration" DROP NOT NULL`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "activity_log" ALTER COLUMN "duration" DROP DEFAULT`);
|
||||||
|
await queryRunner.query(`UPDATE "activity_log" SET "duration" = NULL WHERE "duration" = 0`);
|
||||||
|
}
|
||||||
|
|
||||||
|
async down(queryRunner) {
|
||||||
|
await queryRunner.query(`UPDATE "activity_log" SET "duration" = 0 WHERE "duration" IS NULL`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "activity_log" ALTER COLUMN "duration" SET DEFAULT '0'`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "activity_log" ALTER COLUMN "duration" SET NOT NULL`);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,32 @@
|
||||||
|
/*
|
||||||
|
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
|
||||||
|
* SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
*/
|
||||||
|
|
||||||
|
export class RenameActivityLogToApInboxLog1733756280460 {
|
||||||
|
name = 'RenameActivityLogToApInboxLog1733756280460'
|
||||||
|
|
||||||
|
async up(queryRunner) {
|
||||||
|
await queryRunner.query(`ALTER INDEX "IDX_activity_log_at" RENAME TO "IDX_ap_inbox_log_at"`);
|
||||||
|
await queryRunner.query(`ALTER INDEX "IDX_activity_log_host" RENAME TO "IDX_ap_inbox_log_host"`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "activity_log" RENAME CONSTRAINT "PK_activity_log" TO "PK_ap_inbox_log"`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "activity_log" RENAME CONSTRAINT "FK_activity_log_context_hash" TO "FK_ap_inbox_log_context_hash"`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "activity_log" RENAME CONSTRAINT "FK_activity_log_auth_user_id" TO "FK_ap_inbox_log_auth_user_id"`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "activity_log" RENAME TO "ap_inbox_log"`);
|
||||||
|
|
||||||
|
await queryRunner.query(`ALTER TABLE "activity_context" RENAME CONSTRAINT "PK_activity_context" TO "PK_ap_context"`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "activity_context" RENAME TO "ap_context"`);
|
||||||
|
}
|
||||||
|
|
||||||
|
async down(queryRunner) {
|
||||||
|
await queryRunner.query(`ALTER TABLE "ap_context" RENAME TO "activity_context"`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "activity_context" RENAME CONSTRAINT "PK_ap_context" TO "PK_activity_context"`);
|
||||||
|
|
||||||
|
await queryRunner.query(`ALTER TABLE "ap_inbox_log" RENAME TO "activity_log"`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "activity_log" RENAME CONSTRAINT "FK_ap_inbox_log_auth_user_id" TO "FK_activity_log_auth_user_id"`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "activity_log" RENAME CONSTRAINT "FK_ap_inbox_log_context_hash" TO "FK_activity_log_context_hash"`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "activity_log" RENAME CONSTRAINT "PK_ap_inbox_log" TO "PK_activity_log"`);
|
||||||
|
await queryRunner.query(`ALTER INDEX "IDX_ap_inbox_log_host" RENAME TO "IDX_activity_log_host"`);
|
||||||
|
await queryRunner.query(`ALTER INDEX "IDX_ap_inbox_log_at" RENAME TO "IDX_activity_log_at"`);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,19 @@
|
||||||
|
export class CreateApFetchLog1738293576355 {
|
||||||
|
name = 'CreateApFetchLog1738293576355'
|
||||||
|
|
||||||
|
async up(queryRunner) {
|
||||||
|
await queryRunner.query(`CREATE TABLE "ap_fetch_log" ("id" character varying(32) NOT NULL, "at" TIMESTAMP WITH TIME ZONE NOT NULL, "duration" double precision, "host" text NOT NULL, "request_uri" text NOT NULL, "object_uri" text, "accepted" boolean, "result" text, "object" jsonb, "context_hash" text, CONSTRAINT "PK_ap_fetch_log" PRIMARY KEY ("id"))`);
|
||||||
|
await queryRunner.query(`CREATE INDEX "IDX_ap_fetch_log_at" ON "ap_fetch_log" ("at") `);
|
||||||
|
await queryRunner.query(`CREATE INDEX "IDX_ap_fetch_log_host" ON "ap_fetch_log" ("host") `);
|
||||||
|
await queryRunner.query(`CREATE INDEX "IDX_ap_fetch_log_object_uri" ON "ap_fetch_log" ("object_uri") `);
|
||||||
|
await queryRunner.query(`ALTER TABLE "ap_fetch_log" ADD CONSTRAINT "FK_ap_fetch_log_context_hash" FOREIGN KEY ("context_hash") REFERENCES "ap_context"("md5") ON DELETE CASCADE ON UPDATE NO ACTION`);
|
||||||
|
}
|
||||||
|
|
||||||
|
async down(queryRunner) {
|
||||||
|
await queryRunner.query(`ALTER TABLE "ap_fetch_log" DROP CONSTRAINT "FK_ap_fetch_log_context_hash"`);
|
||||||
|
await queryRunner.query(`DROP INDEX "public"."IDX_ap_fetch_log_object_uri"`);
|
||||||
|
await queryRunner.query(`DROP INDEX "public"."IDX_ap_fetch_log_host"`);
|
||||||
|
await queryRunner.query(`DROP INDEX "public"."IDX_ap_fetch_log_at"`);
|
||||||
|
await queryRunner.query(`DROP TABLE "ap_fetch_log"`);
|
||||||
|
}
|
||||||
|
}
|
|
@ -13,6 +13,7 @@ import { ServerStatsService } from '@/daemons/ServerStatsService.js';
|
||||||
import { ServerService } from '@/server/ServerService.js';
|
import { ServerService } from '@/server/ServerService.js';
|
||||||
import { MainModule } from '@/MainModule.js';
|
import { MainModule } from '@/MainModule.js';
|
||||||
import { envOption } from '@/env.js';
|
import { envOption } from '@/env.js';
|
||||||
|
import { ApLogCleanupService } from '@/daemons/ApLogCleanupService.js';
|
||||||
|
|
||||||
export async function server() {
|
export async function server() {
|
||||||
const app = await NestFactory.createApplicationContext(MainModule, {
|
const app = await NestFactory.createApplicationContext(MainModule, {
|
||||||
|
@ -28,6 +29,7 @@ export async function server() {
|
||||||
if (!envOption.noDaemons) {
|
if (!envOption.noDaemons) {
|
||||||
app.get(QueueStatsService).start();
|
app.get(QueueStatsService).start();
|
||||||
app.get(ServerStatsService).start();
|
app.get(ServerStatsService).start();
|
||||||
|
app.get(ApLogCleanupService).start();
|
||||||
}
|
}
|
||||||
|
|
||||||
return app;
|
return app;
|
||||||
|
|
|
@ -129,6 +129,12 @@ type Source = {
|
||||||
enableQueryParamLogging? : boolean,
|
enableQueryParamLogging? : boolean,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
activityLogging?: {
|
||||||
|
enabled?: boolean;
|
||||||
|
preSave?: boolean;
|
||||||
|
maxAge?: number;
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
export type Config = {
|
export type Config = {
|
||||||
|
@ -238,6 +244,12 @@ export type Config = {
|
||||||
|
|
||||||
pidFile: string;
|
pidFile: string;
|
||||||
filePermissionBits?: string;
|
filePermissionBits?: string;
|
||||||
|
|
||||||
|
activityLogging: {
|
||||||
|
enabled: boolean;
|
||||||
|
preSave: boolean;
|
||||||
|
maxAge: number;
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
export type FulltextSearchProvider = 'sqlLike' | 'sqlPgroonga' | 'meilisearch';
|
export type FulltextSearchProvider = 'sqlLike' | 'sqlPgroonga' | 'meilisearch';
|
||||||
|
@ -380,6 +392,11 @@ export function loadConfig(): Config {
|
||||||
pidFile: config.pidFile,
|
pidFile: config.pidFile,
|
||||||
filePermissionBits: config.filePermissionBits,
|
filePermissionBits: config.filePermissionBits,
|
||||||
logging: config.logging,
|
logging: config.logging,
|
||||||
|
activityLogging: {
|
||||||
|
enabled: config.activityLogging?.enabled ?? false,
|
||||||
|
preSave: config.activityLogging?.preSave ?? false,
|
||||||
|
maxAge: config.activityLogging?.maxAge ?? (1000 * 60 * 60 * 24 * 30),
|
||||||
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -531,4 +548,5 @@ function applyEnvOverrides(config: Source) {
|
||||||
_apply_top(['import', ['downloadTimeout', 'maxFileSize']]);
|
_apply_top(['import', ['downloadTimeout', 'maxFileSize']]);
|
||||||
_apply_top([['signToActivityPubGet', 'checkActivityPubGetSignature', 'setupPassword']]);
|
_apply_top([['signToActivityPubGet', 'checkActivityPubGetSignature', 'setupPassword']]);
|
||||||
_apply_top(['logging', 'sql', ['disableQueryTruncation', 'enableQueryParamLogging']]);
|
_apply_top(['logging', 'sql', ['disableQueryTruncation', 'enableQueryParamLogging']]);
|
||||||
|
_apply_top(['activityLogging', ['enabled', 'preSave', 'maxAge']]);
|
||||||
}
|
}
|
||||||
|
|
207
packages/backend/src/core/ApLogService.ts
Normal file
207
packages/backend/src/core/ApLogService.ts
Normal file
|
@ -0,0 +1,207 @@
|
||||||
|
/*
|
||||||
|
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
|
||||||
|
* SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { createHash } from 'crypto';
|
||||||
|
import { Inject, Injectable } from '@nestjs/common';
|
||||||
|
import { In, LessThan } from 'typeorm';
|
||||||
|
import { DI } from '@/di-symbols.js';
|
||||||
|
import { SkApFetchLog, SkApInboxLog, SkApContext } from '@/models/_.js';
|
||||||
|
import type { ApContextsRepository, ApFetchLogsRepository, ApInboxLogsRepository } from '@/models/_.js';
|
||||||
|
import type { Config } from '@/config.js';
|
||||||
|
import { JsonValue } from '@/misc/json-value.js';
|
||||||
|
import { UtilityService } from '@/core/UtilityService.js';
|
||||||
|
import { IdService } from '@/core/IdService.js';
|
||||||
|
import { IActivity, IObject } from './activitypub/type.js';
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class ApLogService {
|
||||||
|
constructor(
|
||||||
|
@Inject(DI.config)
|
||||||
|
private readonly config: Config,
|
||||||
|
|
||||||
|
@Inject(DI.apContextsRepository)
|
||||||
|
private apContextsRepository: ApContextsRepository,
|
||||||
|
|
||||||
|
@Inject(DI.apInboxLogsRepository)
|
||||||
|
private readonly apInboxLogsRepository: ApInboxLogsRepository,
|
||||||
|
|
||||||
|
@Inject(DI.apFetchLogsRepository)
|
||||||
|
private readonly apFetchLogsRepository: ApFetchLogsRepository,
|
||||||
|
|
||||||
|
private readonly utilityService: UtilityService,
|
||||||
|
private readonly idService: IdService,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates an inbox log from an activity, and saves it if pre-save is enabled.
|
||||||
|
*/
|
||||||
|
public async createInboxLog(data: Partial<SkApInboxLog> & {
|
||||||
|
activity: IActivity,
|
||||||
|
keyId: string,
|
||||||
|
}): Promise<SkApInboxLog> {
|
||||||
|
const { object: activity, context, contextHash } = extractObjectContext(data.activity);
|
||||||
|
const host = this.utilityService.extractDbHost(data.keyId);
|
||||||
|
|
||||||
|
const log = new SkApInboxLog({
|
||||||
|
id: this.idService.gen(),
|
||||||
|
at: new Date(),
|
||||||
|
verified: false,
|
||||||
|
accepted: false,
|
||||||
|
host,
|
||||||
|
...data,
|
||||||
|
activity,
|
||||||
|
context,
|
||||||
|
contextHash,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (this.config.activityLogging.preSave) {
|
||||||
|
await this.saveInboxLog(log);
|
||||||
|
}
|
||||||
|
|
||||||
|
return log;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Saves or finalizes an inbox log.
|
||||||
|
*/
|
||||||
|
public async saveInboxLog(log: SkApInboxLog): Promise<SkApInboxLog> {
|
||||||
|
if (log.context) {
|
||||||
|
await this.saveContext(log.context);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Will be UPDATE with preSave, and INSERT without.
|
||||||
|
await this.apInboxLogsRepository.upsert(log, ['id']);
|
||||||
|
return log;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a fetch log from an activity, and saves it if pre-save is enabled.
|
||||||
|
*/
|
||||||
|
public async createFetchLog(data: Partial<SkApFetchLog> & {
|
||||||
|
requestUri: string
|
||||||
|
host: string,
|
||||||
|
}): Promise<SkApFetchLog> {
|
||||||
|
const log = new SkApFetchLog({
|
||||||
|
id: this.idService.gen(),
|
||||||
|
at: new Date(),
|
||||||
|
accepted: false,
|
||||||
|
...data,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (this.config.activityLogging.preSave) {
|
||||||
|
await this.saveFetchLog(log);
|
||||||
|
}
|
||||||
|
|
||||||
|
return log;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Saves or finalizes a fetch log.
|
||||||
|
*/
|
||||||
|
public async saveFetchLog(log: SkApFetchLog): Promise<SkApFetchLog> {
|
||||||
|
if (log.context) {
|
||||||
|
await this.saveContext(log.context);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Will be UPDATE with preSave, and INSERT without.
|
||||||
|
await this.apFetchLogsRepository.upsert(log, ['id']);
|
||||||
|
return log;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async saveContext(context: SkApContext): Promise<void> {
|
||||||
|
// https://stackoverflow.com/a/47064558
|
||||||
|
await this.apContextsRepository
|
||||||
|
.createQueryBuilder('activity_context')
|
||||||
|
.insert()
|
||||||
|
.into(SkApContext)
|
||||||
|
.values(context)
|
||||||
|
.orIgnore('md5')
|
||||||
|
.execute();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Deletes all logged copies of an object or objects
|
||||||
|
* @param objectUris URIs / AP IDs of the objects to delete
|
||||||
|
*/
|
||||||
|
public async deleteObjectLogs(objectUris: string | string[]): Promise<number> {
|
||||||
|
if (Array.isArray(objectUris)) {
|
||||||
|
const logsDeleted = await this.apFetchLogsRepository.delete({
|
||||||
|
objectUri: In(objectUris),
|
||||||
|
});
|
||||||
|
return logsDeleted.affected ?? 0;
|
||||||
|
} else {
|
||||||
|
const logsDeleted = await this.apFetchLogsRepository.delete({
|
||||||
|
objectUri: objectUris,
|
||||||
|
});
|
||||||
|
return logsDeleted.affected ?? 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Deletes all expired AP logs and garbage-collects the AP context cache.
|
||||||
|
* Returns the total number of deleted rows.
|
||||||
|
*/
|
||||||
|
public async deleteExpiredLogs(): Promise<number> {
|
||||||
|
// This is the date in UTC of the oldest log to KEEP
|
||||||
|
const oldestAllowed = new Date(Date.now() - this.config.activityLogging.maxAge);
|
||||||
|
|
||||||
|
// Delete all logs older than the threshold.
|
||||||
|
const inboxDeleted = await this.deleteExpiredInboxLogs(oldestAllowed);
|
||||||
|
const fetchDeleted = await this.deleteExpiredFetchLogs(oldestAllowed);
|
||||||
|
|
||||||
|
return inboxDeleted + fetchDeleted;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async deleteExpiredInboxLogs(oldestAllowed: Date): Promise<number> {
|
||||||
|
const { affected } = await this.apInboxLogsRepository.delete({
|
||||||
|
at: LessThan(oldestAllowed),
|
||||||
|
});
|
||||||
|
|
||||||
|
return affected ?? 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async deleteExpiredFetchLogs(oldestAllowed: Date): Promise<number> {
|
||||||
|
const { affected } = await this.apFetchLogsRepository.delete({
|
||||||
|
at: LessThan(oldestAllowed),
|
||||||
|
});
|
||||||
|
|
||||||
|
return affected ?? 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function extractObjectContext<T extends IObject>(input: T) {
|
||||||
|
const object = Object.assign({}, input, { '@context': undefined }) as Omit<T, '@context'>;
|
||||||
|
const { context, contextHash } = parseContext(input['@context']);
|
||||||
|
|
||||||
|
return { object, context, contextHash };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseContext(input: JsonValue | undefined): { contextHash: string | null, context: SkApContext | null } {
|
||||||
|
// Empty contexts are excluded for easier querying
|
||||||
|
if (input == null) {
|
||||||
|
return {
|
||||||
|
contextHash: null,
|
||||||
|
context: null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const contextHash = createHash('md5').update(JSON.stringify(input)).digest('base64');
|
||||||
|
const context = new SkApContext({
|
||||||
|
md5: contextHash,
|
||||||
|
json: input,
|
||||||
|
});
|
||||||
|
return { contextHash, context };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function calculateDurationSince(startTime: bigint): number {
|
||||||
|
// Calculate the processing time with correct rounding and decimals.
|
||||||
|
// 1. Truncate nanoseconds to microseconds
|
||||||
|
// 2. Scale to 1/10 millisecond ticks.
|
||||||
|
// 3. Round to nearest tick.
|
||||||
|
// 4. Sale to milliseconds
|
||||||
|
// Example: 123,456,789 ns -> 123,456 us -> 12,345.6 ticks -> 12,346 ticks -> 123.46 ms
|
||||||
|
const endTime = process.hrtime.bigint();
|
||||||
|
return Math.round(Number((endTime - startTime) / 1000n) / 10) / 100;
|
||||||
|
}
|
|
@ -157,6 +157,7 @@ import { QueueService } from './QueueService.js';
|
||||||
import { LoggerService } from './LoggerService.js';
|
import { LoggerService } from './LoggerService.js';
|
||||||
import { SponsorsService } from './SponsorsService.js';
|
import { SponsorsService } from './SponsorsService.js';
|
||||||
import type { Provider } from '@nestjs/common';
|
import type { Provider } from '@nestjs/common';
|
||||||
|
import { ApLogService } from '@/core/ApLogService.js';
|
||||||
|
|
||||||
//#region 文字列ベースでのinjection用(循環参照対応のため)
|
//#region 文字列ベースでのinjection用(循環参照対応のため)
|
||||||
const $LoggerService: Provider = { provide: 'LoggerService', useExisting: LoggerService };
|
const $LoggerService: Provider = { provide: 'LoggerService', useExisting: LoggerService };
|
||||||
|
@ -166,6 +167,7 @@ const $AccountMoveService: Provider = { provide: 'AccountMoveService', useExisti
|
||||||
const $AccountUpdateService: Provider = { provide: 'AccountUpdateService', useExisting: AccountUpdateService };
|
const $AccountUpdateService: Provider = { provide: 'AccountUpdateService', useExisting: AccountUpdateService };
|
||||||
const $AnnouncementService: Provider = { provide: 'AnnouncementService', useExisting: AnnouncementService };
|
const $AnnouncementService: Provider = { provide: 'AnnouncementService', useExisting: AnnouncementService };
|
||||||
const $AntennaService: Provider = { provide: 'AntennaService', useExisting: AntennaService };
|
const $AntennaService: Provider = { provide: 'AntennaService', useExisting: AntennaService };
|
||||||
|
const $ApLogService: Provider = { provide: 'ApLogService', useExisting: ApLogService };
|
||||||
const $AppLockService: Provider = { provide: 'AppLockService', useExisting: AppLockService };
|
const $AppLockService: Provider = { provide: 'AppLockService', useExisting: AppLockService };
|
||||||
const $AchievementService: Provider = { provide: 'AchievementService', useExisting: AchievementService };
|
const $AchievementService: Provider = { provide: 'AchievementService', useExisting: AchievementService };
|
||||||
const $AvatarDecorationService: Provider = { provide: 'AvatarDecorationService', useExisting: AvatarDecorationService };
|
const $AvatarDecorationService: Provider = { provide: 'AvatarDecorationService', useExisting: AvatarDecorationService };
|
||||||
|
@ -322,6 +324,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
|
||||||
AccountUpdateService,
|
AccountUpdateService,
|
||||||
AnnouncementService,
|
AnnouncementService,
|
||||||
AntennaService,
|
AntennaService,
|
||||||
|
ApLogService,
|
||||||
AppLockService,
|
AppLockService,
|
||||||
AchievementService,
|
AchievementService,
|
||||||
AvatarDecorationService,
|
AvatarDecorationService,
|
||||||
|
@ -474,6 +477,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
|
||||||
$AccountUpdateService,
|
$AccountUpdateService,
|
||||||
$AnnouncementService,
|
$AnnouncementService,
|
||||||
$AntennaService,
|
$AntennaService,
|
||||||
|
$ApLogService,
|
||||||
$AppLockService,
|
$AppLockService,
|
||||||
$AchievementService,
|
$AchievementService,
|
||||||
$AvatarDecorationService,
|
$AvatarDecorationService,
|
||||||
|
@ -627,6 +631,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
|
||||||
AccountUpdateService,
|
AccountUpdateService,
|
||||||
AnnouncementService,
|
AnnouncementService,
|
||||||
AntennaService,
|
AntennaService,
|
||||||
|
ApLogService,
|
||||||
AppLockService,
|
AppLockService,
|
||||||
AchievementService,
|
AchievementService,
|
||||||
AvatarDecorationService,
|
AvatarDecorationService,
|
||||||
|
@ -778,6 +783,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
|
||||||
$AccountUpdateService,
|
$AccountUpdateService,
|
||||||
$AnnouncementService,
|
$AnnouncementService,
|
||||||
$AntennaService,
|
$AntennaService,
|
||||||
|
$ApLogService,
|
||||||
$AppLockService,
|
$AppLockService,
|
||||||
$AchievementService,
|
$AchievementService,
|
||||||
$AvatarDecorationService,
|
$AvatarDecorationService,
|
||||||
|
|
|
@ -24,9 +24,14 @@ import { SearchService } from '@/core/SearchService.js';
|
||||||
import { ModerationLogService } from '@/core/ModerationLogService.js';
|
import { ModerationLogService } from '@/core/ModerationLogService.js';
|
||||||
import { isQuote, isRenote } from '@/misc/is-renote.js';
|
import { isQuote, isRenote } from '@/misc/is-renote.js';
|
||||||
import { LatestNoteService } from '@/core/LatestNoteService.js';
|
import { LatestNoteService } from '@/core/LatestNoteService.js';
|
||||||
|
import { ApLogService } from '@/core/ApLogService.js';
|
||||||
|
import Logger from '@/logger.js';
|
||||||
|
import { LoggerService } from './LoggerService.js';
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class NoteDeleteService {
|
export class NoteDeleteService {
|
||||||
|
private readonly logger: Logger;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
@Inject(DI.config)
|
@Inject(DI.config)
|
||||||
private config: Config,
|
private config: Config,
|
||||||
|
@ -55,7 +60,11 @@ export class NoteDeleteService {
|
||||||
private perUserNotesChart: PerUserNotesChart,
|
private perUserNotesChart: PerUserNotesChart,
|
||||||
private instanceChart: InstanceChart,
|
private instanceChart: InstanceChart,
|
||||||
private latestNoteService: LatestNoteService,
|
private latestNoteService: LatestNoteService,
|
||||||
) {}
|
private readonly apLogService: ApLogService,
|
||||||
|
loggerService: LoggerService,
|
||||||
|
) {
|
||||||
|
this.logger = loggerService.getLogger('note-delete-service');
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 投稿を削除します。
|
* 投稿を削除します。
|
||||||
|
@ -156,6 +165,11 @@ export class NoteDeleteService {
|
||||||
note: note,
|
note: note,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (note.uri) {
|
||||||
|
this.apLogService.deleteObjectLogs(note.uri)
|
||||||
|
.catch(err => this.logger.error(err, `Failed to delete AP logs for note '${note.uri}'`));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@bindThis
|
@bindThis
|
||||||
|
|
|
@ -7,7 +7,7 @@ import { Inject, Injectable } from '@nestjs/common';
|
||||||
import { IsNull, Not } from 'typeorm';
|
import { IsNull, Not } from 'typeorm';
|
||||||
import type { MiLocalUser, MiRemoteUser } from '@/models/User.js';
|
import type { MiLocalUser, MiRemoteUser } from '@/models/User.js';
|
||||||
import { InstanceActorService } from '@/core/InstanceActorService.js';
|
import { InstanceActorService } from '@/core/InstanceActorService.js';
|
||||||
import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository, FollowRequestsRepository, MiMeta } from '@/models/_.js';
|
import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository, FollowRequestsRepository, MiMeta, SkApFetchLog } from '@/models/_.js';
|
||||||
import type { Config } from '@/config.js';
|
import type { Config } from '@/config.js';
|
||||||
import { HttpRequestService } from '@/core/HttpRequestService.js';
|
import { HttpRequestService } from '@/core/HttpRequestService.js';
|
||||||
import { DI } from '@/di-symbols.js';
|
import { DI } from '@/di-symbols.js';
|
||||||
|
@ -17,7 +17,8 @@ import { LoggerService } from '@/core/LoggerService.js';
|
||||||
import type Logger from '@/logger.js';
|
import type Logger from '@/logger.js';
|
||||||
import { fromTuple } from '@/misc/from-tuple.js';
|
import { fromTuple } from '@/misc/from-tuple.js';
|
||||||
import { IdentifiableError } from '@/misc/identifiable-error.js';
|
import { IdentifiableError } from '@/misc/identifiable-error.js';
|
||||||
import { isCollectionOrOrderedCollection } from './type.js';
|
import { ApLogService, calculateDurationSince, extractObjectContext } from '@/core/ApLogService.js';
|
||||||
|
import { getNullableApId, isCollectionOrOrderedCollection } from './type.js';
|
||||||
import { ApDbResolverService } from './ApDbResolverService.js';
|
import { ApDbResolverService } from './ApDbResolverService.js';
|
||||||
import { ApRendererService } from './ApRendererService.js';
|
import { ApRendererService } from './ApRendererService.js';
|
||||||
import { ApRequestService } from './ApRequestService.js';
|
import { ApRequestService } from './ApRequestService.js';
|
||||||
|
@ -43,6 +44,7 @@ export class Resolver {
|
||||||
private apRendererService: ApRendererService,
|
private apRendererService: ApRendererService,
|
||||||
private apDbResolverService: ApDbResolverService,
|
private apDbResolverService: ApDbResolverService,
|
||||||
private loggerService: LoggerService,
|
private loggerService: LoggerService,
|
||||||
|
private readonly apLogService: ApLogService,
|
||||||
private recursionLimit = 256,
|
private recursionLimit = 256,
|
||||||
) {
|
) {
|
||||||
this.history = new Set();
|
this.history = new Set();
|
||||||
|
@ -81,6 +83,44 @@ export class Resolver {
|
||||||
return value;
|
return value;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const host = this.utilityService.extractDbHost(value);
|
||||||
|
if (this.config.activityLogging.enabled && !this.utilityService.isSelfHost(host)) {
|
||||||
|
return await this._resolveLogged(value, host);
|
||||||
|
} else {
|
||||||
|
return await this._resolve(value, host);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async _resolveLogged(requestUri: string, host: string): Promise<IObject> {
|
||||||
|
const startTime = process.hrtime.bigint();
|
||||||
|
|
||||||
|
const log = await this.apLogService.createFetchLog({
|
||||||
|
host: host,
|
||||||
|
requestUri,
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await this._resolve(requestUri, host, log);
|
||||||
|
|
||||||
|
log.accepted = true;
|
||||||
|
log.result = 'ok';
|
||||||
|
|
||||||
|
return result;
|
||||||
|
} catch (err) {
|
||||||
|
log.accepted = false;
|
||||||
|
log.result = String(err);
|
||||||
|
|
||||||
|
throw err;
|
||||||
|
} finally {
|
||||||
|
log.duration = calculateDurationSince(startTime);
|
||||||
|
|
||||||
|
// Save or finalize asynchronously
|
||||||
|
this.apLogService.saveFetchLog(log)
|
||||||
|
.catch(err => this.logger.error('Failed to record AP object fetch:', err));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async _resolve(value: string, host: string, log?: SkApFetchLog): Promise<IObject> {
|
||||||
if (value.includes('#')) {
|
if (value.includes('#')) {
|
||||||
// URLs with fragment parts cannot be resolved correctly because
|
// URLs with fragment parts cannot be resolved correctly because
|
||||||
// the fragment part does not get transmitted over HTTP(S).
|
// the fragment part does not get transmitted over HTTP(S).
|
||||||
|
@ -98,7 +138,6 @@ export class Resolver {
|
||||||
|
|
||||||
this.history.add(value);
|
this.history.add(value);
|
||||||
|
|
||||||
const host = this.utilityService.extractDbHost(value);
|
|
||||||
if (this.utilityService.isSelfHost(host)) {
|
if (this.utilityService.isSelfHost(host)) {
|
||||||
return await this.resolveLocal(value);
|
return await this.resolveLocal(value);
|
||||||
}
|
}
|
||||||
|
@ -115,6 +154,20 @@ export class Resolver {
|
||||||
? await this.apRequestService.signedGet(value, this.user) as IObject
|
? await this.apRequestService.signedGet(value, this.user) as IObject
|
||||||
: await this.httpRequestService.getActivityJson(value)) as IObject;
|
: await this.httpRequestService.getActivityJson(value)) as IObject;
|
||||||
|
|
||||||
|
if (log) {
|
||||||
|
const { object: objectOnly, context, contextHash } = extractObjectContext(object);
|
||||||
|
const objectUri = getNullableApId(object);
|
||||||
|
|
||||||
|
if (objectUri) {
|
||||||
|
log.objectUri = objectUri;
|
||||||
|
log.host = this.utilityService.extractDbHost(objectUri);
|
||||||
|
}
|
||||||
|
|
||||||
|
log.object = objectOnly;
|
||||||
|
log.context = context;
|
||||||
|
log.contextHash = contextHash;
|
||||||
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
Array.isArray(object['@context']) ?
|
Array.isArray(object['@context']) ?
|
||||||
!(object['@context'] as unknown[]).includes('https://www.w3.org/ns/activitystreams') :
|
!(object['@context'] as unknown[]).includes('https://www.w3.org/ns/activitystreams') :
|
||||||
|
@ -232,6 +285,7 @@ export class ApResolverService {
|
||||||
private apRendererService: ApRendererService,
|
private apRendererService: ApRendererService,
|
||||||
private apDbResolverService: ApDbResolverService,
|
private apDbResolverService: ApDbResolverService,
|
||||||
private loggerService: LoggerService,
|
private loggerService: LoggerService,
|
||||||
|
private readonly apLogService: ApLogService,
|
||||||
) {
|
) {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -252,6 +306,7 @@ export class ApResolverService {
|
||||||
this.apRendererService,
|
this.apRendererService,
|
||||||
this.apDbResolverService,
|
this.apDbResolverService,
|
||||||
this.loggerService,
|
this.loggerService,
|
||||||
|
this.apLogService,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
62
packages/backend/src/daemons/ApLogCleanupService.ts
Normal file
62
packages/backend/src/daemons/ApLogCleanupService.ts
Normal file
|
@ -0,0 +1,62 @@
|
||||||
|
/*
|
||||||
|
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
|
||||||
|
* SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { Injectable, type OnApplicationShutdown } from '@nestjs/common';
|
||||||
|
import { bindThis } from '@/decorators.js';
|
||||||
|
import { LoggerService } from '@/core/LoggerService.js';
|
||||||
|
import Logger from '@/logger.js';
|
||||||
|
import { ApLogService } from '@/core/ApLogService.js';
|
||||||
|
|
||||||
|
// 10 minutes
|
||||||
|
export const scanInterval = 1000 * 60 * 10;
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class ApLogCleanupService implements OnApplicationShutdown {
|
||||||
|
private readonly logger: Logger;
|
||||||
|
private scanTimer: NodeJS.Timeout | null = null;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private readonly apLogService: ApLogService,
|
||||||
|
loggerService: LoggerService,
|
||||||
|
) {
|
||||||
|
this.logger = loggerService.getLogger('activity-log-cleanup');
|
||||||
|
}
|
||||||
|
|
||||||
|
@bindThis
|
||||||
|
public async start(): Promise<void> {
|
||||||
|
// Just in case start() gets called multiple times.
|
||||||
|
this.dispose();
|
||||||
|
|
||||||
|
// Prune at startup, in case the server was rebooted during the interval.
|
||||||
|
// noinspection ES6MissingAwait
|
||||||
|
this.tick();
|
||||||
|
|
||||||
|
// Prune on a regular interval for the lifetime of the server.
|
||||||
|
this.scanTimer = setInterval(this.tick, scanInterval);
|
||||||
|
}
|
||||||
|
|
||||||
|
@bindThis
|
||||||
|
private async tick(): Promise<void> {
|
||||||
|
try {
|
||||||
|
const affected = await this.apLogService.deleteExpiredLogs();
|
||||||
|
this.logger.info(`Activity Log cleanup complete; removed ${affected} expired logs.`);
|
||||||
|
} catch (err) {
|
||||||
|
this.logger.error('Activity Log cleanup failed:', err as Error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@bindThis
|
||||||
|
public onApplicationShutdown(): void {
|
||||||
|
this.dispose();
|
||||||
|
}
|
||||||
|
|
||||||
|
@bindThis
|
||||||
|
public dispose(): void {
|
||||||
|
if (this.scanTimer) {
|
||||||
|
clearInterval(this.scanTimer);
|
||||||
|
this.scanTimer = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -8,6 +8,7 @@ import { CoreModule } from '@/core/CoreModule.js';
|
||||||
import { GlobalModule } from '@/GlobalModule.js';
|
import { GlobalModule } from '@/GlobalModule.js';
|
||||||
import { QueueStatsService } from './QueueStatsService.js';
|
import { QueueStatsService } from './QueueStatsService.js';
|
||||||
import { ServerStatsService } from './ServerStatsService.js';
|
import { ServerStatsService } from './ServerStatsService.js';
|
||||||
|
import { ApLogCleanupService } from './ApLogCleanupService.js';
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
imports: [
|
imports: [
|
||||||
|
@ -17,10 +18,12 @@ import { ServerStatsService } from './ServerStatsService.js';
|
||||||
providers: [
|
providers: [
|
||||||
QueueStatsService,
|
QueueStatsService,
|
||||||
ServerStatsService,
|
ServerStatsService,
|
||||||
|
ApLogCleanupService,
|
||||||
],
|
],
|
||||||
exports: [
|
exports: [
|
||||||
QueueStatsService,
|
QueueStatsService,
|
||||||
ServerStatsService,
|
ServerStatsService,
|
||||||
|
ApLogCleanupService,
|
||||||
],
|
],
|
||||||
})
|
})
|
||||||
export class DaemonModule {}
|
export class DaemonModule {}
|
||||||
|
|
|
@ -22,6 +22,9 @@ export const DI = {
|
||||||
appsRepository: Symbol('appsRepository'),
|
appsRepository: Symbol('appsRepository'),
|
||||||
avatarDecorationsRepository: Symbol('avatarDecorationsRepository'),
|
avatarDecorationsRepository: Symbol('avatarDecorationsRepository'),
|
||||||
latestNotesRepository: Symbol('latestNotesRepository'),
|
latestNotesRepository: Symbol('latestNotesRepository'),
|
||||||
|
apContextsRepository: Symbol('apContextsRepository'),
|
||||||
|
apFetchLogsRepository: Symbol('apFetchLogsRepository'),
|
||||||
|
apInboxLogsRepository: Symbol('apInboxLogsRepository'),
|
||||||
noteFavoritesRepository: Symbol('noteFavoritesRepository'),
|
noteFavoritesRepository: Symbol('noteFavoritesRepository'),
|
||||||
noteThreadMutingsRepository: Symbol('noteThreadMutingsRepository'),
|
noteThreadMutingsRepository: Symbol('noteThreadMutingsRepository'),
|
||||||
noteReactionsRepository: Symbol('noteReactionsRepository'),
|
noteReactionsRepository: Symbol('noteReactionsRepository'),
|
||||||
|
|
|
@ -80,7 +80,10 @@ import {
|
||||||
MiUserPublickey,
|
MiUserPublickey,
|
||||||
MiUserSecurityKey,
|
MiUserSecurityKey,
|
||||||
MiWebhook,
|
MiWebhook,
|
||||||
NoteEdit
|
NoteEdit,
|
||||||
|
SkApContext,
|
||||||
|
SkApFetchLog,
|
||||||
|
SkApInboxLog,
|
||||||
} from './_.js';
|
} from './_.js';
|
||||||
import type { DataSource } from 'typeorm';
|
import type { DataSource } from 'typeorm';
|
||||||
|
|
||||||
|
@ -126,6 +129,24 @@ const $latestNotesRepository: Provider = {
|
||||||
inject: [DI.db],
|
inject: [DI.db],
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const $apContextRepository: Provider = {
|
||||||
|
provide: DI.apContextsRepository,
|
||||||
|
useFactory: (db: DataSource) => db.getRepository(SkApContext).extend(miRepository as MiRepository<SkApContext>),
|
||||||
|
inject: [DI.db],
|
||||||
|
};
|
||||||
|
|
||||||
|
const $apFetchLogsRepository: Provider = {
|
||||||
|
provide: DI.apFetchLogsRepository,
|
||||||
|
useFactory: (db: DataSource) => db.getRepository(SkApFetchLog).extend(miRepository as MiRepository<SkApFetchLog>),
|
||||||
|
inject: [DI.db],
|
||||||
|
};
|
||||||
|
|
||||||
|
const $apInboxLogsRepository: Provider = {
|
||||||
|
provide: DI.apInboxLogsRepository,
|
||||||
|
useFactory: (db: DataSource) => db.getRepository(SkApInboxLog).extend(miRepository as MiRepository<SkApInboxLog>),
|
||||||
|
inject: [DI.db],
|
||||||
|
};
|
||||||
|
|
||||||
const $noteFavoritesRepository: Provider = {
|
const $noteFavoritesRepository: Provider = {
|
||||||
provide: DI.noteFavoritesRepository,
|
provide: DI.noteFavoritesRepository,
|
||||||
useFactory: (db: DataSource) => db.getRepository(MiNoteFavorite).extend(miRepository as MiRepository<MiNoteFavorite>),
|
useFactory: (db: DataSource) => db.getRepository(MiNoteFavorite).extend(miRepository as MiRepository<MiNoteFavorite>),
|
||||||
|
@ -526,6 +547,9 @@ const $noteScheduleRepository: Provider = {
|
||||||
$appsRepository,
|
$appsRepository,
|
||||||
$avatarDecorationsRepository,
|
$avatarDecorationsRepository,
|
||||||
$latestNotesRepository,
|
$latestNotesRepository,
|
||||||
|
$apContextRepository,
|
||||||
|
$apFetchLogsRepository,
|
||||||
|
$apInboxLogsRepository,
|
||||||
$noteFavoritesRepository,
|
$noteFavoritesRepository,
|
||||||
$noteThreadMutingsRepository,
|
$noteThreadMutingsRepository,
|
||||||
$noteReactionsRepository,
|
$noteReactionsRepository,
|
||||||
|
@ -600,6 +624,9 @@ const $noteScheduleRepository: Provider = {
|
||||||
$appsRepository,
|
$appsRepository,
|
||||||
$avatarDecorationsRepository,
|
$avatarDecorationsRepository,
|
||||||
$latestNotesRepository,
|
$latestNotesRepository,
|
||||||
|
$apContextRepository,
|
||||||
|
$apFetchLogsRepository,
|
||||||
|
$apInboxLogsRepository,
|
||||||
$noteFavoritesRepository,
|
$noteFavoritesRepository,
|
||||||
$noteThreadMutingsRepository,
|
$noteThreadMutingsRepository,
|
||||||
$noteReactionsRepository,
|
$noteReactionsRepository,
|
||||||
|
|
25
packages/backend/src/models/SkApContext.ts
Normal file
25
packages/backend/src/models/SkApContext.ts
Normal file
|
@ -0,0 +1,25 @@
|
||||||
|
/*
|
||||||
|
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
|
||||||
|
* SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { Column, PrimaryColumn, Entity } from 'typeorm';
|
||||||
|
|
||||||
|
@Entity('ap_context')
|
||||||
|
export class SkApContext {
|
||||||
|
@PrimaryColumn('text', {
|
||||||
|
primaryKeyConstraintName: 'PK_ap_context',
|
||||||
|
})
|
||||||
|
public md5: string;
|
||||||
|
|
||||||
|
@Column('jsonb')
|
||||||
|
// https://github.com/typeorm/typeorm/issues/8559
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
|
public json: any;
|
||||||
|
|
||||||
|
constructor(data?: Partial<SkApContext>) {
|
||||||
|
if (data) {
|
||||||
|
Object.assign(this, data);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
89
packages/backend/src/models/SkApFetchLog.ts
Normal file
89
packages/backend/src/models/SkApFetchLog.ts
Normal file
|
@ -0,0 +1,89 @@
|
||||||
|
/*
|
||||||
|
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
|
||||||
|
* SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { Column, Index, JoinColumn, ManyToOne, PrimaryColumn, Entity } from 'typeorm';
|
||||||
|
import { SkApContext } from '@/models/SkApContext.js';
|
||||||
|
import { id } from './util/id.js';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Records objects fetched via AP
|
||||||
|
*/
|
||||||
|
@Entity('ap_fetch_log')
|
||||||
|
export class SkApFetchLog {
|
||||||
|
@PrimaryColumn({
|
||||||
|
...id(),
|
||||||
|
primaryKeyConstraintName: 'PK_ap_fetch_log',
|
||||||
|
})
|
||||||
|
public id: string;
|
||||||
|
|
||||||
|
@Index('IDX_ap_fetch_log_at')
|
||||||
|
@Column('timestamptz')
|
||||||
|
public at: Date;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Processing duration in milliseconds
|
||||||
|
*/
|
||||||
|
@Column('double precision', { nullable: true })
|
||||||
|
public duration: number | null = null;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* DB hostname extracted from responseUri, or requestUri if fetch is incomplete
|
||||||
|
*/
|
||||||
|
@Index('IDX_ap_fetch_log_host')
|
||||||
|
@Column('text')
|
||||||
|
public host: string;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Original requested URI
|
||||||
|
*/
|
||||||
|
@Column('text', {
|
||||||
|
name: 'request_uri',
|
||||||
|
})
|
||||||
|
public requestUri: string;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Canonical URI / object ID, taken from the final payload
|
||||||
|
*/
|
||||||
|
@Column('text', {
|
||||||
|
name: 'object_uri',
|
||||||
|
nullable: true,
|
||||||
|
})
|
||||||
|
@Index('IDX_ap_fetch_log_object_uri')
|
||||||
|
public objectUri: string | null = null;
|
||||||
|
|
||||||
|
@Column('boolean', { nullable: true })
|
||||||
|
public accepted: boolean | null = null;
|
||||||
|
|
||||||
|
@Column('text', { nullable: true })
|
||||||
|
public result: string | null = null;
|
||||||
|
|
||||||
|
@Column('jsonb', { nullable: true })
|
||||||
|
// https://github.com/typeorm/typeorm/issues/8559
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
|
public object: any | null = null;
|
||||||
|
|
||||||
|
@Column({
|
||||||
|
type: 'text',
|
||||||
|
name: 'context_hash',
|
||||||
|
nullable: true,
|
||||||
|
})
|
||||||
|
public contextHash: string | null;
|
||||||
|
|
||||||
|
@ManyToOne(() => SkApContext, {
|
||||||
|
onDelete: 'CASCADE',
|
||||||
|
nullable: true,
|
||||||
|
})
|
||||||
|
@JoinColumn({
|
||||||
|
name: 'context_hash',
|
||||||
|
foreignKeyConstraintName: 'FK_ap_fetch_log_context_hash',
|
||||||
|
})
|
||||||
|
public context: SkApContext | null;
|
||||||
|
|
||||||
|
constructor(data?: Partial<SkApFetchLog>) {
|
||||||
|
if (data) {
|
||||||
|
Object.assign(this, data);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
109
packages/backend/src/models/SkApInboxLog.ts
Normal file
109
packages/backend/src/models/SkApInboxLog.ts
Normal file
|
@ -0,0 +1,109 @@
|
||||||
|
/*
|
||||||
|
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
|
||||||
|
* SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { Column, Entity, Index, JoinColumn, ManyToOne, PrimaryColumn } from 'typeorm';
|
||||||
|
import { SkApContext } from '@/models/SkApContext.js';
|
||||||
|
import { MiUser } from '@/models/_.js';
|
||||||
|
import { id } from './util/id.js';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Records activities received in the inbox
|
||||||
|
*/
|
||||||
|
@Entity('ap_inbox_log')
|
||||||
|
export class SkApInboxLog {
|
||||||
|
@PrimaryColumn({
|
||||||
|
...id(),
|
||||||
|
primaryKeyConstraintName: 'PK_ap_inbox_log',
|
||||||
|
})
|
||||||
|
public id: string;
|
||||||
|
|
||||||
|
@Index('IDX_ap_inbox_log_at')
|
||||||
|
@Column('timestamptz')
|
||||||
|
public at: Date;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Processing duration in milliseconds
|
||||||
|
*/
|
||||||
|
@Column('double precision', { nullable: true })
|
||||||
|
public duration: number | null = null;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Key ID that was used to sign this request.
|
||||||
|
* Untrusted unless verified is true.
|
||||||
|
*/
|
||||||
|
@Column({
|
||||||
|
type: 'text',
|
||||||
|
name: 'key_id',
|
||||||
|
})
|
||||||
|
public keyId: string;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Instance that the activity was sent from.
|
||||||
|
* Untrusted unless verified is true.
|
||||||
|
*/
|
||||||
|
@Index('IDX_ap_inbox_log_host')
|
||||||
|
@Column('text')
|
||||||
|
public host: string;
|
||||||
|
|
||||||
|
@Column('boolean')
|
||||||
|
public verified: boolean;
|
||||||
|
|
||||||
|
@Column('boolean')
|
||||||
|
public accepted: boolean;
|
||||||
|
|
||||||
|
@Column('text', { nullable: true })
|
||||||
|
public result: string | null = null;
|
||||||
|
|
||||||
|
@Column('jsonb')
|
||||||
|
// https://github.com/typeorm/typeorm/issues/8559
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
|
public activity: any;
|
||||||
|
|
||||||
|
@Column({
|
||||||
|
type: 'text',
|
||||||
|
name: 'context_hash',
|
||||||
|
nullable: true,
|
||||||
|
})
|
||||||
|
public contextHash: string | null;
|
||||||
|
|
||||||
|
@ManyToOne(() => SkApContext, {
|
||||||
|
onDelete: 'CASCADE',
|
||||||
|
nullable: true,
|
||||||
|
})
|
||||||
|
@JoinColumn({
|
||||||
|
name: 'context_hash',
|
||||||
|
foreignKeyConstraintName: 'FK_ap_inbox_log_context_hash',
|
||||||
|
})
|
||||||
|
public context: SkApContext | null;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* ID of the user who signed this request.
|
||||||
|
*/
|
||||||
|
@Column({
|
||||||
|
...id(),
|
||||||
|
name: 'auth_user_id',
|
||||||
|
nullable: true,
|
||||||
|
})
|
||||||
|
public authUserId: string | null;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* User who signed this request.
|
||||||
|
*/
|
||||||
|
@ManyToOne(() => MiUser, {
|
||||||
|
onDelete: 'CASCADE',
|
||||||
|
nullable: true,
|
||||||
|
})
|
||||||
|
@JoinColumn({
|
||||||
|
name: 'auth_user_id',
|
||||||
|
foreignKeyConstraintName: 'FK_ap_inbox_log_auth_user_id',
|
||||||
|
})
|
||||||
|
public authUser: MiUser | null;
|
||||||
|
|
||||||
|
constructor(data?: Partial<SkApInboxLog>) {
|
||||||
|
if (data) {
|
||||||
|
Object.assign(this, data);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -82,6 +82,9 @@ import { NoteEdit } from '@/models/NoteEdit.js';
|
||||||
import { MiBubbleGameRecord } from '@/models/BubbleGameRecord.js';
|
import { MiBubbleGameRecord } from '@/models/BubbleGameRecord.js';
|
||||||
import { MiReversiGame } from '@/models/ReversiGame.js';
|
import { MiReversiGame } from '@/models/ReversiGame.js';
|
||||||
import { MiNoteSchedule } from '@/models/NoteSchedule.js';
|
import { MiNoteSchedule } from '@/models/NoteSchedule.js';
|
||||||
|
import { SkApInboxLog } from '@/models/SkApInboxLog.js';
|
||||||
|
import { SkApFetchLog } from '@/models/SkApFetchLog.js';
|
||||||
|
import { SkApContext } from '@/models/SkApContext.js';
|
||||||
import type { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity.js';
|
import type { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity.js';
|
||||||
|
|
||||||
export interface MiRepository<T extends ObjectLiteral> {
|
export interface MiRepository<T extends ObjectLiteral> {
|
||||||
|
@ -129,6 +132,9 @@ export const miRepository = {
|
||||||
|
|
||||||
export {
|
export {
|
||||||
SkLatestNote,
|
SkLatestNote,
|
||||||
|
SkApContext,
|
||||||
|
SkApFetchLog,
|
||||||
|
SkApInboxLog,
|
||||||
MiAbuseUserReport,
|
MiAbuseUserReport,
|
||||||
MiAbuseReportNotificationRecipient,
|
MiAbuseReportNotificationRecipient,
|
||||||
MiAccessToken,
|
MiAccessToken,
|
||||||
|
@ -229,6 +235,9 @@ export type HashtagsRepository = Repository<MiHashtag> & MiRepository<MiHashtag>
|
||||||
export type InstancesRepository = Repository<MiInstance> & MiRepository<MiInstance>;
|
export type InstancesRepository = Repository<MiInstance> & MiRepository<MiInstance>;
|
||||||
export type MetasRepository = Repository<MiMeta> & MiRepository<MiMeta>;
|
export type MetasRepository = Repository<MiMeta> & MiRepository<MiMeta>;
|
||||||
export type LatestNotesRepository = Repository<SkLatestNote> & MiRepository<SkLatestNote>;
|
export type LatestNotesRepository = Repository<SkLatestNote> & MiRepository<SkLatestNote>;
|
||||||
|
export type ApContextsRepository = Repository<SkApContext> & MiRepository<SkApContext>;
|
||||||
|
export type ApFetchLogsRepository = Repository<SkApFetchLog> & MiRepository<SkApFetchLog>;
|
||||||
|
export type ApInboxLogsRepository = Repository<SkApInboxLog> & MiRepository<SkApInboxLog>;
|
||||||
export type ModerationLogsRepository = Repository<MiModerationLog> & MiRepository<MiModerationLog>;
|
export type ModerationLogsRepository = Repository<MiModerationLog> & MiRepository<MiModerationLog>;
|
||||||
export type MutingsRepository = Repository<MiMuting> & MiRepository<MiMuting>;
|
export type MutingsRepository = Repository<MiMuting> & MiRepository<MiMuting>;
|
||||||
export type RenoteMutingsRepository = Repository<MiRenoteMuting> & MiRepository<MiRenoteMuting>;
|
export type RenoteMutingsRepository = Repository<MiRenoteMuting> & MiRepository<MiRenoteMuting>;
|
||||||
|
|
|
@ -85,6 +85,9 @@ import { Config } from '@/config.js';
|
||||||
import MisskeyLogger from '@/logger.js';
|
import MisskeyLogger from '@/logger.js';
|
||||||
import { bindThis } from '@/decorators.js';
|
import { bindThis } from '@/decorators.js';
|
||||||
import { SkLatestNote } from '@/models/LatestNote.js';
|
import { SkLatestNote } from '@/models/LatestNote.js';
|
||||||
|
import { SkApContext } from '@/models/SkApContext.js';
|
||||||
|
import { SkApFetchLog } from '@/models/SkApFetchLog.js';
|
||||||
|
import { SkApInboxLog } from '@/models/SkApInboxLog.js';
|
||||||
|
|
||||||
pg.types.setTypeParser(20, Number);
|
pg.types.setTypeParser(20, Number);
|
||||||
|
|
||||||
|
@ -171,6 +174,9 @@ class MyCustomLogger implements Logger {
|
||||||
|
|
||||||
export const entities = [
|
export const entities = [
|
||||||
SkLatestNote,
|
SkLatestNote,
|
||||||
|
SkApContext,
|
||||||
|
SkApFetchLog,
|
||||||
|
SkApInboxLog,
|
||||||
MiAnnouncement,
|
MiAnnouncement,
|
||||||
MiAnnouncementRead,
|
MiAnnouncementRead,
|
||||||
MiMeta,
|
MiMeta,
|
||||||
|
|
|
@ -15,6 +15,7 @@ import type { MiNoteReaction } from '@/models/NoteReaction.js';
|
||||||
import { EmailService } from '@/core/EmailService.js';
|
import { EmailService } from '@/core/EmailService.js';
|
||||||
import { bindThis } from '@/decorators.js';
|
import { bindThis } from '@/decorators.js';
|
||||||
import { SearchService } from '@/core/SearchService.js';
|
import { SearchService } from '@/core/SearchService.js';
|
||||||
|
import { ApLogService } from '@/core/ApLogService.js';
|
||||||
import { ReactionService } from '@/core/ReactionService.js';
|
import { ReactionService } from '@/core/ReactionService.js';
|
||||||
import { QueueLoggerService } from '../QueueLoggerService.js';
|
import { QueueLoggerService } from '../QueueLoggerService.js';
|
||||||
import type * as Bull from 'bullmq';
|
import type * as Bull from 'bullmq';
|
||||||
|
@ -45,6 +46,7 @@ export class DeleteAccountProcessorService {
|
||||||
private queueLoggerService: QueueLoggerService,
|
private queueLoggerService: QueueLoggerService,
|
||||||
private searchService: SearchService,
|
private searchService: SearchService,
|
||||||
private reactionService: ReactionService,
|
private reactionService: ReactionService,
|
||||||
|
private readonly apLogService: ApLogService,
|
||||||
) {
|
) {
|
||||||
this.logger = this.queueLoggerService.logger.createSubLogger('delete-account');
|
this.logger = this.queueLoggerService.logger.createSubLogger('delete-account');
|
||||||
}
|
}
|
||||||
|
@ -84,6 +86,13 @@ export class DeleteAccountProcessorService {
|
||||||
for (const note of notes) {
|
for (const note of notes) {
|
||||||
await this.searchService.unindexNote(note);
|
await this.searchService.unindexNote(note);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Delete note AP logs
|
||||||
|
const noteUris = notes.map(n => n.uri).filter(u => !!u) as string[];
|
||||||
|
if (noteUris.length > 0) {
|
||||||
|
await this.apLogService.deleteObjectLogs(noteUris)
|
||||||
|
.catch(err => this.logger.error(err, `Failed to delete AP logs for notes of user '${user.uri ?? user.id}'`));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
this.logger.succ('All of notes deleted');
|
this.logger.succ('All of notes deleted');
|
||||||
|
@ -149,6 +158,13 @@ export class DeleteAccountProcessorService {
|
||||||
this.logger.succ('All of files deleted');
|
this.logger.succ('All of files deleted');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
{ // Delete actor logs
|
||||||
|
if (user.uri) {
|
||||||
|
await this.apLogService.deleteObjectLogs(user.uri)
|
||||||
|
.catch(err => this.logger.error(err, `Failed to delete AP logs for user '${user.uri}'`));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
{ // Send email notification
|
{ // Send email notification
|
||||||
const profile = await this.userProfilesRepository.findOneByOrFail({ userId: user.id });
|
const profile = await this.userProfilesRepository.findOneByOrFail({ userId: user.id });
|
||||||
if (profile.email && profile.emailVerified) {
|
if (profile.email && profile.emailVerified) {
|
||||||
|
|
|
@ -29,6 +29,9 @@ import { CollapsedQueue } from '@/misc/collapsed-queue.js';
|
||||||
import { MiNote } from '@/models/Note.js';
|
import { MiNote } from '@/models/Note.js';
|
||||||
import { MiMeta } from '@/models/Meta.js';
|
import { MiMeta } from '@/models/Meta.js';
|
||||||
import { DI } from '@/di-symbols.js';
|
import { DI } from '@/di-symbols.js';
|
||||||
|
import { SkApInboxLog } from '@/models/_.js';
|
||||||
|
import type { Config } from '@/config.js';
|
||||||
|
import { ApLogService, calculateDurationSince } from '@/core/ApLogService.js';
|
||||||
import { QueueLoggerService } from '../QueueLoggerService.js';
|
import { QueueLoggerService } from '../QueueLoggerService.js';
|
||||||
import type { InboxJobData } from '../types.js';
|
import type { InboxJobData } from '../types.js';
|
||||||
|
|
||||||
|
@ -46,6 +49,9 @@ export class InboxProcessorService implements OnApplicationShutdown {
|
||||||
@Inject(DI.meta)
|
@Inject(DI.meta)
|
||||||
private meta: MiMeta,
|
private meta: MiMeta,
|
||||||
|
|
||||||
|
@Inject(DI.config)
|
||||||
|
private config: Config,
|
||||||
|
|
||||||
private utilityService: UtilityService,
|
private utilityService: UtilityService,
|
||||||
private apInboxService: ApInboxService,
|
private apInboxService: ApInboxService,
|
||||||
private federatedInstanceService: FederatedInstanceService,
|
private federatedInstanceService: FederatedInstanceService,
|
||||||
|
@ -57,6 +63,7 @@ export class InboxProcessorService implements OnApplicationShutdown {
|
||||||
private apRequestChart: ApRequestChart,
|
private apRequestChart: ApRequestChart,
|
||||||
private federationChart: FederationChart,
|
private federationChart: FederationChart,
|
||||||
private queueLoggerService: QueueLoggerService,
|
private queueLoggerService: QueueLoggerService,
|
||||||
|
private readonly apLogService: ApLogService,
|
||||||
) {
|
) {
|
||||||
this.logger = this.queueLoggerService.logger.createSubLogger('inbox');
|
this.logger = this.queueLoggerService.logger.createSubLogger('inbox');
|
||||||
this.updateInstanceQueue = new CollapsedQueue(process.env.NODE_ENV !== 'test' ? 60 * 1000 * 5 : 0, this.collapseUpdateInstanceJobs, this.performUpdateInstance);
|
this.updateInstanceQueue = new CollapsedQueue(process.env.NODE_ENV !== 'test' ? 60 * 1000 * 5 : 0, this.collapseUpdateInstanceJobs, this.performUpdateInstance);
|
||||||
|
@ -64,6 +71,41 @@ export class InboxProcessorService implements OnApplicationShutdown {
|
||||||
|
|
||||||
@bindThis
|
@bindThis
|
||||||
public async process(job: Bull.Job<InboxJobData>): Promise<string> {
|
public async process(job: Bull.Job<InboxJobData>): Promise<string> {
|
||||||
|
if (this.config.activityLogging.enabled) {
|
||||||
|
return await this._processLogged(job);
|
||||||
|
} else {
|
||||||
|
return await this._process(job);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async _processLogged(job: Bull.Job<InboxJobData>): Promise<string> {
|
||||||
|
const startTime = process.hrtime.bigint();
|
||||||
|
const activity = job.data.activity;
|
||||||
|
const keyId = job.data.signature.keyId;
|
||||||
|
const log = await this.apLogService.createInboxLog({ activity, keyId });
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await this._process(job, log);
|
||||||
|
|
||||||
|
log.accepted = result.startsWith('ok');
|
||||||
|
log.result = result;
|
||||||
|
|
||||||
|
return result;
|
||||||
|
} catch (err) {
|
||||||
|
log.accepted = false;
|
||||||
|
log.result = String(err);
|
||||||
|
|
||||||
|
throw err;
|
||||||
|
} finally {
|
||||||
|
log.duration = calculateDurationSince(startTime);
|
||||||
|
|
||||||
|
// Save or finalize asynchronously
|
||||||
|
this.apLogService.saveInboxLog(log)
|
||||||
|
.catch(err => this.logger.error('Failed to record AP activity:', err));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async _process(job: Bull.Job<InboxJobData>, log?: SkApInboxLog): Promise<string> {
|
||||||
const signature = job.data.signature; // HTTP-signature
|
const signature = job.data.signature; // HTTP-signature
|
||||||
let activity = job.data.activity;
|
let activity = job.data.activity;
|
||||||
|
|
||||||
|
@ -197,6 +239,13 @@ export class InboxProcessorService implements OnApplicationShutdown {
|
||||||
delete activity.id;
|
delete activity.id;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Record verified user in log
|
||||||
|
if (log) {
|
||||||
|
log.verified = true;
|
||||||
|
log.authUser = authUser.user;
|
||||||
|
log.authUserId = authUser.user.id;
|
||||||
|
}
|
||||||
|
|
||||||
this.apRequestChart.inbox();
|
this.apRequestChart.inbox();
|
||||||
this.federationChart.inbox(authUser.user.host);
|
this.federationChart.inbox(authUser.user.host);
|
||||||
|
|
||||||
|
|
|
@ -23,6 +23,7 @@ import type {
|
||||||
PollsRepository,
|
PollsRepository,
|
||||||
UsersRepository,
|
UsersRepository,
|
||||||
} from '@/models/_.js';
|
} from '@/models/_.js';
|
||||||
|
import { ApLogService } from '@/core/ApLogService.js';
|
||||||
|
|
||||||
type MockResponse = {
|
type MockResponse = {
|
||||||
type: string;
|
type: string;
|
||||||
|
@ -49,6 +50,7 @@ export class MockResolver extends Resolver {
|
||||||
{} as ApRendererService,
|
{} as ApRendererService,
|
||||||
{} as ApDbResolverService,
|
{} as ApDbResolverService,
|
||||||
loggerService,
|
loggerService,
|
||||||
|
{} as ApLogService,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Add table
Reference in a new issue