Реализация системы модерации контента на сайте

Реализация системы модерации контента

Система модерации проверяет пользовательский контент перед публикацией или после неё. Включает ручную модерацию, автоматическую фильтрацию (стоп-слова, ссылки, ML-модели), жалобы пользователей, очередь задач для модераторов.

Архитектура

[User posts content]
         ↓
[Auto-filter: spam/toxicity check]
    ↙           ↘
[Auto-approve]  [Send to moderation queue]
                         ↓
               [Moderator dashboard]
               ↙          ↘       ↘
         [Approve]    [Reject]  [Shadow-ban]
              ↓           ↓
       [Publish]    [Notify user]

Модель с состояниями модерации

// ModerationStatus: pending, approved, rejected, spam, shadow_banned

class Comment extends Model
{
    use HasModerationStatus;

    protected $casts = [
        'moderation_metadata' => 'array',
        'moderated_at'        => 'datetime',
    ];

    public function scopeVisible(Builder $query): Builder
    {
        return $query->where('status', 'approved');
    }

    public function scopePendingModeration(Builder $query): Builder
    {
        return $query->where('status', 'pending');
    }
}

Автоматическая фильтрация

class ContentModerationService
{
    private array $spamPatterns = [
        '/https?:\/\/[^\s]+/i',         // внешние ссылки
        '/\b(casino|viagra|loan|crypto)\b/i',
        '/(.)\1{5,}/',                  // повторяющиеся символы (ааааааа)
    ];

    private array $bannedWords;  // загружается из БД/конфига

    public function analyze(string $content, User|null $author): ModerationResult
    {
        $signals = [];
        $spamScore = 0;

        // Проверка стоп-слов
        foreach ($this->bannedWords as $word) {
            if (stripos($content, $word) !== false) {
                $signals[] = "banned_word:{$word}";
                $spamScore += 40;
            }
        }

        // Проверка паттернов
        foreach ($this->spamPatterns as $pattern) {
            if (preg_match($pattern, $content)) {
                $signals[] = "pattern_match:{$pattern}";
                $spamScore += 30;
            }
        }

        // История автора
        if ($author) {
            $authorSpamRate = $author->comments()
                ->where('status', 'spam')
                ->count() / max($author->comments()->count(), 1);

            if ($authorSpamRate > 0.3) {
                $signals[] = 'author_spam_history';
                $spamScore += 50;
            }
        } else {
            $spamScore += 10;  // анонимные — выше риск
        }

        // Длина контента
        if (strlen($content) < 5) {
            $signals[] = 'too_short';
            $spamScore += 20;
        }

        return new ModerationResult(
            score: $spamScore,
            signals: $signals,
            decision: match (true) {
                $spamScore >= 80 => ModerationDecision::SPAM,
                $spamScore >= 50 => ModerationDecision::PENDING,
                default          => ModerationDecision::APPROVE,
            }
        );
    }
}

// В контроллере
public function store(Request $request): JsonResponse
{
    $content = $request->validated('body');
    $result  = $this->moderation->analyze($content, auth()->user());

    $status = match ($result->decision) {
        ModerationDecision::APPROVE => 'approved',
        ModerationDecision::SPAM    => 'spam',
        default                     => 'pending',
    };

    $comment = Comment::create([
        'body'                 => $content,
        'status'               => $status,
        'moderation_score'     => $result->score,
        'moderation_metadata'  => ['signals' => $result->signals],
    ]);

    if ($status === 'pending') {
        event(new CommentPendingModerationEvent($comment));
    }

    return response()->json(['id' => $comment->id, 'status' => $status], 201);
}

Панель модератора

class ModerationController extends Controller
{
    public function __construct()
    {
        $this->middleware('can:moderate-content');
    }

    public function queue(Request $request): JsonResponse
    {
        $items = Comment::pendingModeration()
            ->with('user:id,name,email', 'entity')
            ->when($request->type, fn($q) => $q->where('entity_type', $request->type))
            ->orderBy('moderation_score', 'desc')  // сначала наиболее подозрительные
            ->paginate(50);

        return response()->json($items);
    }

    public function decision(Request $request, Comment $comment): JsonResponse
    {
        $request->validate([
            'action' => 'required|in:approve,reject,spam,shadow_ban',
            'reason' => 'nullable|string|max:500',
        ]);

        $previousStatus = $comment->status;

        $comment->update([
            'status'         => match ($request->action) {
                'approve'     => 'approved',
                'reject'      => 'rejected',
                'spam'        => 'spam',
                'shadow_ban'  => 'shadow_banned',
            },
            'moderated_by'   => auth()->id(),
            'moderated_at'   => now(),
            'rejection_reason' => $request->reason,
        ]);

        ModerationDecisionLog::create([
            'moderator_id'  => auth()->id(),
            'comment_id'    => $comment->id,
            'action'        => $request->action,
            'reason'        => $request->reason,
            'previous_status' => $previousStatus,
        ]);

        // При approve — опубликовать и уведомить
        if ($request->action === 'approve') {
            event(new CommentApprovedEvent($comment));
        }

        // При rejection — уведомить автора
        if (in_array($request->action, ['reject', 'spam'])) {
            $comment->user?->notify(new CommentRejectedNotification($comment));
        }

        // Обновить репутацию автора
        UpdateAuthorReputationJob::dispatch($comment->user, $request->action);

        return response()->json(['status' => 'ok']);
    }

    // Пакетные действия
    public function batchDecision(Request $request): JsonResponse
    {
        $request->validate([
            'comment_ids' => 'required|array|max:100',
            'action'      => 'required|in:approve,reject,spam',
        ]);

        Comment::whereIn('id', $request->comment_ids)
            ->where('status', 'pending')
            ->update([
                'status'       => $request->action === 'approve' ? 'approved' : $request->action,
                'moderated_by' => auth()->id(),
                'moderated_at' => now(),
            ]);

        return response()->json(['updated' => count($request->comment_ids)]);
    }
}

Жалобы пользователей

class ReportController extends Controller
{
    public function store(Request $request, Comment $comment): JsonResponse
    {
        $request->validate(['reason' => 'required|in:spam,abuse,misinformation,other']);

        Report::firstOrCreate(
            ['reporter_id' => auth()->id(), 'comment_id' => $comment->id],
            ['reason' => $request->reason]
        );

        // Если 3+ жалоб — автоматически в очередь на модерацию
        $reportCount = Report::where('comment_id', $comment->id)->count();

        if ($reportCount >= 3 && $comment->status === 'approved') {
            $comment->update(['status' => 'pending', 'auto_flagged' => true]);
        }

        return response()->json(['reported' => true]);
    }
}

Интеграция с Perspective API (Google)

Perspective API анализирует токсичность текста:

class PerspectiveApiService
{
    public function analyzeToxicity(string $text): float
    {
        $response = Http::post(
            "https://commentanalyzer.googleapis.com/v1alpha1/comments:analyze?key={$this->apiKey}",
            [
                'comment'           => ['text' => $text],
                'languages'         => ['ru', 'en'],
                'requestedAttributes' => ['TOXICITY' => new \stdClass()],
            ]
        );

        return $response->json('attributeScores.TOXICITY.summaryScore.value', 0.0);
    }
}

// В ModerationService
$toxicity = $this->perspective->analyzeToxicity($content);
if ($toxicity > 0.8) {
    $signals[] = "high_toxicity:{$toxicity}";
    $spamScore += 60;
}

Статистика модерации

// Dashboard для руководителя команды модерации
public function stats(): JsonResponse
{
    $period = now()->subDays(30);

    return response()->json([
        'total_reviewed'  => ModerationDecisionLog::where('created_at', '>=', $period)->count(),
        'by_action'       => ModerationDecisionLog::where('created_at', '>=', $period)
            ->groupBy('action')->selectRaw('action, COUNT(*) as count')->pluck('count', 'action'),
        'avg_queue_time'  => Comment::where('status', '!=', 'pending')
            ->where('created_at', '>=', $period)
            ->whereNotNull('moderated_at')
            ->selectRaw('AVG(EXTRACT(EPOCH FROM (moderated_at - created_at))/60) as avg_minutes')
            ->value('avg_minutes'),
        'by_moderator'    => ModerationDecisionLog::where('created_at', '>=', $period)
            ->groupBy('moderator_id')
            ->selectRaw('moderator_id, COUNT(*) as count')
            ->with('moderator:id,name')
            ->get(),
    ]);
}

Срок реализации