LLM 추론 최적�� Part 3 — Sparse Attention 실전

import torch
import matplotlib.pyplot as plt
import numpy as np

def visualize_attention_pattern(seq_len=32):
    """Full Attention 패턴 시각화"""
    # Causal mask: 하삼각 행렬
    mask = torch.tril(torch.ones(seq_len, seq_len))

    fig, ax = plt.subplots(figsize=(8, 8))
    ax.imshow(mask, cmap='Blues', aspect='equal')
    ax.set_xlabel('Key Position')
    ax.set_ylabel('Query Position')
    ax.set_title(f'Full Causal Attention ({seq_len}×{seq_len})')

    # 계산량 표시
    total_ops = mask.sum().item()
    ax.text(seq_len//2, -2, f'Total attention pairs: {int(total_ops)}',
            ha='center', fontsize=11)

    plt.tight_layout()
    plt.savefig('full_attention_pattern.png', dpi=150)
    plt.show()

visualize_attention_pattern()