Word2Vec、GloVe 與 FastText：詞向量從哪來？

2 + 3 + 4 + 4 + 4 + 4 + 3 + 2 = 26

L = −log sigmoid(s_positive)
    − Σ_k log sigmoid(−s_negative_k)

  = softplus(−s_positive) + Σ_k softplus(s_negative_k)

中心向量 c = [0.4, −0.2]
上下文 u0 = [ 0.1,  0.3]   正例
       u1 = [−0.4,  0.2]   負例，出現兩次
       u2 = [ 0.2, −0.5]   負例

配對分數 = [−0.02, −0.20, −0.20, 0.18]

預測分數 = word_vector · context_vector + word_bias + context_bias
單格損失 = f(X_ij) × (預測分數 − ln(X_ij))²

f(x) = (x / x_max)^α，若 x < x_max
       1，其他情況

<wh、whe、her、ere、re>

python3 -m venv .venv
source .venv/bin/activate
python -m pip install torch==2.3.1 --index-url https://download.pytorch.org/whl/cpu
python embedding_checks.py

"""Hand-set mechanism examples, not trained semantic word embeddings.

Python 3.10.12 / torch 2.3.1+cu118, CPU, float64.
No corpus download, pretrained weights, TensorFlow or fastText API execution.
"""

import json
import math
import platform

import torch
from torch.nn import functional as F


DTYPE = torch.float64


def position_pairs(tokens, radius):
    return [(i, j) for i in range(len(tokens))
            for j in range(max(0, i - radius), min(len(tokens), i + radius + 1))
            if i != j]


def sgns(scores):
    # First score is one observed pair; others are supplied noise occurrences.
    return F.softplus(-scores[0]) + F.softplus(scores[1:]).sum()


def cooccurrence(tokens, words, radius):
    index = {w: i for i, w in enumerate(words)}
    result = torch.zeros((len(words), len(words)), dtype=DTYPE)
    for i, j in position_pairs(tokens, radius):
        result[index[tokens[i]], index[tokens[j]]] += 1 / abs(i - j)
    return result


def character_trigrams(word):
    bounded = "<" + word + ">"
    return [bounded[i:i + 3] for i in range(len(bounded) - 2)]


def run():
    words = "the wide road shimmered in the hot sun".split()
    pairs = position_pairs(words, radius=2)
    assert len(pairs) == 26  # Positions, not a general unique-word-pair count.
    road_contexts = [words[j] for i, j in pairs if i == 2]
    assert road_contexts == ["the", "wide", "shimmered", "in"]

    zero_scores = torch.zeros(5, dtype=DTYPE, requires_grad=True)
    independent_loss = sgns(zero_scores)
    independent_loss.backward()
    candidate_scores = torch.zeros(5, dtype=DTYPE, requires_grad=True)
    candidate_loss = F.cross_entropy(candidate_scores[None, :], torch.tensor([0]))
    candidate_loss.backward()
    assert math.isclose(float(independent_loss), 5 * math.log(2))
    assert math.isclose(float(candidate_loss), math.log(5))
    torch.testing.assert_close(zero_scores.grad,
                               torch.tensor([-.5, .5, .5, .5, .5], dtype=DTYPE))
    torch.testing.assert_close(candidate_scores.grad,
                               torch.tensor([-.8, .2, .2, .2, .2], dtype=DTYPE))

    # One hand-set center, three context rows; negative row1 occurs twice.
    center = torch.tensor([.4, -.2], dtype=DTYPE, requires_grad=True)
    contexts = torch.tensor([[.1, .3], [-.4, .2], [.2, -.5]],
                            dtype=DTYPE, requires_grad=True)
    rows = torch.tensor([0, 1, 1, 2])  # Positive0; fixed negatives1,1,2.
    c0, u0 = center.detach().clone(), contexts.detach().clone()
    scores = contexts[rows] @ center
    before = sgns(scores)
    before.backward()
    coefficients = scores.detach().sigmoid() - torch.tensor([1., 0., 0., 0.])
    expected_c = u0[rows].T @ coefficients
    expected_u = torch.zeros_like(u0)
    expected_u.index_add_(0, rows, coefficients[:, None] * c0)
    torch.testing.assert_close(center.grad, expected_c)
    torch.testing.assert_close(contexts.grad, expected_u)
    c_grad, u_grad = center.grad.clone(), contexts.grad.clone()
    # Both updates use gradients from the SAME pre-update state.
    with torch.no_grad():
        center -= 0.05 * c_grad
        contexts -= 0.05 * u_grad
        after = sgns(contexts[rows] @ center)
    assert after < before

    frequency = torch.tensor([8., 1.], dtype=DTYPE)
    noise = {}
    for name, power in [("mikolov_2013_power_0_75", .75),
                        ("fasttext_2017_power_0_5", .5)]:
        weights = frequency.pow(power)
        noise[name] = (weights / weights.sum()).tolist()

    tiny = ["甲", "乙", "甲"]
    names = ["甲", "乙"]
    x1 = cooccurrence(tiny, names, radius=1)
    x2 = cooccurrence(tiny, names, radius=2)
    torch.testing.assert_close(x1, torch.tensor([[0., 2.], [2., 0.]], dtype=DTYPE))
    torch.testing.assert_close(x2, torch.tensor([[1., 2.], [2., 0.]], dtype=DTYPE))
    conditional = x2 / x2.sum(1, keepdim=True)
    assert not torch.equal(conditional, conditional.T)
    # One nonzero GloVe cell; paper weighting x_max100 / alpha.75, no 1/2.
    observed = float(x1[0, 1])
    score = torch.tensor(.5, dtype=DTYPE, requires_grad=True)
    weight = min((observed / 100) ** .75, 1.)
    glove_loss = weight * (score - math.log(observed)) ** 2
    glove_loss.backward()
    assert math.isclose(float(score.grad), 2 * weight * (.5 - math.log(2)))
    # Zero cells are omitted BEFORE log, not evaluated as 0 * log(0).
    nonzero_cells = int((x1 > 0).sum())
    assert nonzero_cells == 2

    # Self-chosen vectors and zero fallback; NOT the fastText hash-bucket API.
    table = {"<wh": [.1, 0.], "whe": [0., .2], "her": [.3, .1],
             "ere": [-.1, .2], "re>": [.2, -.1], "<where>": [1., 1.]}

    def compose(word, known):
        parts = character_trigrams(word)
        if known:
            parts.append("<" + word + ">")
        missing = [p for p in parts if p not in table]
        vectors = torch.tensor([table.get(p, [0., 0.]) for p in parts], dtype=DTYPE)
        return {"parts": parts, "missing_zero_fallback": missing,
                "sum": vectors.sum(0).tolist(), "mean_all_parts": vectors.mean(0).tolist()}

    known, oov = compose("where", True), compose("here", False)
    assert character_trigrams("where") == ["<wh", "whe", "her", "ere", "re>"]
    assert "<here>" not in oov["parts"] and oov["missing_zero_fallback"] == ["<he"]
    torch.testing.assert_close(torch.tensor(known["sum"]), torch.tensor([1.5, 1.4]))
    torch.testing.assert_close(torch.tensor(oov["sum"]), torch.tensor([.4, .2]))

    return {
        "environment": {"python": platform.python_version(), "torch": torch.__version__,
                        "device": "cpu", "float_dtype": str(DTYPE)},
        "fixed_radius2_position_pairs": len(pairs), "road_contexts": road_contexts,
        "zero_score_sgns_sum": float(independent_loss.detach()),
        "zero_score_sgns_gradients": zero_scores.grad.tolist(),
        "zero_score_candidate_ce": float(candidate_loss.detach()),
        "zero_score_candidate_ce_gradients": candidate_scores.grad.tolist(),
        "single_sgns_update": {"context_row_occurrences": rows.tolist(),
                               "scores_before": scores.detach().tolist(),
                               "loss_before": float(before.detach()), "loss_after": float(after),
                               "center_gradient": c_grad.tolist(),
                               "context_gradients": u_grad.tolist(),
                               "center_after": center.detach().tolist(),
                               "contexts_after": contexts.detach().tolist()},
        "two_word_noise_probabilities": noise,
        "distance_weighted_cooccurrence_radius1": x1.tolist(),
        "distance_weighted_cooccurrence_radius2": x2.tolist(),
        "conditional_context_given_center_radius2": conditional.tolist(),
        "glove_nonzero_cell": {"X": observed, "weight": weight,
                               "loss_without_half": float(glove_loss.detach()),
                               "score_gradient": float(score.grad),
                               "zero_cells_omitted_before_log": True},
        "hand_set_subword_known": known, "hand_set_subword_oov": oov,
        "scope": "One artificial SGD step, scalar GloVe cell and hand-set subword vectors; no learned semantic corpus model or benchmark.",
    }


if __name__ == "__main__":
    print(json.dumps(run(), ensure_ascii=False, indent=2, allow_nan=False))