電腦怎麼讀文字：斷詞、n-gram、BoW 與 TF-IDF

D0：模型 好用 好用 推薦
D1：模型 便宜 推薦
D2：服務 好用 快速 推薦
D3：服務 便宜 推薦

IDF(t) = ln((1 + N) / (1 + DF(t))) + 1
未正規化權重(t,d) = TF(t,d) × IDF(t)

計數：           [2,          1,          1         ]
乘上 IDF：      [3.02165125, 1.00000000, 1.51082562]
除以 L2 長度：  [0.85764287, 0.28383251, 0.42882143]

python3 -m venv .venv
source .venv/bin/activate
python -m pip install numpy==1.26.3 scipy==1.14.0 scikit-learn==1.5.1
python text_vectorization_checks.py

"""Hand-written text examples, not a real-world classification benchmark.

Executed with Python 3.10.12, scikit-learn 1.5.1, NumPy 1.26.3 on CPU.
Spaces in TRAIN are manually specified token boundaries, not tokenizer output.
"""

import json
import math
import platform

import numpy as np
import scipy
import sklearn
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import make_pipeline


TRAIN = ["模型 好用 好用 推薦", "模型 便宜 推薦",
         "服務 好用 快速 推薦", "服務 便宜 推薦"]
LABELS = ["產品", "產品", "服務", "服務"]


def word_options():
    # Replace only tokenizer: the word analyzer still constructs n-grams.
    return {"tokenizer": str.split, "token_pattern": None}


def rounded(array):
    return np.asarray(array).round(8).tolist()


def run():
    defaults = CountVectorizer().build_analyzer()
    assert defaults("我 愛 資料 科學") == ["資料", "科學"]
    assert defaults("我愛資料科學") == ["我愛資料科學"]
    chars = CountVectorizer(analyzer="char", ngram_range=(2, 2))
    assert chars.build_analyzer()("資料科學") == ["資料", "料科", "科學"]

    cv = CountVectorizer(**word_options())
    counts = cv.fit_transform(TRAIN)
    words = cv.get_feature_names_out().tolist()
    expected = np.array([[0, 2, 0, 1, 0, 1], [1, 0, 0, 1, 0, 1],
                         [0, 1, 1, 1, 1, 0], [1, 0, 0, 1, 1, 0]])
    assert words == ["便宜", "好用", "快速", "推薦", "服務", "模型"]
    np.testing.assert_array_equal(counts.toarray(), expected)
    df = np.asarray((counts > 0).sum(axis=0)).ravel()
    np.testing.assert_array_equal(df, [2, 2, 1, 4, 2, 2])

    raw = TfidfVectorizer(norm=None, **word_options())
    weighted = raw.fit_transform(TRAIN)
    expected_idf = np.log((1 + len(TRAIN)) / (1 + df)) + 1
    np.testing.assert_allclose(raw.idf_, expected_idf)
    np.testing.assert_allclose(weighted.toarray(), expected * expected_idf)
    normal = TfidfVectorizer(**word_options())
    unit = normal.fit_transform(TRAIN)
    dense = weighted.toarray()
    np.testing.assert_allclose(unit.toarray(), dense / np.linalg.norm(dense, axis=1)[:, None])
    np.testing.assert_allclose(np.asarray(unit.multiply(unit).sum(axis=1)).ravel(), 1)
    common = words.index("推薦")
    assert raw.idf_[common] == 1

    sub = TfidfVectorizer(norm=None, sublinear_tf=True, **word_options())
    sub.fit(TRAIN)
    sub_value = sub.transform([TRAIN[0]])[0, words.index("好用")]
    assert math.isclose(sub_value, (1 + math.log(2)) * expected_idf[1])
    no_smoothing = TfidfVectorizer(norm=None, smooth_idf=False, **word_options())
    no_smoothing.fit(TRAIN)
    np.testing.assert_allclose(no_smoothing.idf_, np.log(len(TRAIN) / df) + 1)

    filtered = CountVectorizer(min_df=2, max_df=0.75, **word_options())
    filtered.fit(TRAIN)
    assert filtered.get_feature_names_out().tolist() == ["便宜", "好用", "服務", "模型"]
    integer_max = CountVectorizer(max_df=1, **word_options()).fit(TRAIN)
    float_max = CountVectorizer(max_df=1.0, **word_options()).fit(TRAIN)
    assert integer_max.get_feature_names_out().tolist() == ["快速"]
    assert len(float_max.vocabulary_) == 6

    sentences = ["我 喜歡 你", "你 喜歡 我"]
    unigrams = CountVectorizer(**word_options()).fit_transform(sentences).toarray()
    np.testing.assert_array_equal(unigrams[0], unigrams[1])
    bigrams = CountVectorizer(ngram_range=(2, 2), **word_options())
    pairs = bigrams.fit_transform(sentences).toarray()
    assert not np.array_equal(pairs[0], pairs[1])
    tokens_and_pairs = CountVectorizer(ngram_range=(1, 2), **word_options())
    combined = tokens_and_pairs.build_analyzer()(sentences[0])
    assert combined == ["我", "喜歡", "你", "我 喜歡", "喜歡 你"]

    # transform uses frozen vocabulary and IDF. All unseen -> zero row.
    old_idf, old_vocab = normal.idf_.copy(), dict(normal.vocabulary_)
    unseen = normal.transform(["神奇 新品"])
    assert unseen.nnz == 0
    np.testing.assert_array_equal(normal.idf_, old_idf)
    assert normal.vocabulary_ == old_vocab
    query = normal.transform(["好用 模型"])
    # All four training rows and this query have nonzero L2 norm.
    # Dot products are cosine similarities in THIS nonzero normalized case.
    similarities = (unit @ query.T).toarray().ravel()
    assert int(similarities.argmax()) == 0

    # Deliberately wrong evaluation setup to expose what changes, not to use it.
    leaked = TfidfVectorizer(**word_options()).fit(TRAIN + ["神奇 新品"])
    assert "神奇" in leaked.vocabulary_ and "神奇" not in old_vocab
    leaked_common_idf = leaked.idf_[leaked.vocabulary_["推薦"]]
    assert math.isclose(leaked_common_idf, math.log(6 / 5) + 1)

    # One actual fit/predict on four artificial documents; no accuracy claim.
    baseline = make_pipeline(TfidfVectorizer(**word_options()),
                             LogisticRegression(solver="liblinear", random_state=0))
    baseline.fit(TRAIN, LABELS)
    prediction = baseline.predict(["模型 神奇"]).tolist()
    assert prediction == ["產品"]

    return {
        "environment": {"python": platform.python_version(), "numpy": np.__version__,
                        "scipy": scipy.__version__, "sklearn": sklearn.__version__,
                        "device": "cpu"},
        "training_texts": TRAIN, "feature_names": words,
        "count_matrix": expected.tolist(), "document_frequency": df.tolist(),
        "shape": list(counts.shape), "stored_nonzeros": counts.nnz,
        "idf_smooth": rounded(raw.idf_), "weighted_before_l2": rounded(weighted.toarray()),
        "tfidf_l2": rounded(unit.toarray()),
        "sublinear_double_count_weight": round(float(sub_value), 8),
        "idf_without_smoothing": rounded(no_smoothing.idf_),
        "filtered_features": filtered.get_feature_names_out().tolist(),
        "integer_max_df_1_features": integer_max.get_feature_names_out().tolist(),
        "word_unigrams_and_bigrams": combined,
        "bigram_feature_names": bigrams.get_feature_names_out().tolist(),
        "bigram_matrix": pairs.tolist(), "unseen_document_nnz": unseen.nnz,
        "nonzero_query_cosines": rounded(similarities),
        "leaked_common_term_idf": round(float(leaked_common_idf), 8),
        "artificial_pipeline_prediction": prediction,
        "scope": "Artificial fit/predict and arithmetic only; no held-out accuracy, CV or benchmark.",
    }


if __name__ == "__main__":
    print(json.dumps(run(), ensure_ascii=False, indent=2, allow_nan=False))