我們再看一下example_test.sh腳本，這個腳本能生成多種詞向量，包括word2vec，PPMI，SVD在不同超參下面的詞向量。流程就是先預處理，然後訓練，或是計算出向量，最後用analogy和similarity任務去評估。
#!/bin/sh
# Download and install word2vecf#下載並且安裝word2vecf
if [ ! -f word2vecf ]; then
scripts/install_word2vecf.sh
fi
# Download corpus. We chose a small corpus for the example, and larger corpora will yield better results.#下載語料
wget http://www.statmt.org/wmt14/training-monolingual-news-crawl/news.2010.en.shuffled.gz
gzip -d news.2010.en.shuffled.gz
CORPUS=news.2010.en.shuffled
# Clean the corpus from non alpha-numeric symbols#清洗語料
scripts/clean_corpus.sh $CORPUS > $CORPUS.clean
# Create two example collections of word-context pairs:
# A) Window size 2 with "clean" subsampling#兩種超參選擇，這是A選擇
mkdir w2.sub
python hyperwords/corpus2pairs.py --win 2 --sub 1e-5 ${CORPUS}.clean > w2.sub/pairs#從語料到單詞對
scripts/pairs2counts.sh w2.sub/pairs > w2.sub/counts#從單詞對到共現矩陣
python hyperwords/counts2vocab.py w2.sub/counts#從共現矩陣到中心詞和上下文詞典
# B) Window size 5 with dynamic contexts and "dirty" subsampling#兩種超參選擇，這是B選擇
mkdir w5.dyn.sub.del
python hyperwords/corpus2pairs.py --win 5 --dyn --sub 1e-5 --del ${CORPUS}.clean > w5.dyn.sub.del/pairs
scripts/pairs2counts.sh w5.dyn.sub.del/pairs > w5.dyn.sub.del/counts
python hyperwords/counts2vocab.py w5.dyn.sub.del/counts
# Calculate PMI matrices for each collection of pairs
python hyperwords/counts2pmi.py --cds 0.75 w2.sub/counts w2.sub/pmi#從共現矩陣到PMI矩陣
python hyperwords/counts2pmi.py --cds 0.75 w5.dyn.sub.del/counts w5.dyn.sub.del/pmi
# Create embeddings with SVD
python hyperwords/pmi2svd.py --dim 500 --neg 5 w2.sub/pmi w2.sub/svd#從PMI矩陣到SVD
cp w2.sub/pmi.words.vocab w2.sub/svd.words.vocab#PMI和SVD共享中心詞和上下文詞典
cp w2.sub/pmi.contexts.vocab w2.sub/svd.contexts.vocab
python hyperwords/pmi2svd.py --dim 500 --neg 5 w5.dyn.sub.del/pmi w5.dyn.sub.del/svd
cp w5.dyn.sub.del/pmi.words.vocab w5.dyn.sub.del/svd.words.vocab
cp w5.dyn.sub.del/pmi.contexts.vocab w5.dyn.sub.del/svd.contexts.vocab
# Create embeddings with SGNS (A). Commands 2-5 are necessary for loading the vectors with embeddings.py#對情況A產生的pairs進行訓練，用word2vecf
word2vecf/word2vecf -train w2.sub/pairs -pow 0.75 -cvocab w2.sub/counts.contexts.vocab -wvocab w2.sub/counts.words.vocab -dumpcv w2.sub/sgns.contexts -output w2.sub/sgns.words -threads 10 -negative 15 -size 500;
python hyperwords/text2numpy.py w2.sub/sgns.words#把文本形式的詞向量轉成numpy數組的形式
rm w2.sub/sgns.words
python hyperwords/text2numpy.py w2.sub/sgns.contexts
rm w2.sub/sgns.contexts
# Create embeddings with SGNS (B). Commands 2-5 are necessary for loading the vectors with embeddings.py
word2vecf/word2vecf -train w5.dyn.sub.del/pairs -pow 0.75 -cvocab w5.dyn.sub.del/counts.contexts.vocab -wvocab w5.dyn.sub.del/counts.words.vocab -dumpcv w5.dyn.sub.del/sgns.contexts -output w5.dyn.sub.del/sgns.words -threads 10 -negative 15 -size 500;
python hyperwords/text2numpy.py w5.dyn.sub.del/sgns.words
rm w5.dyn.sub.del/sgns.words
python hyperwords/text2numpy.py w5.dyn.sub.del/sgns.contexts
rm w5.dyn.sub.del/sgns.contexts
# Evaluate on Word Similarity#最後是評估 先是similarity評估
echo
echo "WS353 Results"
echo "-------------"
python hyperwords/ws_eval.py --neg 5 PPMI w2.sub/pmi testsets/ws/ws353.txt
python hyperwords/ws_eval.py --eig 0.5 SVD w2.sub/svd testsets/ws/ws353.txt
python hyperwords/ws_eval.py --w+c SGNS w2.sub/sgns testsets/ws/ws353.txt
python hyperwords/ws_eval.py --neg 5 PPMI w5.dyn.sub.del/pmi testsets/ws/ws353.txt
python hyperwords/ws_eval.py --eig 0.5 SVD w5.dyn.sub.del/svd testsets/ws/ws353.txt
python hyperwords/ws_eval.py --w+c SGNS w5.dyn.sub.del/sgns testsets/ws/ws353.txt
# Evaluate on Analogies#然後是analogy評估
echo
echo "Google Analogy Results"
echo "----------------------"
python hyperwords/analogy_eval.py PPMI w2.sub/pmi testsets/analogy/google.txt
python hyperwords/analogy_eval.py --eig 0 SVD w2.sub/svd testsets/analogy/google.txt
python hyperwords/analogy_eval.py SGNS w2.sub/sgns testsets/analogy/google.txt
python hyperwords/analogy_eval.py PPMI w5.dyn.sub.del/pmi testsets/analogy/google.txt
python hyperwords/analogy_eval.py --eig 0 SVD w5.dyn.sub.del/svd testsets/analogy/google.txt
python hyperwords/analogy_eval.py SGNS w5.dyn.sub.del/sgns testsets/analogy/google.txt