Image for post
Image for post
1984這部小說裡,最常見的100個單字

安裝NLTK

pip install nltk

安裝NLTK包

import nltk
nltk.download()
#跳出GUI界面,下載需要的資料

載入內建的書籍文字檔

#共九本書(如果找不到書的話,依照螢幕提示進行下載)
from nltk.book import *

計算單字頻率並繪圖

from bs4 import BeautifulSoup

import urllib.request

import nltk

response = urllib.request.urlopen('http://php.net/')

html = response.read()

soup = BeautifulSoup(html,"html5lib")

text = soup.get_text(strip=True)

tokens = [t for t in text.split()]

freq = nltk.FreqDist(tokens)

for key,val in freq.items():

print (str(key) + ':' + str(val))
freq.plot(20, cumulative=False)

移除停用詞Stop Words

停用詞大致分為兩類。
1)人類語言中包含的功能詞,如'the'、'is'、'at'、'which'、'on'等。
2)詞彙詞,比如'want'等,這些詞應用十分廣泛,但是對這樣的詞搜尋引擎無法保證能夠給出真正相關的搜索結果。
#stopwords必須使用nltk.download()下載from bs4 import BeautifulSoup

import urllib.request

import nltk

from nltk.corpus import stopwords

response = urllib.request.urlopen('http://php.net/')

html = response.read()

soup = BeautifulSoup(html,"html5lib")

text = soup.get_text(strip=True)

tokens = [t for t in text.split()]

clean_tokens = tokens[:]

sr = stopwords.words('english')

for token in tokens:

if token in stopwords.words('english'):

clean_tokens.remove(token)

freq = nltk.FreqDist(clean_tokens)

for key,val in freq.items():

print (str(key) + ':' + str(val))
freq.plot(20,cumulative=False)可使用1984這本小說來分析看看,哪些是高頻率單字如下連結http://gutenberg.net.au/ebooks01/0100021.txt

分開英文句子(斷句)

from nltk.tokenize import sent_tokenizemytext = "Hello Adam, how are you? I hope everything is going well. Today is a good day, see you dude."print(sent_tokenize(mytext))['Hello Adam, how are you?', 'I hope everything is going well.', 'Today is a good day, see you dude.']

分開英文單字(斷詞)

from nltk.tokenize import word_tokenizemytext = "Hello Mr. Adam, how are you? I hope everything is going well. Today is a good day, see you dude."print(word_tokenize(mytext))['Hello', 'Mr.', 'Adam', ',', 'how', 'are', 'you', '?', 'I', 'hope', 'everything', 'is', 'going', 'well', '.', 'Today', 'is', 'a', 'good', 'day', ',', 'see', 'you', 'dude', '.']

分開非英文文字

from nltk.tokenize import sent_tokenizemytext = "Bonjour M. Adam, comment allez-vous? J'espère que tout va bien. Aujourd'hui est un bon jour."print(sent_tokenize(mytext,"french"))['Bonjour M. Adam, comment allez-vous?', "J'espère que tout va bien.", "Aujourd'hui est un bon jour."]

使用WordNet顯示同義/反義詞

#wordnet必須使用nltk.download()下載from nltk.corpus import wordnetsyn = wordnet.synsets("pain")print(syn[0].definition())print(syn[0].examples())#用For迴圈取得大量相關代名詞from nltk.corpus import wordnet

synonyms = []

for syn in wordnet.synsets('Computer'):

for lemma in syn.lemmas():

synonyms.append(lemma.name())

print(synonyms)
#用For迴圈取得大量反義詞
from nltk.corpus import wordnet

antonyms = []

for syn in wordnet.synsets("small"):

for l in syn.lemmas():

if l.antonyms():

antonyms.append(l.antonyms()[0].name())

print(antonyms)
Image for post
Image for post

去除字尾Stemming

from nltk.stem import PorterStemmer

stemmer = PorterStemmer()

print(stemmer.stem('working'))
#顯示work
-----------------------------------------------
支持去除以下語言的字尾from nltk.stem import SnowballStemmerprint(SnowballStemmer.languages)('arabic', 'danish', 'dutch', 'english', 'finnish', 'french', 'german', 'hungarian', 'italian', 'norwegian', 'porter', 'portuguese', 'romanian', 'russian', 'spanish', 'swedish')------------------------------------------------去除德文字尾
from nltk.stem import SnowballStemmer
french_stemmer = SnowballStemmer('german')
print(french_stemmer.stem("Guten"))

更精確的去除字尾Lemmatization

from nltk.stem import WordNetLemmatizer

lemmatizer = WordNetLemmatizer()

print(lemmatizer.lemmatize('playing', pos="v"))

print(lemmatizer.lemmatize('playing', pos="n"))

print(lemmatizer.lemmatize('playing', pos="a"))

print(lemmatizer.lemmatize('playing', pos="r"))

找出高頻率單字

# Importing necessary library
import pandas as pd
import numpy as np
import nltk
import os
import nltk.corpus

text = “In Brazil they drive on the right-hand side of the road. Brazil has a large coastline on the eastern
side of South America"
from nltk.tokenize import word_tokenizetoken = word_tokenize(text)
token
from nltk.probability import FreqDist
fdist = FreqDist(token)
fdist
#出現頻率最高的10個單字
fdist1 = fdist.most_common(10)
fdist1

Part of speech tagging (POS)詞性標記

text = “vote to choose a particular man or a group (party) to represent them in parliament”
#Tokenize the text
tex = word_tokenize(text)
for token in tex:
print(nltk.pos_tag([token]))

Named entity recognition命名實體標註(NER)

text = “Google’s CEO Sundar Pichai introduced the new Pixel at Minnesota Roi Centre Event”#importing chunk library from nltk
from nltk import ne_chunk
# tokenize and POS Tagging before doing chunk
token = word_tokenize(text)
tags = nltk.pos_tag(token)
chunk = ne_chunk(tags)
chunk

Chunking資訊提取

text = “We saw the yellow dog”
token = word_tokenize(text)
tags = nltk.pos_tag(token)reg = “NP: {<DT>?<JJ>*<NN>}”
a = nltk.RegexpParser(reg)
result = a.parse(tags)
print(result)

n-gram

from nltk.book import *
from nltk import bigrams
bigrams = bigrams(text6)
bigramsDist = FreqDist(bigrams)
bigramDist[("Sir","Robin")]

拼字錯誤(edit-distance)

#代表著從rain變成shine這個單字需要三步驟(sain->shin->shine)
from nltk.metrics import edit_distance
edit_distance("rain","shine")
#3

Written by

Machine Learning / Deep Learning / Python / Flutter cakeresume.com/yanwei-liu

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store