自然语言处理开源工具介绍系列一：英文NLP工具NLTK详解及在线测试

python -m venv venv
source venv/bin/activate
pip install ipython
pip install nltk

Python 3.8.10 (default, Jun  2 2021, 10:49:15) 
Type 'copyright', 'credits' or 'license' for more information
IPython 7.26.0 -- An enhanced Interactive Python. Type '?' for help.
 
In [1]: import nltk
 
In [2]: text = """The Natural Language Toolkit, or more commonly NLTK, is a suite of libraries and progra
   ...: ms for symbolic and statistical natural language processing (NLP) for English written in the Pyth
   ...: on programming language. It was developed by Steven Bird and Edward Loper in the Department of Co
   ...: mputer and Information Science at the University of Pennsylvania. NLTK includes graphical demo
   ...: nstrations and sample data. It is accompanied by a book that explains the underlying concepts beh
   ...: ind the language processing tasks supported by the toolkit plus a cookbook."""
 
In [3]: tokens = nltk.word_tokenize(text)
...
 
LookupError: 
**********************************************************************
  Resource punkt not found.
  Please use the NLTK Downloader to obtain the resource:
 
  >>> import nltk
  >>> nltk.download('punkt')
 
  For more information see: https://www.nltk.org/data.html
 
  Attempted to load tokenizers/punkt/PY3/english.pickle
 
  Searched in:
    - '/home/yzone/nltk_data'
    - '/home/yzone/textminer/nlp_tools/venv/nltk_data'
    - '/home/yzone/textminer/nlp_tools/venv/share/nltk_data'
    - '/home/yzone/textminer/nlp_tools/venv/lib/nltk_data'
    - '/usr/share/nltk_data'
    - '/usr/local/share/nltk_data'
    - '/usr/lib/nltk_data'
    - '/usr/local/lib/nltk_data'
    - ''
**********************************************************************

In [4]: nltk.download('punkt')
[nltk_data] Error loading punkt: <urlopen error [Errno 111] Connection
[nltk_data]     refused>
Out[4]: False

Python 3.8.10 (default, Jun  2 2021, 10:49:15) 
Type 'copyright', 'credits' or 'license' for more information
IPython 7.26.0 -- An enhanced Interactive Python. Type '?' for help.
 
In [1]: import nltk
 
In [2]: text = "The Natural Language Toolkit, or more commonly NLTK, it's a suite of libraries and programs for 
   ...: symbolic and statistical natural language processing (NLP) for English written in the Python programming
   ...:  language. It was developed by Steven Bird and Edward Loper in the Department of Computer and Informatio
   ...: n Science at the University of Pennsylvania. NLTK includes graphical demonstrations and sample data. It 
   ...: is accompanied by a book that explains the underlying concepts behind the language processing tasks supp
   ...: orted by the toolkit, plus a cookbook."
 
# 测试英文 word tokenize 功能
In [3]: tokens = nltk.word_tokenize(text)
 
In [4]: tokens[0:20]
Out[4]: 
['The',
 'Natural',
 'Language',
 'Toolkit',
 ',',
 'or',
 'more',
 'commonly',
 'NLTK',
 ',',
 'it',
 "'s",
 'a',
 'suite',
 'of',
 'libraries',
 'and',
 'programs',
 'for',
 'symbolic']
 
# 测试英文词性标注功能
In [5]: tagged_tokens = nltk.pos_tag(tokens)
 
In [6]: tagged_tokens[0:20]
Out[6]: 
[('The', 'DT'),
 ('Natural', 'NNP'),
 ('Language', 'NNP'),
 ('Toolkit', 'NNP'),
 (',', ','),
 ('or', 'CC'),
 ('more', 'JJR'),
 ('commonly', 'RB'),
 ('NLTK', 'NNP'),
 (',', ','),
 ('it', 'PRP'),
 ("'s", 'VBZ'),
 ('a', 'DT'),
 ('suite', 'NN'),
 ('of', 'IN'),
 ('libraries', 'NNS'),
 ('and', 'CC'),
 ('programs', 'NNS'),
 ('for', 'IN'),
 ('symbolic', 'JJ')]
 
# 测试英文断句功能
In [8]: sents = nltk.sent_tokenize(text)
 
In [9]: sents
Out[9]: 
["The Natural Language Toolkit, or more commonly NLTK, it's a suite of libraries and programs for symbolic and statistical natural language processing (NLP) for English written in the Python programming language.",
 'It was developed by Steven Bird and Edward Loper in the Department of Computer and Information Science at the University of Pennsylvania.',
 'NLTK includes graphical demonstrations and sample data.',
 'It is accompanied by a book that explains the underlying concepts behind the language processing tasks supported by the toolkit, plus a cookbook.']
 
# 测试英文命名实体标注功能
In [10]: entities = nltk.chunk.ne_chunk(tagged_tokens)
---------------------------------------------------------------------------
ModuleNotFoundError                       Traceback (most recent call last)
<ipython-input-10-a85268718ce4> in <module>
----&gt; 1 entities = nltk.chunk.ne_chunk(tagged_tokens)
 
~/textminer/nlp_tools/venv/lib/python3.8/site-packages/nltk/chunk/__init__.py in ne_chunk(tagged_tokens, binary)
    183     else:
    184         chunker_pickle = _MULTICLASS_NE_CHUNKER
--&gt; 185     chunker = load(chunker_pickle)
    186     return chunker.parse(tagged_tokens)
    187 
 
~/textminer/nlp_tools/venv/lib/python3.8/site-packages/nltk/data.py in load(resource_url, format, cache, verbose, logic_parser, fstruct_reader, encoding)
    753         resource_val = opened_resource.read()
    754     elif format == "pickle":
--&gt; 755         resource_val = pickle.load(opened_resource)
    756     elif format == "json":
    757         import json
 
ModuleNotFoundError: No module named 'numpy'
 
# 发现缺少 numpy 模块，直接 pip install 安装
In [11]: pip install numpy
Looking in indexes: https://pypi.tuna.tsinghua.edu.cn/simple
Collecting numpy
  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/aa/69/260a4a1cc89cc00b51f432db048c396952f5c05dfa1345a1b3dbd9ea3544/numpy-1.21.2-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (15.8 MB)
     |████████████████████████████████| 15.8 MB 11.2 MB/s 
Installing collected packages: numpy
Successfully installed numpy-1.21.2
Note: you may need to restart the kernel to use updated packages.
 
# 继续测试英文命名实体识别
In [12]: entities = nltk.chunk.ne_chunk(tagged_tokens)
 
In [13]: entities
Out[13]: Tree('S', [('The', 'DT'), Tree('ORGANIZATION', [('Natural', 'NNP'), ('Language', 'NNP'), ('Toolkit', 'NNP')]), (',', ','), ('or', 'CC'), ('more', 'JJR'), ('commonly', 'RB'), Tree('ORGANIZATION', [('NLTK', 'NNP')]), (',', ','), ('it', 'PRP'), ("'s", 'VBZ'), ('a', 'DT'), ('suite', 'NN'), ('of', 'IN'), ('libraries', 'NNS'), ('and', 'CC'), ('programs', 'NNS'), ('for', 'IN'), ('symbolic', 'JJ'), ('and', 'CC'), ('statistical', 'JJ'), ('natural', 'JJ'), ('language', 'NN'), ('processing', 'NN'), ('(', '('), Tree('ORGANIZATION', [('NLP', 'NNP')]), (')', ')'), ('for', 'IN'), Tree('GPE', [('English', 'NNP')]), ('written', 'VBN'), ('in', 'IN'), ('the', 'DT'), Tree('GPE', [('Python', 'NNP')]), ('programming', 'NN'), ('language', 'NN'), ('.', '.'), ('It', 'PRP'), ('was', 'VBD'), ('developed', 'VBN'), ('by', 'IN'), Tree('PERSON', [('Steven', 'NNP'), ('Bird', 'NNP')]), ('and', 'CC'), Tree('PERSON', [('Edward', 'NNP'), ('Loper', 'NNP')]), ('in', 'IN'), ('the', 'DT'), Tree('ORGANIZATION', [('Department', 'NNP')]), ('of', 'IN'), Tree('ORGANIZATION', [('Computer', 'NNP')]), ('and', 'CC'), Tree('ORGANIZATION', [('Information', 'NNP'), ('Science', 'NNP')]), ('at', 'IN'), ('the', 'DT'), Tree('ORGANIZATION', [('University', 'NNP')]), ('of', 'IN'), Tree('GPE', [('Pennsylvania', 'NNP')]), ('.', '.'), Tree('ORGANIZATION', [('NLTK', 'NNP')]), ('includes', 'VBZ'), ('graphical', 'JJ'), ('demonstrations', 'NNS'), ('and', 'CC'), ('sample', 'JJ'), ('data', 'NNS'), ('.', '.'), ('It', 'PRP'), ('is', 'VBZ'), ('accompanied', 'VBN'), ('by', 'IN'), ('a', 'DT'), ('book', 'NN'), ('that', 'WDT'), ('explains', 'VBZ'), ('the', 'DT'), ('underlying', 'JJ'), ('concepts', 'NNS'), ('behind', 'IN'), ('the', 'DT'), ('language', 'NN'), ('processing', 'NN'), ('tasks', 'NNS'), ('supported', 'VBN'), ('by', 'IN'), ('the', 'DT'), ('toolkit', 'NN'), (',', ','), ('plus', 'CC'), ('a', 'DT'), ('cookbook', 'NN'), ('.', '.')])

# Standard word tokenizer.
_word_tokenize = TreebankWordTokenizer().tokenize
def word_tokenize(text):
　　"""
　　Return a tokenized copy of *text*,
　　using NLTK's recommended word tokenizer
　　(currently :class:`.TreebankWordTokenizer`).
　　This tokenizer is designed to work on a sentence at a time.
　　"""
　　return _word_tokenize(text)

In [15]: from nltk.tokenize import TreebankWordTokenizer
 
In [16]: tokenizer = TreebankWordTokenizer()
 
In [17]: tokenizer.tokenize("This's one of the best NLP tools I've ever used")
Out[17]: 
['This',
 "'s",
 'one',
 'of',
 'the',
 'best',
 'NLP',
 'tools',
 'I',
 "'ve",
 'ever',
 'used']

In [21]: from nltk.tokenize import WordPunctTokenizer
 
In [22]: tokenizer = WordPunctTokenizer()
 
In [23]: tokenizer.tokenize("This's one of the best NLP tools I've ever used")
Out[23]: 
['This',
 "'",
 's',
 'one',
 'of',
 'the',
 'best',
 'NLP',
 'tools',
 'I',
 "'",
 've',
 'ever',
 'used']

In [24]: 
 
In [24]: from nltk.stem.porter import PorterStemmer
 
In [25]: porter_stemmer = PorterStemmer()
 
In [29]: plurals = ['caresses', 'flies', 'dies', 'mules', 'denied',
    ...:            'died', 'agreed', 'owned', 'humbled', 'sized',
    ...:            'meeting', 'stating', 'siezing', 'itemization',
    ...:            'sensational', 'traditional', 'reference', 'colonizer',
    ...:            'plotted']
 
In [30]: singles = [porter_stemmer.stem(plural) for plural in plurals]
 
In [31]: singles
Out[31]: 
['caress',
 'fli',
 'die',
 'mule',
 'deni',
 'die',
 'agre',
 'own',
 'humbl',
 'size',
 'meet',
 'state',
 'siez',
 'item',
 'sensat',
 'tradit',
 'refer',
 'colon',
 'plot']
 
In [33]: from nltk.stem.snowball import SnowballStemmer
 
# Snowball Stemmer 支持多种语言，这里需要指定英文包
In [34]: snowball_stemmer = SnowballStemmer("english")
 
In [35]: singles = [snowball_stemmer.stem(plural) for plural in plurals]
 
In [36]: singles
Out[36]: 
['caress',
 'fli',
 'die',
 'mule',
 'deni',
 'die',
 'agre',
 'own',
 'humbl',
 'size',
 'meet',
 'state',
 'siez',
 'item',
 'sensat',
 'tradit',
 'refer',
 'colon',
 'plot']
 
In [37]: from nltk.stem.lancaster import LancasterStemmer
 
In [38]: lancaster_stemmer = LancasterStemmer()
 
In [39]: singles = [lancaster_stemmer.stem(plural) for plural in plurals]
 
In [40]: singles
Out[40]: 
['caress',
 'fli',
 'die',
 'mul',
 'deny',
 'died',
 'agree',
 'own',
 'humbl',
 'siz',
 'meet',
 'stat',
 'siez',
 'item',
 'sens',
 'tradit',
 'ref',
 'colon',
 'plot']

In [41]: from nltk.stem import WordNetLemmatizer
 
In [42]: wordnet_lemmatizer = WordNetLemmatizer()
 
In [43]: singles = [wordnet_lemmatizer.lemmatize(plural) for plural in plurals]
 
In [44]: singles
Out[44]: 
['caress',
 'fly',
 'dy',
 'mule',
 'denied',
 'died',
 'agreed',
 'owned',
 'humbled',
 'sized',
 'meeting',
 'stating',
 'siezing',
 'itemization',
 'sensational',
 'traditional',
 'reference',
 'colonizer',
 'plotted']
 
In [45]: wordnet_lemmatizer.lemmatize('are')
Out[45]: 'are'
 
In [46]: wordnet_lemmatizer.lemmatize('is')
Out[46]: 'is'

In [47]: wordnet_lemmatizer.lemmatize('are', pos='v')
Out[47]: 'be'
 
In [48]: wordnet_lemmatizer.lemmatize('is', pos='v')
Out[48]: 'be'