[Py3.5] Sent_tokenization, Noun-Phrasing and NER w/ spaCy

/*******************************************************************************************************************
-- Title : [Py3.5] Sent_tokenization, Noun-Phrasing and NER  w/ spaCy
-- Reference : blog.sharepointexperience.com/2016/01/nlp-and-sharepoint-part-1/
-- Key word : nlp spacy pandas set_option txt open text  read sent_token word_token tokenizer sentence
                  noun phrase chunk chunker chuning tokenizing named entity recognizion ner 자연어처리
                  텍스트 단어 토큰 문장 토큰 토크나이저 청킹 청크 청커 
*******************************************************************************************************************/

■ Scripts

import spacy
import pandas as pd
 
# Ref : http://blog.sharepointexperience.com/2016/01/nlp-and-sharepoint-part-1/
 
########################################
# -- Set Dataframe Option
########################################
pd.set_option('display.max_colwidth', -1)
 
 
########################################
# -- Sourcing and Declaration
########################################
 
# --
# -- REad Source from file
# --
fl_source = open("review.txt", "r")
source = fl_source.read()
 
print(source[:150])
print("... source", "." * 100, "\n")
 
# --
# -- Making document
# --
 
nlp = spacy.load("en")
doc = nlp(source)
 
 
########################################
# -- Sentence Tokenizing
########################################
lst_sent = [sent_token.orth_ for sent_token in doc.sents]
df_sent = pd.DataFrame(lst_sent)
 
print(lst_sent)
print(",,, list_sent_token", "," * 100, "\n")
 
print(df_sent.head(5))
print(",,, df_sent_token", "," * 100, "\n")
 
 
########################################
# -- Noun Phrase
########################################
lst_nounphrase = [[nchunk.orth_, nchunk, nchunk.root, nchunk.root.head.orth_] for nchunk in doc.noun_chunks]
df_nounphrase = pd.DataFrame(lst_nounphrase)
 
print(lst_nounphrase)
print(";;; lst_nounphrase", ";" * 100, "\n")
 
print(df_nounphrase.head(5))
print(";;; df_nounphrase", ";" * 100, "\n")
 
 
########################################
# -- Named Entity Recognition(NER)
########################################
 
entities = list(doc.ents)
print("There were {} entities found".format(len(entities)))
 
lst_ner_org_people = [entity.orth_ for entity in entities if entity.label_ in ['ORG','PERSON']]
df_ner_org_people = pd.DataFrame(lst_ner_org_people)
 
print(df_ner_org_people)
print("^^^ df_ner", "^" * 100, "\n")
 

■ Files

review.txt