NLP

Question 1

How to create a Doc object in Spacy?

Answer 1

import spacy
nlp = spacy.load(‘en_core_web_sm’)
doc = nlp(u’This is some text’)

Question 2

What is a span in Spacy?

Answer 2

span is a slice of Doc object, Doc[start:end]

Question 3

What are noun_chunks in Spacy?

Answer 3

base noun phrases

Question 4

How to visualize in Spacy?

Answer 4

from spacy import displacy

displacy. render(doc, style=’dep|ent’, jupyter=True, options={‘distance’: 110})
displacy. serve(doc, style=’dep’)
127. 0.0.1:port

Question 5

How to get a list of stopwords in Spacy?

Answer 5

import spacy
nlp = spacy.load(‘en_core_web_sm’)
print(nlp.Defaults.stop_words)

Question 6

How to check if a word is a stop word in Spacy?

Answer 6

nlp.vocab[‘word’].is_stop

Question 7

How to add a stop word in Spacy?

Answer 7

nlp. Defaults.stop_words.add(‘btw’)

nlp. vocab[‘btw’].is_stop = True

Question 8

How to remove a stop word in Spacy?

Answer 8

nlp. Defaults.stop_words.remove(‘btw’)

nlp. vocab[‘btw’].is_stop = False

Question 9

How to build a library of token patterns in Spacy?

Answer 9

from spacy.matcher import Matcher
matcher = Matcher(nlp.vocab)
pattern1 = [{‘LOWER’: ‘solarpower’}]
pattern2 = [{‘LOWER’: ‘solar’}, {‘IS_PUNCT’: True, ‘OP’:’*’}, {‘LOWER’: ‘power’}]
matcher.add(‘SolarPower’, None, pattern1, pattern2)
found_matches = matcher(doc)
print(found_matches)

Question 10

How to use a matcher for terminology lists in Spacy?

Answer 10

from spacy.matcher import PhraseMatcher
matcher = PhraseMatcher(nlp.vocab)
phrase_list = [‘voodoo economics’, ‘supply-side economics’, ‘trickle-down economics’, ‘free-market economics’]
phrase_patterns = [nlp(text) for text in phrase_list]
matcher.add(‘VoodooEconomics’, None, *phrase_patterns)
matches = matcher(doc)

Question 11

How to count POS frequency in a text in Spacy?

Answer 11

POS_counts = doc.count_by(spacy.attrs.POS)
for k,v in sorted(POS_counts.items()):
print(f’{k}. {doc.vocab[k].text:{5}}: {v}’)

Question 12

How to add a named entity in Spacy?

Answer 12

from spacy.tokens import Span
ORG = doc.vocab.strings[u’ORG’]
new_ent = Span(doc, start, end, label=ORG)
doc.ents = list(doc.ents) + [new_ent]

Question 13

How to add named entities to all matching spans in Spacy?

Answer 13

from spacy.matcher import PhraseMatcher
matcher = PhraseMatcher(nlp.vocab)
phrase_list = [‘vacuum cleaner’, ‘vacuum-cleaner’]
phrase_patterns = [nlp(text) for text in phrase_list]
matcher.add(‘newproduct’, None, *phrase_patterns)
matches = matcher(doc)
from spacy.tokens import Span
PROD = doc.vocab.strings[u’PRODUCT’]
new_ents = [Span(doc, match[1],match[2],label=PROD) for match in matches]
doc.ents = list(doc.ents) + new_ents

Question 14

How to add a new rule to pipeline in Spacy?

Answer 14

def set_custom_boundaries(doc):
    for token in doc[:-1]:
        if token.text == ';':
            doc[token.i+1].is_sent_start = True
    return doc

nlp.add_pipe(set_custom_boundaries, before=’parser’)

Question 15

How to change segmentation rules in Spacy?

Answer 15

from spacy.pipeline import SentenceSegmenter

def split_on_newlines(doc):
    start = 0
    seen_newline = False
    for word in doc:
        if seen_newline:
            yield doc[start:word.i]
            start = word.i
            seen_newline = False
        elif word.text.startswith('\n'): # handles multiple occurrences
            seen_newline = True
    yield doc[start:]      # handles the last group of tokens

sbd = SentenceSegmenter(nlp.vocab, strategy=split_on_newlines)
nlp.add_pipe(sbd)

Question 16

TF-IDF

Answer 16

term frequency * (1/document frequency)

Question 17

How to extract text features using scikit-learn?

Answer 17

from sklearn.feature_extraction.text import CountVectorizer
cv = CountVectorizer()
X_train_count = cv.fit_transform(X_train)

Question 18

How to make a prediction for a new text using classification model in scikit-learn?

Answer 18

text_clf.predict([‘some text here’])

Question 19

How to clear dataset from missing data in scikit-learn?

Answer 19

df.dropna(inplace=True)

Question 20

How to clear data set from empty strings in scikit-learn?

Answer 20

blanks = []

for i, lb, rv in df.itertuples():
if rv.isspace():
blanks.append(i)
df.drop(blanks,inplace=True)

Question 21

How does Word2vec train words agains words in a corpus?

Answer 21

• Using context to predict a target word (continuous bag of words)
• Using a word to predict a target context (skip-gram)

Question 22

check word similarity in Spacy

Answer 22

tokens = nlp(u’fox dog animal’)

for token1 in tokens:
for token2 in tokens:
print(token1.text, token2.text, token1.similarity(token2))

Question 23

vector arithmetic in Spacy (finding similar words using vectors)

Answer 23

from scipy import spatial

cosine_similarity = lambda x, y: 1 - spatial.distance.cosine(x, y)

king = nlp.vocab['king'].vector
man = nlp.vocab['man'].vector
woman = nlp.vocab['woman'].vector

# Now we find the closest vector in the vocabulary to the result of "man" - "woman" + "queen"
new_vector = king - man + woman
computed_similarities = []

for word in nlp.vocab:
# Ignore words without vectors and mixed-case words:
if word.has_vector:
if word.is_lower:
if word.is_alpha:
similarity = cosine_similarity(new_vector, word.vector)
computed_similarities.append((word, similarity))

computed_similarities = sorted(computed_similarities, key=lambda item: -item[1])

print([w[0].text for w in computed_similarities[:10]])

Question 24

How to do sentiment analysis using NLTK?

Answer 24

import nltk
nltk.download(‘vader_lexicon’)
from nltk.sentiment.vader import SentimentIntensityAnalyzer
sid = SentimentIntensityAnalyzer()
import pandas as pd
df = pd.read_csv(‘../TextFiles/amazonreviews.tsv’, sep=’\t’)
df.head()
df[‘scores’] = df[‘review’].apply(lambda review: sid.polarity_scores(review))
df[‘compound’] = df[‘scores’].apply(lambda d: d[‘compound’])
df[‘score’] = df[‘compound’].apply(lambda s: ‘pos’ if s>= 0 else ‘neg’)

Question 25

How to use Latent Dirichlet Allocation for topic modelling?

Answer 25

import pandas as pd npr = pd.read_csv('npr.csv') from sklearn.feature_extraction.text import CountVectorizer cv = CountVectorizer(max_df = 0.95, min_df = 2, stop_words = 'english') dtf = cv.fit_transform(npr['Article']) from sklearn.decomposition import LatentDirichletAllocation LDA = LatentDirichletAllocation(n_components=7, random_state=42) LDA.fit(dtf) single_topic = LDA.components_[0] top_ten_words = single_topic.argsort()[-10:] for i in top_ten_words: print(cv.get_feature_names()[i]) for index,topic in enumerate(LDA.components_): print(f'THE TOP 15 WORDS FOR TOPIC #{index}') print([cv.get_feature_names()[i] for i in topic.argsort()[-15:]]) print('\n') topic_results = LDA.transform(dtf) npr['Topic'] = topic_results.argmax(axis=1)

Question 26

How to use non-negative matrix vectorization for topic modelling?

Answer 26

import pandas as pd npr = pd.read_csv('npr.csv') from sklearn.feature_extraction.text import TfidfVectorizer tfidf = TfidfVectorizer(max_df=0.95, min_df=2, stop_words='english') dtf = tfidf.fit_transform(npr['Article']) from sklearn.decomposition import NMF nmf_model = NMF(n_components=7,random_state=42) nmf_model.fit(dtf) for index, topic in enumerate(nmf_model.components_): print(f'Top 15 words for topic # {index}') print([tfidf.get_feature_names()[i] for i in topic.argsort()[-15:]]) print('\n') topic_results = nmf_model.transform(dtf) npr['Topic'] = topic_results.argmax(axis=1)

Question 27

text generation using Keras

Answer 27

``` def read_file(filepath): with open(filepath) as f: file_text = f.read() ``` return file_text import spacy nlp = spacy.load('en',disable=['parser','tagger','ner']) nlp_max_length = 1198623 ``` def separate_punct(doc_text): return [token.text.lower() for token in nlp(doc_text) if token.text not in '\n\n \n\n\n!"-#$%&()--.*+,-/:;<=>?@[\\]^_`{|}~\t\n '] ``` d = read_file('moby_dick_four_chapters.txt') tokens = separate_punct(d) ``` train_len = 24 + 1 text_sequences = [] for i in range(train_len, len(tokens)): seq = tokens[i-train_len:i] text_sequences.append(seq) ``` from keras.preprocessing.text import Tokenizer tokenizer = Tokenizer() tokenizer.fit_on_texts(text_sequences) sequences = tokenizer.texts_to_sequences(text_sequences) vocabulary_size = len(tokenizer.word_counts) import numpy as np sequences = np.array(sequences) from keras.utils import to_categorical ``` X = sequences[:,:-1] y = sequences[:,-1] ``` ``` y = to_categorical(y,num_classes=vocabulary_size+1) seq_len = X.shape[1] ``` from keras.models import Sequential from keras.layers import Dense, LSTM, Embedding ``` def create_model(vocab_size, seq_len): model = Sequential() model.add(Embedding(vocab_size, seq_len, input_length=seq_len)) model.add(LSTM(50,return_sequences=True)) model.add(LSTM(50)) model.add(Dense(50, activation='relu')) model.add(Dense(vocab_size,activation='softmax')) model.compile(loss='categorical_crossentropy',optimizer='adam',metrics=['accuracy']) model.summary() return model ``` model = create_model(vocabulary_size+1,seq_len) from pickle import dump, load model. fit(X,y,batch_size=128,epochs=50,verbose=1) model. save('mobydick_model.h5') dump(tokenizer,open('mysimpletokenzr','wb')) from keras.preprocessing.sequence import pad_sequences def generate_text(model, tokenizer, seq_len, seed_text, num_gen_words): ''' INPUTS: model : model that was trained on text data tokenizer : tokenizer that was fit on text data seq_len : length of training sequence seed_text : raw string text to serve as the seed num_gen_words : number of words to be generated by model ''' ``` # Final Output output_text = [] ``` ``` # Intial Seed Sequence input_text = seed_text ``` ``` # Create num_gen_words for i in range(num_gen_words): ``` ``` # Take the input text string and encode it to a sequence encoded_text = tokenizer.texts_to_sequences([input_text])[0] ``` ``` # Pad sequences to our trained rate (50 words in the video) pad_encoded = pad_sequences([encoded_text], maxlen=seq_len, truncating='pre') ``` ``` # Predict Class Probabilities for each word pred_word_ind = model.predict_classes(pad_encoded, verbose=0)[0] ``` ``` # Grab word pred_word = tokenizer.index_word[pred_word_ind] ``` ``` # Update the sequence of input text (shifting one over with the new word) input_text += ' ' + pred_word ``` output_text.append(pred_word) ``` # Make it look like a sentence. return ' '.join(output_text) ``` import random random.seed(101) random_pick = random.randint(0, len(text_sequences)) random_seed_text = text_sequences[random_pick] seed_text = ' '.join(random_seed_text) generate_text(model, tokenizer, seq_len, seed_text, 25)

Question 28

Building a Q&A chatbot with Keras

Answer 28

import pickle import numpy as np ``` with open('train_qa.txt', 'rb') as f: train_data = pickle.load(f) ``` ``` with open('test_qa.txt','rb') as f: test_data = pickle.load(f) ``` all_data = train_data + test_data vocab = set() for story, question, answer in all_data: vocab = vocab.union(set(story)) vocab = vocab.union(set(question)) vocab. add('no') vocab. add('yes') vocab_len = len(vocab) + 1 all_story_lens = [len(data[0]) for data in all_data] max_story_len = max(all_story_lens) max_question_len = max([len(data[1]) for data in all_data]) from keras.preprocessing.sequence import pad_sequences from keras.preprocessing.text import Tokenizer tokenizer = Tokenizer(filters=[]) tokenizer.fit_on_texts(vocab) def vectorize_stories(data, word_index=tokenizer.word_index, max_story_len=max_story_len,max_question_len=max_question_len): X = [] Xq = [] Y = [] for story, query, answer in data: x = [word_index[word.lower()] for word in story] xq = [word_index[word.lower()] for word in query] y = np.zeros(len(word_index)+1) y[word_index[answer]] = 1 X.append(x) Xq.append(xq) Y.append(y) return (pad_sequences(X,maxlen=max_story_len),pad_sequences(Xq,maxlen=max_question_len), np.array(Y)) inputs_train, queries_train, answers_train = vectorize_stories(train_data) inputs_test, queries_test, answers_test = vectorize_stories(test_data) from keras.models import Sequential, Model from keras.layers.embeddings import Embedding from keras.layers import Input, Activation, Dense, Permute, Dropout, add, dot, concatenate, LSTM ``` input_sequence = Input((max_story_len,)) question = Input((max_question_len,)) ``` vocab_size = len(vocab) + 1 input_encoder_m = Sequential() input_encoder_m.add(Embedding(input_dim=vocab_size,output_dim=64)) input_encoder_m.add(Dropout(0.3)) input_encoder_c = Sequential() input_encoder_c.add(Embedding(input_dim=vocab_size,output_dim=max_question_len)) input_encoder_c.add(Dropout(0.3)) question_encoder = Sequential() question_encoder.add(Embedding(input_dim=vocab_size,output_dim=64, input_length=max_question_len)) question_encoder.add(Dropout(0.3)) ``` input_encoded_m = input_encoder_m(input_sequence) input_encoded_c = input_encoder_c(input_sequence) question_encoded = question_encoder(question) ``` ``` match = dot([input_encoded_m,question_encoded],axes=(2,2)) match = Activation('softmax')(match) ``` ``` response = add([match,input_encoded_c]) response = Permute((2,1))(response) ``` answer = concatenate([response,question_encoded]) answer = LSTM(32)(answer) ``` answer = Dropout(0.5)(answer) answer = Dense(vocab_size)(answer) ``` answer = Activation('softmax')(answer) model = Model([input_sequence,question],answer) model. compile(optimizer='rmsprop',loss='categorical_crossentropy',metrics=['accuracy']) model. summary() history = model.fit([inputs_train,queries_train],answers_train,batch_size=32,epochs=100,validation_data=([inputs_test,queries_test],answers_test)) pred_results = model.predict([inputs_test,queries_test]) my_story = 'John left the kitchen . Sandra dropped the football in the garden .' my_question = 'Is the football in the garden ?' mydata = [(my_story.split(),my_question.split(),'yes')] mystory, myques, myans = vectorize_stories(mydata) pred_results = model.predict([mystory, myques]) val_max = np.argmax(pred_results[0]) for key, val in tokenizer.word_index.items(): if val == val_max: k = key k

Question 29

Building a corpus from individual files using Pandas and python.os

Answer 29

import numpy as np import pandas as pd import os row_list = [] for subdir in ['neg','pos']: for folder, subfolders, filenames in os.walk('../moviereviews/'+subdir): for file in filenames: d = {'label':subdir} # assign the name of the subdirectory to the label field with open('moviereviews/'+subdir+'/'+file) as f: if f.read(): # handles the case of empty files, which become NaN on import f.seek(0) d['review'] = f.read() # assign the contents of the file to the review field row_list.append(d) break df = pd.DataFrame(row_list)

Question 30

how to efficiently process lots of text data in Spacy?

Answer 30

docs = list(nlp.pipe(LOTS_OF_TEXTS))

Question 31

disabling pipeline components in Spacy

Answer 31

``` # Disable tagger and parser with nlp.disable_pipes("tagger", "parser"): # Process the text and print the entities doc = nlp(text) print(doc.ents) ```

Question 32

custom extensions for docs, tokens and spans in Spacy

Answer 32

from spacy.tokens import Token ``` # Define getter function def get_is_color(token): colors = ["red", "yellow", "blue"] return token.text in colors ``` ``` # Set extension on the Token with getter Token.set_extension("is_color", getter=get_is_color) ``` doc = nlp("The sky is blue.") print(doc[3]._.is_color, "-", doc[3].text) from spacy.tokens import Span ``` # Define getter function def get_has_color(span): colors = ["red", "yellow", "blue"] return any(token.text in colors for token in span) ``` ``` # Set extension on the Span with getter Span.set_extension("has_color", getter=get_has_color) ``` doc = nlp("The sky is blue.") print(doc[1:4]._.has_color, "-", doc[1:4].text) print(doc[0:2]._.has_color, "-", doc[0:2].text) from spacy.tokens import Doc ``` # Define method with arguments def has_token(doc, token_text): in_doc = token_text in [token.text for token in doc] return in_doc ``` ``` # Set extension on the Doc with method Doc.set_extension("has_token", method=has_token) ``` doc = nlp("The sky is blue.") print(doc._.has_token("blue"), "- blue") print(doc._.has_token("cloud"), "- cloud")

Question 33

how to update a statistical model in Spacy

Answer 33

TRAINING_DATA = [ ("How to preorder the iPhone X", {"entities": [(20, 28, "GADGET")]}) # And many more examples... ] # Loop for 10 iterations for i in range(10): # Shuffle the training data random.shuffle(TRAINING_DATA) # Create batches and iterate over them for batch in spacy.util.minibatch(TRAINING_DATA): # Split the batch in texts and annotations texts = [text for text, annotation in batch] annotations = [annotation for text, annotation in batch] # Update the model nlp.update(texts, annotations) ``` # Save the model nlp.to_disk(path_to_model) ```