# pip3 install transformers
from transformers import BertTokenizer, BertModel

tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')

text2 = "I got a book and after I book for an hour, it's time to book it."
marked_text2 = "[CLS] " + text2 + " [SEP]"
tokenized_text2 = tokenizer.tokenize(marked_text2)

print("input sentence #2:") 
print(text2)
print()

print("Tokens from input sentence #2:") 
print(tokenized_text2)
print()

# Map token strings to their vocabulary indices:
indexed_tokens2 = tokenizer.convert_tokens_to_ids(tokenized_text2)

# Display the words with their indices:
for pair in zip(tokenized_text2, indexed_tokens2):
  print('{:<12} {:>6,}'.format(pair[0], pair[1]))

