# constants
MAX_LENGTH = 200  # The length of each chunk
DATA_NAME = "your-hf-username/messagesv1"  # for uploading
ME = "Edward" # your name here!
base_model_name = "meta-llama/Llama-2-7b-chat-hf"  # for the tokenizer

# installs
!pip install ipywidgets datasets cryptography torch transformers sentencepiece matplotlib wordcloud

# imports
import csv
import datetime
import random
from collections import Counter
import datasets
from cryptography.fernet import Fernet  # to encrypt our texts
import tqdm
import torch
from transformers import AutoTokenizer
import matplotlib
import matplotlib.pyplot as plt
from wordcloud import WordCloud
%matplotlib inline

txts = []
filenames = ['all_text_messages.csv', 'all_whatsapp_messages.csv']
for filename in filenames:
    with open(filename, newline='') as csvfile:
        reader = csv.DictReader(csvfile, delimiter=',', quotechar='"')
        txts.extend(reader)
        
print(f'Read in a total of {len(txts):,} messages')

# For me, this outputs: Read in a total of 266,337 messages

class Message:

    def __init__(self, chat_session, message_type, text, when):
        self.name = chat_session
        self.sender = self.name if message_type == 'Incoming' else ME
        self.receiver = ME if message_type == 'Incoming' else self.name
        self.text = text
        self.when = datetime.datetime.strptime(when, '%Y-%m-%d %H:%M:%S')
        self.massage_text()

    def massage_text(self):
        # Replace special characters used in our format for training
        self.text = self.text.replace('\n','  ').replace(':',';').replace('#',';')
        # Indicate if the message is an image
        if self.text == '': self.text = '***'
    
    def should_exclude(self):
        return any(ch in self.name for ch in '+&,') or all(ch.isdigit() for ch in self.name)

# Create lists of messages

messages = [Message(t['Chat Session'], t['Type'], t['Text'], t['Message Date']) for t in txts]
messages = [m for m in messages if not m.should_exclude()]
print(f'A total of {len(messages):,} messages')

# For me, this outputs: A total of 242,230 messages

# Organize into dict with key = chat name, value = list of messages

chats = {}
for message in messages:
    if message.name not in chats:
        chats[message.name] = []
    chats[message.name].append(message)

# Sort the chats by time

for message_list in chats.values():
    message_list.sort(key = lambda m: m.when)

print(f'{sum([len(v) for v in chats.values()]):,} messages with {len(chats)} people')

# Gives me 242,230 messages with 472 people

AT_LEAST = 20
chats = {name: messages for name, messages in chats.items() if len(messages)>=AT_LEAST}
print(f'{sum([len(v) for v in chats.values()]):,} messages with {len(chats)} people')

# Gives me 240,985 messages with 290 people

# Prepare data
dates = [message.when for message in messages]

# Plot
fig, ax = plt.subplots(1, 1)
plt.title("How many texts I've sent over time")
ax.set_xlabel('Year')
ax.set_ylabel('How many texts');
ax.get_yaxis().set_major_formatter(matplotlib.ticker.FuncFormatter(lambda y, p: format(int(y), ',')))
_ = ax.hist(dates, bins=20, color='purple', rwidth=0.5)

# Prepare data
counter = Counter(message.name for message in messages)
results = counter.most_common(40)
names, counts = zip(*results)

# Plot
fig, ax = plt.subplots(1, 1, figsize = (10, 5))
ax.set_ylabel('How many texts');
ax.get_yaxis().set_major_formatter(matplotlib.ticker.FuncFormatter(lambda y, p: format(int(y), ',')))
plt.xticks(range(len(names)), names, rotation='vertical')
_ = ax.bar(names, counts, color ='teal', width = 0.5)

# Prepare data
text = ' '.join([message.text for message in messages])

# Alternatively - only my sent messages, or only chats with 1 person
# text = ' '.join([message.text for message in messages if message.sender==ME])
# text = ' '.join([message.text for message in chats['recipient name here']])

# Plot
wordcloud = WordCloud(max_font_size=60, max_words=50).generate(text)
plt.figure(figsize = (10, 10))
plt.imshow(wordcloud, interpolation="bilinear")
plt.axis("off")
plt.show()

¿Aún no estás inscrito en el curso completo?

Accede a todo el contenido del curso Ingeniería de LLMs e IA Generativa y domina las técnicas más avanzadas de IA generativa. ¡No pierdas esta oportunidad para transformar tu carrera profesional! 🎓

Apúntate al curso ahora

tokenizer = AutoTokenizer.from_pretrained(base_model_name, trust_remote_code=True)
tokenizer.pad_token = tokenizer.eos_token
tokenizer.padding_side = "right"

tokens = tokenizer.encode('### Edward: hello me ### Edward: hi me')
print(tokens)
print(tokenizer.decode(tokens))
print(tokenizer.convert_ids_to_tokens(tokens))