docnow / twarc-hashtags Goto Github PK

def hash_retrieve(df):
    """
    df : dataframe of tweets
    Description: 
        The function takes as an object a df of tweets obtained via twarc and 
        returns a generator object.
    
    """

    for line, id in zip(df['entities.hashtags'], df['id']):
        if pd.isna(line):
            continue
        line = line.strip()
        data = json.loads(line)
        for hashtag in ensure_flattened(data):
            #print(hashtag['tag'],id)
            yield [hashtag['tag'], id]


# Using the generator to create a new dataframe.
a = pd.DataFrame(list(hash_retrieve(df)),
                 columns=['hashtag', 'id'])


a.hashtag.value_counts()