Tukukkk

I wrote a Twitter scraper using the tweepy so I can scrape user information and tweets. Given that the free API doesn't let me get the number of messages per tweet, I had to rely on BeautifulSoup to do so.

class TweetAPI():



    def __init__(self, k1,k2,k3,k4):

        self.key = k1

        self.secret_key = k2

        self.token = k3

        self.secret_token = k4



        auth = tweepy.OAuthHandler(self.key, self.secret_key)

        auth.set_access_token(self.token, self.secret_token)



        api = tweepy.API(auth, wait_on_rate_limit=True)

        self.api = api



    def tweet_getter(self, user_id, n):

        api = self.api

        tweets = 

        try:

            for tweet in tweepy.Cursor(self.api.user_timeline, id = user_id).items(n):

                url = "https://twitter.com/{}/status/{}".format(user_id, tweet.id_str)

                page = requests.get(url)

                page = BeautifulSoup(page.content, 'html.parser')

                message_count = int(page.find('span',{"class":"ProfileTweet-actionCount"}).text.strip().split()[0])

                temp = [user_id, tweet.created_at, tweet.id_str,

                        tweet.favorite_count, tweet.retweet_count, message_count, tweet.text]

                tweets.append(temp)

            return tweets

        except:

            print("Unable to get user @{} tweets".format(user_id))

            pass

I'm worried about two things:

1. Does the wait_on_rate_limit=True really prevent the over doing request in terms of limits?




2. Should I add an artificial delayer to the in the BeautifulSoup part in order to avoid getting blocked off from the page content of the Twitter website?