Commit ebad7e68 authored by Ian Dennis Miller's avatar Ian Dennis Miller

Merge branch 'master' of projects.sisrlab.com:idm/twitter-save

parents 0b7ba99a aefb02d7
Pipeline #268 passed with stage
in 21 seconds
......@@ -2,7 +2,7 @@
import json
import configparser
from twython import Twython, TwythonRateLimitError, TwythonAuthError
from twython import Twython, TwythonRateLimitError, TwythonAuthError, TwythonError
from . import make_uri
import time
import datetime
......@@ -93,10 +93,9 @@ class Downloader:
success = False
self.wait_for_reset()
except TwythonAuthError as e:
print(e)
self.reconnect()
success = False
time.sleep(5)
success = True
response = []
logging.info("Returned 401; Favorites are private")
for status in response:
self.snapshot.add_favorite_to_graph(subject, status)
......@@ -143,12 +142,22 @@ class Downloader:
next_cursor = -1
while(next_cursor):
response = endpoint(
screen_name=screen_name,
count=200,
cursor=next_cursor
)
next_cursor = response["next_cursor"]
success = False
while not success:
try:
response = endpoint(
screen_name=screen_name,
count=200,
cursor=next_cursor
)
next_cursor = response["next_cursor"]
success = True
except TwythonError as e:
print(e)
time.sleep(15)
self.reconnect()
time.sleep(15)
success = False
for acct in response["users"]:
acct_uri = make_uri(acct["screen_name"])
......@@ -156,6 +165,7 @@ class Downloader:
self.snapshot.g.add((subject, relationship, acct_uri))
iterations += 1
logging.info("Got page {0}".format(iterations))
# in case this is a secondary search, cap the number of pages retrieved
if secondary is True and iterations >= iterator_max:
......@@ -167,7 +177,6 @@ class Downloader:
time.sleep(61)
break
else:
logging.info("Got page {0}".format(iterations))
time.sleep(61)
def get_followers(self, screen_name, secondary=False):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment