Commit c80485a9 authored by Ian Dennis Miller's avatar Ian Dennis Miller

error handling for slow_get

parent 9f08793a
Pipeline #266 passed with stage
in 14 seconds
......@@ -2,7 +2,7 @@
import json
import configparser
from twython import Twython, TwythonRateLimitError, TwythonAuthError
from twython import Twython, TwythonRateLimitError, TwythonAuthError, TwythonError
from . import make_uri
import time
import datetime
......@@ -143,12 +143,22 @@ class Downloader:
next_cursor = -1
while(next_cursor):
response = endpoint(
screen_name=screen_name,
count=200,
cursor=next_cursor
)
next_cursor = response["next_cursor"]
success = False
while not success:
try:
response = endpoint(
screen_name=screen_name,
count=200,
cursor=next_cursor
)
next_cursor = response["next_cursor"]
success = True
except TwythonError as e:
print(e)
time.sleep(15)
self.reconnect()
time.sleep(15)
success = False
for acct in response["users"]:
acct_uri = make_uri(acct["screen_name"])
......@@ -156,6 +166,7 @@ class Downloader:
self.snapshot.g.add((subject, relationship, acct_uri))
iterations += 1
logging.info("Got page {0}".format(iterations))
# in case this is a secondary search, cap the number of pages retrieved
if secondary is True and iterations >= iterator_max:
......@@ -167,7 +178,6 @@ class Downloader:
time.sleep(61)
break
else:
logging.info("Got page {0}".format(iterations))
time.sleep(61)
def get_followers(self, screen_name, secondary=False):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment