Commit 4cb61427 authored by Ian Dennis Miller's avatar Ian Dennis Miller

try to catch 404 error

parent 1df90771
Pipeline #292 passed with stage
in 14 seconds
......@@ -15,6 +15,29 @@ class Downloader:
self.config = configparser.ConfigParser()
self.config.read('twitter-save.ini')
def all(self):
screen_name = self.snapshot.screen_name
success = False
self.connect()
try:
self.get_user(screen_name)
self.get_favorites(screen_name)
self.get_timeline(screen_name)
self.get_followers(screen_name, secondary)
self.get_leaders(screen_name, secondary)
success = True
except TwythonError as e:
if e.error_code == 404:
self.snapshot.logging.error("User {0} not found".format(screen_name))
else:
self.snapshot.logging.error(e)
time.sleep(5)
self.disconnect()
return(success)
def connect(self):
app_key = self.config["twitter"]["app_key"]
app_secret = self.config["twitter"]["app_secret"]
......
......@@ -254,19 +254,17 @@ class Snapshot:
"""
screen_name = screen_name.lower()
print("Download {0}".format(screen_name))
snapshot = cls(screen_name=screen_name, degree=1)
snapshot.logging.info("Create snapshot for {0}".format(screen_name))
print("Start {0}".format(screen_name))
if not snapshot.file_exists():
snapshot.downloader.connect()
snapshot.downloader.get_user(screen_name)
snapshot.downloader.get_favorites(screen_name)
snapshot.downloader.get_timeline(screen_name)
snapshot.downloader.get_followers(screen_name, secondary)
snapshot.downloader.get_leaders(screen_name, secondary)
snapshot.downloader.disconnect()
snapshot.save()
snapshot.logging.info("Create snapshot for {0}".format(screen_name))
success = snapshot.downloader.all()
if success:
snapshot.save()
else:
pass
# here, remove the user from the list so it isn't spidered again
else:
snapshot.logging.info("Skipping {0}; already downloaded".format(screen_name))
if not secondary:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment