Commit 0e0add67 authored by Ian Dennis Miller's avatar Ian Dennis Miller

downloader inside snapshot

parent 5b6a5bb0
Pipeline #257 passed with stage
in 14 seconds
......@@ -25,7 +25,6 @@ class Downloader:
self.config = configparser.ConfigParser()
self.config.read('twitter-save.ini')
self.connect()
def connect(self):
app_key = self.config["twitter"]["app_key"]
......@@ -37,8 +36,11 @@ class Downloader:
self.get_limits()
def reconnect(self):
def disconnect(self):
del(self.twitter)
def reconnect(self):
self.disconnect()
self.connect()
def wait_for_reset(self):
......
......@@ -8,6 +8,7 @@ from rdflib.namespace import RDF
from . import make_uri
from .transformation import Transformation
from .query import Query
from .downloader import Downloader
class Snapshot:
......@@ -19,6 +20,7 @@ class Snapshot:
self.transformation = Transformation(self)
self.query = Query(self)
self.downloader = Downloader(self)
if store is None:
self.store = IOMemory()
......@@ -294,30 +296,29 @@ class Snapshot:
By default, the individual RDF graphs have radius of degree = 1.
"""
from .downloader import Downloader
screen_name = screen_name.lower()
snapshot = cls(screen_name=screen_name, degree=1)
if not snapshot.file_exists():
downloader = Downloader(snapshot)
downloader.get_user(screen_name)
downloader.get_favorites(screen_name)
downloader.get_timeline(screen_name)
downloader.get_followers(screen_name, secondary)
downloader.get_leaders(screen_name, secondary)
snapshot.downloader.connect()
snapshot.downloader.get_user(screen_name)
snapshot.downloader.get_favorites(screen_name)
snapshot.downloader.get_timeline(screen_name)
snapshot.downloader.get_followers(screen_name, secondary)
snapshot.downloader.get_leaders(screen_name, secondary)
snapshot.downloader.disconnect()
snapshot.save()
else:
if not secondary:
snapshot.load()
downloader.logging.info("Skipping {0} because snapshot exists".format(screen_name))
snapshot.downloader.logging.info("Skipping {0} because snapshot exists".format(screen_name))
# if secondary is false, do not spider recursively
if not secondary:
downloader.logging.info("Spider connections from {} ".format(screen_name))
snapshot.downloader.logging.info("Spider connections from {} ".format(screen_name))
for func in [snapshot.query.followers, snapshot.query.leaders]:
downloader.logging.info(func)
snapshot.downloader.logging.info(func)
screen_names = func(screen_name)
for sub_screen_name in screen_names:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment