class ResultQueue(object):
""" If the aggregator can't be reached for some reason, we want
to still keep calculating results and store them locally until we can reach the agg
again."""
# Variables for ResultQueue
#QUEUE_MAX_RESULTS = 150000 # We put results in a queue if aggregator can't be reached.
#QUEUE_BATCH_SIZE = 10000 # How many queue results we send back to the aggregator at once
def __init__(self, queue_max_results=None, queue_batch_size=None):
self.data = []
self.queue_max_results = queue_max_results
if self.queue_max_results is None:
self.queue_max_results = 150000
self.queue_batch_size = queue_batch_size
if self.queue_batch_size is None:
self.queue_batch_size = 10000
def pop(self):
# remove the oldest results first
self.data.pop(0)
# Pop only the most recent result
def pop_latest(self):
self.data.pop()
def pop_results(self, chunk=None):
if chunk is None:
chunk = self.queue_batch_size
# Pop and return 'chunk' of the newest items
if len(self.data) <= chunk:
result = self.data
self.data = []
else:
x = len(self.data) - chunk
result = self.data[x:]
self.data = self.data[:x]
return result
def push(self, result):
if len(self.data) == self.queue_max_results:
self.pop()
self.data.append(result)
def update(self, results):
if len(results) + len(self.data) <= self.queue_max_results:
self.data.extend(results)
else:
# Make room so we stay under QUEUE_MAX_RESULTS
free_space = self.queue_max_results - len(self.data)
removal = len(results) - free_space
self.data = self.data[removal:]
self.data.extend(results)
def isEmpty(self):
return len(self.data) == 0
def queueLength(self): return len(self.data)