Here is the pooling:
for host in ri.get_next_most_crowded_host():
results = pool.apply_async(self.__get_host_urls, [ri])
And the exception:
Exception in thread Thread-2:
Traceback (most recent call last):
File "/usr/lib/python3.2/threading.py", line 736, in _bootstrap_inner
self.run()
File "/usr/lib/python3.2/threading.py", line 689, in run
self._target(*self._args, **self._kwargs)
File "/usr/lib/python3.2/multiprocessing/pool.py", line 338, in _handle_tasks
put(task)
_pickle.PicklingError: Can't pickle <class 'method'>: attribute lookup builtins.method failed
Here is the functions trying to be called:
...
def __get_host_urls(self, ri):
pool = Comm.ThreadPool(self.poolsize)
sp = Comm.SocketPool(size=self.poolsize)
results = []
for lpi in ri.links[host]:
pool.add_task(self.__get_url_by_sp, host, self.count, sp, lpi, results)
self.count += 1
pool.wait_completion()
return results
def __get_url_by_sp(self, host, nr, sp, lpi, resultlist):
try:
resp = sp.urlopen(nr, host, lpi.url)
lpi.set_info('html', resp)
resultlist.append(lpi)
except Exception as e:
self.logger.exception(e)
...