Redis broken pipe errors
The preliminary alert for S190707q failed due to the following Redis error:
[2019-07-07 02:37:24,255: ERROR/ForkPoolWorker-32] Task gwcelery.tasks.orchestrator._download[b20d6d89-d458-4d5c-b703-7ae209b39c07] raised unexpected: ConnectionError('Error 32 while writing to socket. Broken pipe.',)
Traceback (most recent call last):
File "/home/emfollow/.local/lib/python3.6/site-packages/celery/backends/asynchronous.py", line 154, in add_pending_result
self._maybe_resolve_from_buffer(result)
File "/home/emfollow/.local/lib/python3.6/site-packages/celery/backends/asynchronous.py", line 160, in _maybe_resolve_from_buffer
result._maybe_set_cache(self._pending_messages.take(result.id))
File "/home/emfollow/.local/lib/python3.6/site-packages/celery/utils/collections.py", line 877, in take
raise self.Empty()
queue.Empty
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/emfollow/.local/lib/python3.6/site-packages/redis/connection.py", line 600, in send_packed_command
self._sock.sendall(item)
BrokenPipeError: [Errno 32] Broken pipe
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/emfollow/.local/lib/python3.6/site-packages/celery/app/trace.py", line 446, in trace_task
priority=task_priority
File "/home/emfollow/.local/lib/python3.6/site-packages/celery/canvas.py", line 1277, in apply_async
return self.run(tasks, body, args, task_id=task_id, **options)
File "/home/emfollow/.local/lib/python3.6/site-packages/celery/canvas.py", line 1316, in run
bodyres = body.freeze(task_id, root_id=root_id)
File "/home/emfollow/.local/lib/python3.6/site-packages/celery/canvas.py", line 1238, in freeze
parent_id=parent_id, root_id=root_id, chord=self.body)
File "/home/emfollow/.local/lib/python3.6/site-packages/celery/canvas.py", line 1138, in freeze
return self.app.GroupResult(gid, results)
File "/home/emfollow/.local/lib/python3.6/site-packages/celery/result.py", line 899, in __init__
ResultSet.__init__(self, results, **kwargs)
File "/home/emfollow/.local/lib/python3.6/site-packages/celery/result.py", line 532, in __init__
self._on_full = ready_barrier or barrier(results)
File "/home/emfollow/.local/lib/python3.6/site-packages/vine/synchronization.py", line 57, in __init__
[self.add_noincr(p) for p in promises or []]
File "/home/emfollow/.local/lib/python3.6/site-packages/vine/synchronization.py", line 57, in <listcomp>
[self.add_noincr(p) for p in promises or []]
File "/home/emfollow/.local/lib/python3.6/site-packages/vine/synchronization.py", line 82, in add_noincr
p.then(self)
File "/home/emfollow/.local/lib/python3.6/site-packages/celery/result.py", line 119, in then
self.backend.add_pending_result(self, weak=weak)
File "/home/emfollow/.local/lib/python3.6/site-packages/celery/backends/asynchronous.py", line 156, in add_pending_result
self._add_pending_result(result.id, result, weak=weak)
File "/home/emfollow/.local/lib/python3.6/site-packages/celery/backends/asynchronous.py", line 166, in _add_pending_result
self.result_consumer.consume_from(task_id)
File "/home/emfollow/.local/lib/python3.6/site-packages/celery/backends/redis.py", line 136, in consume_from
self._consume_from(task_id)
File "/home/emfollow/.local/lib/python3.6/site-packages/celery/backends/redis.py", line 142, in _consume_from
self._pubsub.subscribe(key)
File "/home/emfollow/.local/lib/python3.6/site-packages/redis/client.py", line 3096, in subscribe
ret_val = self.execute_command('SUBSCRIBE', *iterkeys(new_channels))
File "/home/emfollow/.local/lib/python3.6/site-packages/redis/client.py", line 3009, in execute_command
self._execute(connection, connection.send_command, *args)
File "/home/emfollow/.local/lib/python3.6/site-packages/redis/client.py", line 3013, in _execute
return command(*args)
File "/home/emfollow/.local/lib/python3.6/site-packages/redis/connection.py", line 620, in send_command
self.send_packed_command(self.pack_command(*args))
File "/home/emfollow/.local/lib/python3.6/site-packages/redis/connection.py", line 613, in send_packed_command
(errno, errmsg))
redis.exceptions.ConnectionError: Error 32 while writing to socket. Broken pipe.