Double free of snr_series can occur on long (2wk) runs leading to segfault
The pipeline attempts to free memory that has already been freed. According to the pipeline's stdout, this seems to occur during the writing of the zerolag file, here is an example backtrace:
*** Error in `python': munmap_chunk(): invalid pointer: 0x00007f6a0403eaf0 ***
======= Backtrace: =========
#0 0x00007ffff6d41387 in raise () from /lib64/libc.so.6
#1 0x00007ffff6d42a78 in abort () from /lib64/libc.so.6
#2 0x00007ffff6d83f67 in __libc_message () from /lib64/libc.so.6
#3 0x00007ffff6d8a474 in malloc_printerr () from /lib64/libc.so.6
#4 0x00007fff779defb2 in free_snr_series (wrapped_postcohtable=wrapped_postcohtable@entry=0x7fff1864fca0) at _postcohtable.c:595
#5 0x00007fff779df0a3 in __del__ (self=0x7fff1864fca0) at _postcohtable.c:615
#6 0x00007ffff7a97e9e in subtype_dealloc () from /home/luke.davis/spiir-scripts/build/dependencies/bin/../lib/libpython2.7.so.1.0
#7 0x00007ffff7a6e8c4 in list_clear () from /home/luke.davis/spiir-scripts/build/dependencies/bin/../lib/libpython2.7.so.1.0
#8 0x00007ffff7b10da2 in collect () from /home/luke.davis/spiir-scripts/build/dependencies/bin/../lib/libpython2.7.so.1.0
#9 0x00007ffff7b117b1 in _PyObject_GC_Malloc () from /home/luke.davis/spiir-scripts/build/dependencies/bin/../lib/libpython2.7.so.1.0
#10 0x00007ffff7a974bc in PyType_GenericAlloc () from /home/luke.davis/spiir-scripts/build/dependencies/bin/../lib/libpython2.7.so.1.0
#11 0x00007fff779df219 in from_buffer (cls=0x5555569fe750, args=<optimized out>) at _postcohtable.c:636
#12 0x00007ffff7ae0645 in PyEval_EvalFrameEx () from /home/luke.davis/spiir-scripts/build/dependencies/bin/../lib/libpython2.7.so.1.0
#13 0x00007ffff7ae0ddc in PyEval_EvalFrameEx () from /home/luke.davis/spiir-scripts/build/dependencies/bin/../lib/libpython2.7.so.1.0
#14 0x00007ffff7ae2519 in PyEval_EvalCodeEx () from /home/luke.davis/spiir-scripts/build/dependencies/bin/../lib/libpython2.7.so.1.0
#15 0x00007ffff7a6b00a in function_call () from /home/luke.davis/spiir-scripts/build/dependencies/bin/../lib/libpython2.7.so.1.0
#16 0x00007ffff7a467a3 in PyObject_Call () from /home/luke.davis/spiir-scripts/build/dependencies/bin/../lib/libpython2.7.so.1.0
#17 0x00007ffff7a5553d in instancemethod_call () from /home/luke.davis/spiir-scripts/build/dependencies/bin/../lib/libpython2.7.so.1.0
#18 0x00007ffff7a467a3 in PyObject_Call () from /home/luke.davis/spiir-scripts/build/dependencies/bin/../lib/libpython2.7.so.1.0
#19 0x00007ffff7ad8708 in PyEval_CallObjectWithKeywords () from /home/luke.davis/spiir-scripts/build/dependencies/bin/../lib/libpython2.7.so.1.0
#20 0x00007fffedd13462 in pyg_closure_marshal () from /home/luke.davis/spiir-scripts/build/dependencies/lib/python2.7/site-packages/gobject/_gobject.so
#21 0x00007fffee8f9a45 in g_closure_invoke (closure=0x55555871e960, return_value=return_value@entry=0x0, n_param_values=1, param_values=param_values@entry=0x7fff657397a0, invocation_hint=invocation_hint@entry=0x7fff65739740) at ../gobject/gclosure.c:810
#22 0x00007fffee90c622 in signal_emit_unlocked_R (node=node@entry=0x55555848c740, detail=detail@entry=0, instance=instance@entry=0x55555955a290, emission_return=emission_return@entry=0x0, instance_and_params=instance_and_params@entry=0x7fff657397a0)
at ../gobject/gsignal.c:3641
#23 0x00007fffee9152d5 in g_signal_emit_valist (instance=<optimized out>, signal_id=<optimized out>, detail=<optimized out>, var_args=var_args@entry=0x7fff65739928) at ../gobject/gsignal.c:3397
#24 0x00007fffee9156e2 in g_signal_emit (instance=<optimized out>, signal_id=<optimized out>, detail=<optimized out>) at ../gobject/gsignal.c:3453
#25 0x00007fff65afd5de in gst_app_sink_render_common (psink=0x55555955a290, data=0x7ffedc1da850, is_list=0) at gstappsink.c:797
#26 0x00007fff65afd6d5 in gst_app_sink_render (psink=0x55555955a290, buffer=0x7ffedc1da850) at gstappsink.c:817
#27 0x00007fffec83a280 in gst_base_sink_render_object (basesink=0x55555955a290, pad=0x555558484820, obj_type=2 '\002', obj=0x7ffedc1da850) at gstbasesink.c:3012
#28 0x00007fffec83b50f in gst_base_sink_queue_object_unlocked (basesink=0x55555955a290, pad=0x555558484820, obj_type=2 '\002', obj=0x7ffedc1da850, prerollable=1) at gstbasesink.c:3293
#29 0x00007fffec83c608 in gst_base_sink_chain_unlocked (basesink=0x55555955a290, pad=0x555558484820, obj_type=2 '\002', obj=0x7ffedc1da850) at gstbasesink.c:3675
#30 0x00007fffec83c79c in gst_base_sink_chain_main (basesink=0x55555955a290, pad=0x555558484820, obj_type=2 '\002', obj=0x7ffedc1da850) at gstbasesink.c:3713
#31 0x00007fffec83c90c in gst_base_sink_chain (pad=0x555558484820, buf=0x7ffedc1da850) at gstbasesink.c:3742
#32 0x00007fffec574a9f in gst_pad_push (pad=0x5555584846a0, buffer=0x7ffedc1da850) at gstpad.c:4715
#33 0x00007fff69e420a9 in gst_queue_push_one (queue=0x555558aa3540) at gstqueue.c:1156
#34 0x00007fff69e42b45 in gst_queue_loop (pad=0x5555584846a0) at gstqueue.c:1264
#35 0x00007fffec5a4c28 in gst_task_func (task=0x55555e6c40d0) at gsttask.c:328
#36 0x00007fffec5a5d14 in default_func (tdata=0x555559490aa0, pool=0x555555908880) at gsttaskpool.c:70
#37 0x00007fffee43bf88 in g_thread_pool_thread_proxy (data=<optimized out>) at ../glib/gthreadpool.c:308
#38 0x00007fffee43b5ad in g_thread_proxy (data=0x555555a384a0) at ../glib/gthread.c:805
#39 0x00007ffff77e9ea5 in start_thread () from /lib64/libpthread.so.0
#40 0x00007ffff6e09b0d in clone () from /lib64/libc.so.6
Core files are available on ozgrav here:
/fred/oz016/gwdc_spiir_pipeline_codebase/scripts_n_things/build/ldavis/source/H1L1V1_HL_gw170817_bypass_long/core.3785
/fred/oz016/gwdc_spiir_pipeline_codebase/scripts_n_things/build/ldavis/source/H1L1V1_HLV_gw170817_bypass_long/core.16259
/fred/oz016/gwdc_spiir_pipeline_codebase/scripts_n_things/build/ldavis/source/H1L1V1_HLV_gw170817_snr_series_long/core.35694
/fred/oz016/gwdc_spiir_pipeline_codebase/scripts_n_things/build/ldavis/source/H1L1V1_HLV_gw170817_snr_series_long/core.10374
/fred/oz016/gwdc_spiir_pipeline_codebase/scripts_n_things/build/ldavis/source/H1L1V1_HLV_gw170817_snr_series_long_test2/core.10239
and CIT here:
/home/luke.davis/spiir-scripts/build/luke.davis/source/core.415264
This has not occured on MDC possibly because the input arguments for the pipeline differ in these ways, implying that it may still occur just a lot less frequently:
Segfault run ==> MDC run
--cohfar-assignfar-refresh-interval 200 ==> 3600
--finalsink-fapupdater-interval 1800 ==> 86400
--finalsink-snapshot-interval 1200 ==> 86400
Edited by Luke Davis