diff --git a/gstlal-inspiral/bin/gstlal_inspiral_pipe b/gstlal-inspiral/bin/gstlal_inspiral_pipe index 49c3f90c2870f63934650dbf73270a1cf93f6421..043a8d9e9d62131b0e8cf8505d64767cdf4ef900 100755 --- a/gstlal-inspiral/bin/gstlal_inspiral_pipe +++ b/gstlal-inspiral/bin/gstlal_inspiral_pipe @@ -593,7 +593,7 @@ def rank_and_merge(dag, createPriorDistStatsJob, calcRankPDFsJob, calcRankPDFsWi db = inspiral_pipe.group_T050017_filename_from_T050017_files([CacheEntry.from_T050017("file://localhost%s" % os.path.abspath(filename)) for filename in inputs], '.sqlite') db = os.path.join(subdir_path([toSqliteJob.output_path, CacheEntry.from_T050017(db).description[:4]]), db) sqlitenode = inspiral_pipe.generic_node(toSqliteJob, dag, parent_nodes = merge_nodes, - opts = {"replace":"", "tmp-space":inspiral_pipe.condor_scratch_space()}, + opts = {"replace":"", "tmp-space":inspiral_pipe.condor_scratch_space(), "ilwdchar-compat":""}, input_cache_files = {"input-cache":inputs}, output_files = {"database":db}, input_cache_file_name = os.path.basename(db).replace('.sqlite','.cache') @@ -620,7 +620,7 @@ def rank_and_merge(dag, createPriorDistStatsJob, calcRankPDFsJob, calcRankPDFsWi injdb = inspiral_pipe.group_T050017_filename_from_T050017_files([CacheEntry.from_T050017("file://localhost%s" % os.path.abspath(filename)) for filename in inputs], '.sqlite') injdb = os.path.join(subdir_path([toSqliteJob.output_path, CacheEntry.from_T050017(injdb).description[:4]]), injdb) sqlitenode = inspiral_pipe.generic_node(toSqliteJob, dag, parent_nodes = merge_nodes, - opts = {"replace":"", "tmp-space":inspiral_pipe.condor_scratch_space()}, + opts = {"replace":"", "tmp-space":inspiral_pipe.condor_scratch_space(), "ilwdchar-compat":""}, input_cache_files = {"input-cache":inputs}, output_files = {"database":injdb}, input_cache_file_name = os.path.basename(injdb).replace('.sqlite','.cache') @@ -868,7 +868,7 @@ def finalize_runs(dag, lalappsRunSqliteJob, toXMLJob, ligolwInspinjFindJob, toSq # merge sqlitenode = inspiral_pipe.generic_node(toSqliteJob, dag, parent_nodes = chunk_nodes + ligolw_add_nodes, - opts = {"replace":"", "tmp-space":inspiral_pipe.condor_scratch_space()}, + opts = {"replace":"", "tmp-space":inspiral_pipe.condor_scratch_space(), "ilwdchar-compat":""}, input_files = {"": (vetoes + [options.frame_segments_file, injections])}, input_cache_files = {"input-cache": [node.input_files[""] for node in chunk_nodes]}, output_files = {"database":injdb}, @@ -883,7 +883,7 @@ def finalize_runs(dag, lalappsRunSqliteJob, toXMLJob, ligolwInspinjFindJob, toSq clusternode = inspiral_pipe.generic_node(toXMLJob, dag, parent_nodes = [clusternode], - opts = {"tmp-space":inspiral_pipe.condor_scratch_space()}, + opts = {"tmp-space":inspiral_pipe.condor_scratch_space(), "ilwdchar-compat":""}, output_files = {"extract":injxml}, input_files = {"database":injdb} ) @@ -894,7 +894,7 @@ def finalize_runs(dag, lalappsRunSqliteJob, toXMLJob, ligolwInspinjFindJob, toSq ) sqlitenode = inspiral_pipe.generic_node(toSqliteNoCacheJob, dag, parent_nodes = [inspinjnode], - opts = {"replace":"", "tmp-space":inspiral_pipe.condor_scratch_space()}, + opts = {"replace":"", "tmp-space":inspiral_pipe.condor_scratch_space(), "ilwdchar-compat":""}, output_files = {"database":injdb}, input_files = {"":xml_input} ) diff --git a/gstlal-inspiral/bin/gstlal_ll_inspiral_calculate_range b/gstlal-inspiral/bin/gstlal_ll_inspiral_calculate_range index 27c15a4cd074e0f6d6c0f72acf011bc1cb8598ad..6448b229dbd0abc3459159927f475204c043dd6d 100755 --- a/gstlal-inspiral/bin/gstlal_ll_inspiral_calculate_range +++ b/gstlal-inspiral/bin/gstlal_ll_inspiral_calculate_range @@ -257,7 +257,7 @@ while true; do # Try to generate the sqlite database, but if something goes # wrong then delete the database and try again. This should fix # issues with the database locking - ligolw_sqlite --verbose --database ${LIGOLW_SQLITE} --replace ${TMP_LIGOLW_XML_GZ} || { rm ${LIGOLW_SQLITE} && ligolw_sqlite --verbose --database ${LIGOLW_SQLITE} ${TMP_LIGOLW_XML_GZ}; } + ligolw_sqlite --ilwdchar-compat --verbose --database ${LIGOLW_SQLITE} --replace ${TMP_LIGOLW_XML_GZ} || { rm ${LIGOLW_SQLITE} && ligolw_sqlite --ilwdchar-compat --verbose --database ${LIGOLW_SQLITE} ${TMP_LIGOLW_XML_GZ}; } gstlal_inspiral_plot_sensitivity --data-segments-name=statevectorsegments --bin-by-chirp-mass --verbose ${LIGOLW_SQLITE} echo plotting... diff --git a/gstlal-inspiral/bin/gstlal_ll_inspiral_daily_page b/gstlal-inspiral/bin/gstlal_ll_inspiral_daily_page index 3f9b5f91ed513c0012efcbaacf18ec3357e096d0..2b99c5a35be1e9d0338997765ec3b26be51aafca 100755 --- a/gstlal-inspiral/bin/gstlal_ll_inspiral_daily_page +++ b/gstlal-inspiral/bin/gstlal_ll_inspiral_daily_page @@ -53,7 +53,7 @@ def process_mass_bin(args): files = sorted([os.path.join(os.path.join(options.directory, d), xml) for xml in os.listdir(os.path.join(options.directory, d)) if pattern.match(xml) and "~" not in xml]) for f in files: try: - subprocess.check_call(["ligolw_sqlite", "--tmp-space", os.environ["TMPDIR"], "--database", "%s" % db, "%s" % f]) + subprocess.check_call(["ligolw_sqlite", "--ilwdchar-compat", "--tmp-space", os.environ["TMPDIR"], "--database", "%s" % db, "%s" % f]) except: print >> sys.stderr, "couldn't process %s" % f continue @@ -131,12 +131,12 @@ if __name__ == '__main__': # Parallel process the data reduction args = ([massbin, result_dirs, n, d, options, "ALL_LLOID", 0, cluster_file] for massbin in massbins) # Merge the files of this directory - subprocess.check_call(["ligolw_sqlite", "--tmp-space", os.environ["TMPDIR"], "--replace", "--verbose", "--database", "%s" % noninjdball] + list(pool.map(process_mass_bin, args))) + subprocess.check_call(["ligolw_sqlite", "--ilwdchar-compat", "--tmp-space", os.environ["TMPDIR"], "--replace", "--verbose", "--database", "%s" % noninjdball] + list(pool.map(process_mass_bin, args))) noninj_files_to_merge.append(noninjdball) for injection_file in inj_file_bins: args = ([massbin, result_dirs, n, d, options, injtag(injection_file), 1, cluster_file] for massbin in inj_file_bins[injection_file]) - subprocess.check_call(["ligolw_sqlite", "--tmp-space", os.environ["TMPDIR"], "--replace", "--verbose", "--database", "%s" % injdball[injection_file]] + list(pool.map(process_mass_bin, args))) + subprocess.check_call(["ligolw_sqlite", "--ilwdchar-compat", "--tmp-space", os.environ["TMPDIR"], "--replace", "--verbose", "--database", "%s" % injdball[injection_file]] + list(pool.map(process_mass_bin, args))) inj_files_to_merge.setdefault(injection_file,[]).append(injdball[injection_file]) # @@ -159,7 +159,7 @@ if __name__ == '__main__': progressbar = ProgressBar("Merge noninjection files", len(noninj_files_to_merge)) for f in noninj_files_to_merge: - subprocess.check_call(["ligolw_sqlite", "--tmp-space", os.environ["TMPDIR"], "--verbose", "--database", "%s" % noninjdb, "%s" % f]) + subprocess.check_call(["ligolw_sqlite", "--ilwdchar-compat", "--tmp-space", os.environ["TMPDIR"], "--verbose", "--database", "%s" % noninjdb, "%s" % f]) # Do not cluster! the online analysis doesn't do a global clustering stage!! try: subprocess.check_call(["lalapps_run_sqlite", "--verbose", "--sql-file", cluster_file, "%s" % noninjdb]) @@ -171,7 +171,7 @@ if __name__ == '__main__': for injection_file in inj_file_bins: progressbar = ProgressBar("Merge injection files", len(inj_files_to_merge[injection_file])) for f in inj_files_to_merge[injection_file]: - subprocess.check_call(["ligolw_sqlite", "--tmp-space", os.environ["TMPDIR"], "--verbose", "--database", "%s" % injdb[injection_file], "%s" % f]) + subprocess.check_call(["ligolw_sqlite", "--ilwdchar-compat", "--tmp-space", os.environ["TMPDIR"], "--verbose", "--database", "%s" % injdb[injection_file], "%s" % f]) # Do not cluster! the online analysis doesn't do a global clustering stage!! try: subprocess.check_call(["lalapps_run_sqlite", "--verbose", "--sql-file", cluster_file, "%s" % injdb[injection_file]]) @@ -182,13 +182,13 @@ if __name__ == '__main__': # Find injections progressbar = ProgressBar("Find injections", 4) - subprocess.check_call(["ligolw_sqlite", "--tmp-space", os.environ["TMPDIR"], "--verbose", "--database", "%s" % injdb[injection_file], "%s" % injection_file]) + subprocess.check_call(["ligolw_sqlite", "--ilwdchar-compat", "--tmp-space", os.environ["TMPDIR"], "--verbose", "--database", "%s" % injdb[injection_file], "%s" % injection_file]) progressbar.increment() - subprocess.check_call(["ligolw_sqlite", "--tmp-space", os.environ["TMPDIR"], "--verbose", "--database", "%s" % injdb[injection_file], "--extract", "%s.xml" % injdb[injection_file]]) + subprocess.check_call(["ligolw_sqlite", "--ilwdchar-compat", "--tmp-space", os.environ["TMPDIR"], "--verbose", "--database", "%s" % injdb[injection_file], "--extract", "%s.xml" % injdb[injection_file]]) progressbar.increment() subprocess.check_call(["lalapps_inspinjfind", "--verbose", "%s.xml" % injdb[injection_file]]) progressbar.increment() - subprocess.check_call(["ligolw_sqlite", "--tmp-space", os.environ["TMPDIR"], "--verbose", "--database", "%s" % injdb[injection_file], "--replace", "%s.xml" % injdb[injection_file]]) + subprocess.check_call(["ligolw_sqlite", "--ilwdchar-compat", "--tmp-space", os.environ["TMPDIR"], "--verbose", "--database", "%s" % injdb[injection_file], "--replace", "%s.xml" % injdb[injection_file]]) progressbar.increment() # diff --git a/gstlal-inspiral/tests/Makefile.offline_tutorial_test b/gstlal-inspiral/tests/Makefile.offline_tutorial_test index 988aa6d256fb66feb8f53ed2e9583318f653b7d9..d6dd41eec5996b03ea564cd4dea091b398711c3d 100644 --- a/gstlal-inspiral/tests/Makefile.offline_tutorial_test +++ b/gstlal-inspiral/tests/Makefile.offline_tutorial_test @@ -545,6 +545,7 @@ lalapps_run_sqlite_0001: gstlal_inspiral_calc_likelihood_0001 H1L1-ALL_LLOID-1000000000-2048.sqlite: lalapps_run_sqlite_0001 # ligolw_sqlite_from_xml_0004 $(TIME) ligolw_sqlite \ + --ilwdchar-compat \ --database H1L1-ALL_LLOID-1000000000-2048.sqlite \ --replace \ --tmp-space $(TMPDIR) \ diff --git a/gstlal-ugly/examples/configurations/NINJA2/post/copyCombineTrigs.sh b/gstlal-ugly/examples/configurations/NINJA2/post/copyCombineTrigs.sh index 8c1f169e745ab5b856a174d56feb2b2f59956f18..181d92b22a47ef6f739b7fc971d5cc3de8e2b733 100644 --- a/gstlal-ugly/examples/configurations/NINJA2/post/copyCombineTrigs.sh +++ b/gstlal-ugly/examples/configurations/NINJA2/post/copyCombineTrigs.sh @@ -1,5 +1,5 @@ for i in `ls ../week*/*ALL*.sqlite`;do - ligolw_sqlite -v --database ${i} --extract `basename ${i%%.s*}`.xml.gz + ligolw_sqlite --ilwdchar-compat -v --database ${i} --extract `basename ${i%%.s*}`.xml.gz done ligolw_add --ilwdchar-compat -v --output H1L1V1-ALL-COMBINED-871147552-876357464-LLOID.xml.gz `ls H1L1V1-ALL-8*.xml.gz` diff --git a/gstlal-ugly/examples/configurations/PBH/scripts/find_injections.sh b/gstlal-ugly/examples/configurations/PBH/scripts/find_injections.sh index 67d224537c651cd3af811770678854568b6695d0..2dbb42896150b89a3a7aee62b799d20334b6cd03 100755 --- a/gstlal-ugly/examples/configurations/PBH/scripts/find_injections.sh +++ b/gstlal-ugly/examples/configurations/PBH/scripts/find_injections.sh @@ -2,7 +2,7 @@ set -e for k in *.sqlite do -ligolw_sqlite --verbose --tmp-space /dev/shm --extract ${k}.xml --database ${k} +ligolw_sqlite --ilwdchar-compat --verbose --tmp-space /dev/shm --extract ${k}.xml --database ${k} ligolw_sicluster --verbose --cluster-window 1 ${k}.xml done @@ -10,10 +10,10 @@ for i in H1 H2 L1 do ligolw_add --ilwdchar-compat --verbose --output ${i}all_added.xml --verbose svd_*${i}_split_bank*.xml ../segments/injections.xml ligolw_sicluster --verbose --cluster-window 1 ${i}all_added.xml -ligolw_sqlite --verbose --database ${i}all.sqlite --tmp-space /dev/shm --replace ${i}all_added.xml +ligolw_sqlite --ilwdchar-compat --verbose --database ${i}all.sqlite --tmp-space /dev/shm --replace ${i}all_added.xml sqlite3 ${i}all.sqlite 'DROP TABLE sim_inspiral' -ligolw_sqlite --verbose --database ${i}all.sqlite --tmp-space /dev/shm --extract ${i}all_added.xml +ligolw_sqlite --ilwdchar-compat --verbose --database ${i}all.sqlite --tmp-space /dev/shm --extract ${i}all_added.xml ligolw_add --ilwdchar-compat --verbose --output ${i}all.xml ../segments/injections.xml ${i}all_added.xml lalapps_inspinjfind ${i}all.xml -ligolw_sqlite --verbose --database ${i}all.sqlite --tmp-space /dev/shm --replace ${i}all.xml +ligolw_sqlite --ilwdchar-compat --verbose --database ${i}all.sqlite --tmp-space /dev/shm --replace ${i}all.xml done diff --git a/gstlal-ugly/examples/gstlal_inspiral.ini b/gstlal-ugly/examples/gstlal_inspiral.ini index 998d4e1b1bb2b0326671156a10fe2e970b998cb6..eddba38968edc24de889c832a18f3643fccc2381 100644 --- a/gstlal-ugly/examples/gstlal_inspiral.ini +++ b/gstlal-ugly/examples/gstlal_inspiral.ini @@ -92,6 +92,7 @@ comment = $(macrocomment) match-algorithm = inspiral [ligolw_sqlite] +ilwdchar-compat = replace = preserve-ids = database = $(macrodatabase)