Rotate logs to avoid running out of disk space

.bashrc

@@ -6,6 +6,9 @@ if [ -f /etc/bashrc ]; then
     . /etc/bashrc
 fi
 
+# Create log directories.
+mkdir -p $HOME/.local/state/log
+
 # Add user site directory to the PATH. On Linux, this is usuall ~/.local/bin.
 export PATH="$(python3.11 -m site --user-base)/bin${PATH+:${PATH}}"
 

.config/logrotate.conf

+
+"~/.local/state/log/*.log" {
+    compress
+    dateext
+    missingok
+    rotate 4
+    weekly
+}

.gitignore

@@ -23,7 +23,6 @@ htmlcov
 appendonly.aof
 celerybeat-schedule
 *.pid
-*.log
 *.rdb
 *.sock
 

CHANGES.rst

@@ -45,6 +45,18 @@ Changelog
 -   Don't download p-astro file for SSM alert canvas, since they are not required
     by policy.
 
+-   Automatically rotate and compress log files to avoid running out of disk
+    space. Logs are rotated and compressed once per week and deleted after one
+    month.
+
+-   Logs are now stored in the directory ``~/.local/state/log``, which is the
+    per-user equivalent of ``/var/log`` (at least according to the conventions
+    of `systemd <https://www.freedesktop.org/software/systemd/man/latest/systemd.unit.html#Specifiers>`_).
+
+-   Rename the certificate renewal script from ``renew-cert.sh`` to ``cron.sh``
+    to reflect the fact that it is now responsible for additional maintenance
+    tasks including log rotation.
+
 2.5.2 "Cactus cat" (2024-11-18)
 -------------------------------
 

renew-cert.sh → cron.sh

 #!/usr/bin/bash
-# Renew the robot certificate from the Kerberos keytab.
+# Perform hourly maintenance activities.
+
+# Stop on error.
 set -e
 
 # This script is designed to be run by cron.
@@ -11,6 +13,7 @@ kinit_keytab() {
     kinit "${principal}" -k -t "$1"
 }
 
+# Renew GraceDB credentials.
 X509_USER_CERT="$HOME/.globus/usercert.pem"
 X509_USER_KEY="$HOME/.globus/userkey.pem"
 kinit_keytab "${HOME}/.globus/krb5.keytab"
@@ -19,5 +22,9 @@ GRID_PROXY_PATH="$(ecp-cert-info -path)"
 cp "${GRID_PROXY_PATH}" "${X509_USER_CERT}"
 cp "${GRID_PROXY_PATH}" "${X509_USER_KEY}"
 
+# Renew CVMFS credentials.
 kinit_keytab "${HOME}/read-cvmfs.keytab"
 chronic htgettoken -v -a vault.ligo.org -i igwn -r read-cvmfs-${USER} --scopes=read:/virgo --credkey=read-cvmfs-${USER}/robot/${USER}.ligo.caltech.edu --nooidc
+
+# Rotate log files.
+logrotate --state ~/.local/state/logrotate.status ~/.config/logrotate.conf

doc/configuration.rst

@@ -114,3 +114,16 @@ systems)::
 
 If you have to make any changes to your Redis configuration, be sure to restart
 the Redis daemon.
+
+Cron
+----
+
+For deployments of GWCelery at
+`LIGO Data Grid computing sites <https://computing.docs.ligo.org/guide/computing-centres/ldg/>`_,
+it is recommended that you configure :manpage:`cron <cron(8)>` to call the
+script ``cron.sh`` once per hour by adding the following to your
+:manpage:`crontab <crontab(1)>`::
+
+    @hourly $HOME/cron.sh
+
+This script automatically renews credentials and rotates log files.

gwcelery/data/gwcelery.sub

@@ -3,7 +3,7 @@ accounting_group_user = cody.messick
 universe = local
 getenv = true
 executable = /usr/bin/env
-log = gwcelery-condor.log
+log = .local/state/log/gwcelery-condor.log
 on_exit_remove = false
 request_disk = 7GB
 JobBatchName = gwcelery
@@ -12,46 +12,46 @@ JobBatchName = gwcelery
 job_max_vacate_time = 20
 kill_sig = SIGKILL
 
-arguments = "gwcelery beat -f gwcelery-beat.log"
+arguments = "gwcelery beat -f .local/state/log/gwcelery-beat.log"
 description = gwcelery-beat
 queue
 
-arguments = "gwcelery flask -l info -f gwcelery-flask.log run --with-threads --host 127.0.0.1"
+arguments = "gwcelery flask -l info -f .local/state/log/gwcelery-flask.log run --with-threads --host 127.0.0.1"
 description = gwcelery-flask
 queue
 
-arguments = "gwcelery flower --address=127.0.0.1 --log-file-prefix=gwcelery-flower.log"
+arguments = "gwcelery flower --address=127.0.0.1 --log-file-prefix=.local/state/log/gwcelery-flower.log"
 description = gwcelery-flower
 queue
 
 # FIXME: The GraceDB tasks are not very computationally intensive, but take a
 # very long time to execute. Manually bump up the concurrency well past the
 # number of hardware threads until the GraceDB API throughput is improved.
-arguments = "gwcelery worker -l info -n gwcelery-worker@%h -f %n.log -Q celery --igwn-alert --email --concurrency 32 --max-memory-per-child 2097152"
+arguments = "gwcelery worker -l info -n gwcelery-worker@%h -f .local/state/log/%n.log -Q celery --igwn-alert --email --concurrency 32 --max-memory-per-child 2097152"
 description = gwcelery-worker
 queue
 
-arguments = "gwcelery worker -l info -n gwcelery-exttrig-worker@%h -f %n.log -Q exttrig -c 1 --prefetch-multiplier 1"
+arguments = "gwcelery worker -l info -n gwcelery-exttrig-worker@%h -f .local/state/log/%n.log -Q exttrig -c 1 --prefetch-multiplier 1"
 description = gwcelery-exttrig-worker
 queue
 
-arguments = "gwcelery worker -l info -n gwcelery-superevent-worker@%h -f %n.log -Q superevent -c 1 --prefetch-multiplier 1"
+arguments = "gwcelery worker -l info -n gwcelery-superevent-worker@%h -f .local/state/log/%n.log -Q superevent -c 1 --prefetch-multiplier 1"
 description = gwcelery-superevent-worker
 queue
 
-arguments = "gwcelery worker -l info -n gwcelery-voevent-worker@%h -f %n.log -Q voevent -P solo"
+arguments = "gwcelery worker -l info -n gwcelery-voevent-worker@%h -f .local/state/log/%n.log -Q voevent -P solo"
 description = gwcelery-voevent-worker
 queue
 
-arguments = "gwcelery worker -l info -n gwcelery-kafka-worker@%h -f %n.log -Q kafka -P solo"
+arguments = "gwcelery worker -l info -n gwcelery-kafka-worker@%h -f .local/state/log/%n.log -Q kafka -P solo"
 description = gwcelery-kafka-worker
 queue
 
-arguments = "gwcelery worker -l info -n gwcelery-em-bright-worker@%h -f %n.log -Q em-bright -c 2 --prefetch-multiplier 1"
+arguments = "gwcelery worker -l info -n gwcelery-em-bright-worker@%h -f .local/state/log/%n.log -Q em-bright -c 2 --prefetch-multiplier 1"
 description = gwcelery-em-bright-worker
 queue
 
-arguments = "gwcelery worker -l info -n gwcelery-highmem-worker@%h -f %n.log -Q highmem -c 2 --prefetch-multiplier 1"
+arguments = "gwcelery worker -l info -n gwcelery-highmem-worker@%h -f .local/state/log/%n.log -Q highmem -c 2 --prefetch-multiplier 1"
 description = gwcelery-highmem-worker
 queue
 
@@ -65,10 +65,10 @@ universe = vanilla
 executable = /bin/env
 
 
-arguments = "gwcelery-condor-submit-helper gwcelery worker -l info -n gwcelery-multiprocessing-worker@%h -f %n.log -Q multiprocessing -c 1 --prefetch-multiplier 1"
+arguments = "gwcelery-condor-submit-helper gwcelery worker -l info -n gwcelery-multiprocessing-worker@%h -f .local/state/log/%n.log -Q multiprocessing -c 1 --prefetch-multiplier 1"
 description = gwcelery-multiprocessing-worker
 queue
 
-arguments = "--unset OMP_NUM_THREADS gwcelery-condor-submit-helper gwcelery worker -l info -n gwcelery-openmp-worker-$(Process)@%h -f %n.log -Q openmp -c 1 --prefetch-multiplier 1"
+arguments = "--unset OMP_NUM_THREADS gwcelery-condor-submit-helper gwcelery worker -l info -n gwcelery-openmp-worker-$(Process)@%h -f .local/state/log/%n.log -Q openmp -c 1 --prefetch-multiplier 1"
 description = gwcelery-openmp-worker-$(Process)
 queue 15