diff --git a/gracedb/alert.py b/gracedb/alert.py
index 8b2cfb6b8dd61d06c7defc96fbfb85f7dd027964..f61d4353aec3a617aa480fb1aa094756c6aec462 100644
--- a/gracedb/alert.py
+++ b/gracedb/alert.py
@@ -14,6 +14,10 @@ from userprofile.models import Trigger, AnalysisType
import glue.ligolw.utils
import ligo.lvalert.utils
+import logging
+
+log = logging.getLogger('gracedb.alert')
+
def issueAlert(event, location, temp_data_loc):
issueXMPPAlert(event, location, temp_data_loc)
issueEmailAlert(event, location)
@@ -116,9 +120,13 @@ def issueXMPPAlert(event, location, temp_data_loc, alert_type="new", description
nodename = "%s_%s"% (event.group.name, event.get_analysisType_display())
nodename = nodename.lower()
+ log.debug('issueXMPPAlert: %s %s' % (event.graceid(), nodename))
+
if nodename not in settings.XMPP_ALERT_CHANNELS:
+ log.debug("issueXMPPAlert: did not send alert")
return
+ log.debug("issueXMPPAlert: attempting to send alert")
env = {}
env["PYTHONPATH"] = ":".join(sys.path)
@@ -146,12 +154,18 @@ def issueXMPPAlert(event, location, temp_data_loc, alert_type="new", description
glue.ligolw.utils.write_fileobj(xmldoc, buf)
msg = buf.getvalue()
+ log.debug("issueXMPPAlert: writing message %s" % msg)
+
p.stdin.write(msg)
p.stdin.close()
+ res = None
for i in range(1,10):
res = p.poll()
if res == None:
time.sleep(1)
else:
+ log.debug("issueXMPPAlert: return code %s" % res)
break
+ if res is None:
+ log.debug("issueXMPPAlert: failed to see child process terminate")
diff --git a/gracedb/api.py b/gracedb/api.py
index c5ba81c791c4ea43d09caa7e16d462527bdbea34..cb138f868da27b27ddabf01631719d238ba1967e 100644
--- a/gracedb/api.py
+++ b/gracedb/api.py
@@ -12,12 +12,16 @@ from gracedb.models import Event, Group, EventLog, Slot
from gracedb.views import create_label
from translator import handle_uploaded_data
+from alert import issueAlertForUpdate
+
import os
import urllib
import errno
import logging
import shutil
+from utils.vfile import VersionedFile
+
##################################################################
REST_FRAMEWORK_SETTINGS = getattr(settings, 'REST_FRAMEWORK', {})
@@ -341,7 +345,7 @@ class EventDetail(APIView):
# XXX handle duplicate file names.
f = request.FILES['eventFile']
uploadDestination = os.path.join(eventDir, "private", f.name)
- fdest = open(uploadDestination, 'w')
+ fdest = VersionedFile(uploadDestination, 'w')
# Save uploaded file into user private area.
#for chunk in f.chunks():
# fdest.write(chunk)
@@ -446,7 +450,11 @@ class EventLabel(APIView):
def put(self, request, graceid, label):
#return Response("Not Implemented", status=status.HTTP_501_NOT_IMPLEMENTED)
- create_label(graceid, label, request.ligouser)
+ try:
+ create_label(graceid, label, request.ligouser)
+ except ValueError, e:
+ return Response(e.message,
+ status=status.HTTP_400_BAD_REQUEST)
return Response("Created", status=status.HTTP_201_CREATED)
@@ -751,100 +759,37 @@ class Files(APIView):
except Event.DoesNotExist:
return HttpResponseNotFound("Event not found")
- # Construct the file path just as Brian does above.
- general = False
if filename.startswith("general/"):
- filename = filename[len("general/"):]
- general = True
-
- filepath = os.path.join(event.datadir(general), filename)
-
- if not os.path.exists(filepath):
- # Awesome. The thing does not exist. This is the first time a file
- # by this name is being uploaded.
- # Write the file as "filename,0".
- linkpath = filepath
- filepath += ',0'
- filename += ',0'
- fdest = open(filepath, 'w')
- # Check out line 392 in the client.
- # I think the key name for the file is 'upload'
- f = request.FILES['upload']
- for chunk in f.chunks():
- fdest.write(chunk)
- fdest.close()
-
- # Make a relative symlink.
- os.symlink(filename,linkpath)
+ # No writing to general/
+ return HttpResponseForbidden("cannot write to general directory")
- rv = {}
- rv['permalink'] = reverse("files", args=[graceid, filename], request=request)
- response = Response(rv, status=status.HTTP_201_CREATED)
+ filepath = os.path.join(event.datadir(), filename)
- elif os.path.islink(filepath):
- # Great. The thing is a symlink. We can do our version-y stuff now.
-
- # Read contents of directory. Establish the number of existing versions.
- # All we need is the bare filename (i.e., not the full path)
- filedir = event.datadir(general)
- lastVersion = 0
- for dirname, dirnames, filenames in os.walk(filedir):
- for fname in filenames:
- if fname.find(',') > 0:
- if fname.split(',')[0] == filename:
- lastVersion = max(lastVersion,int(fname.split(',')[1]))
-
- linkpath = filepath # Set the link path to the original file path.
- notOpenYet = True
- failedAttempts = 0
- while notOpenYet:
- # find the new filename
- newFilename = filename + ',%d' % (lastVersion+1)
- # update the file path according to the new filename.
- filepath = os.path.join(filedir,newFilename)
- try:
- # os.O_EXCL causes the open to fail if the file already exists.
- fd = os.open(filepath, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0644)
- fdest = os.fdopen(fd,"w")
- notOpenYet = False
- except OSError as e:
- if e.errno==errno.EACCES:
- return HttpResponseForbidden("No permission to write to event directory.")
- else:
- # Note: could also check whether e.errno==errno.EEXIST
- ++failedAttempts
- if failedAttempts >= MAX_FAILED_OPEN_ATTEMPTS:
- return HttpResponseServerError("Cannot open file for writing: %s" % e)
- # Under race conditions, increment lastVersion.
- if e.errno==errno.EACCES:
- ++lastVersion
-
- # Still with me? Then write the file.
+ try:
+ # Open / Write the file.
+ fdest = VersionedFile(filepath, 'w')
f = request.FILES['upload']
for chunk in f.chunks():
fdest.write(chunk)
fdest.close()
- # Move the symlink, using os.rename to avoid race conditions.
- tmplink = os.path.join(filedir,'tmplink')
- os.symlink(newFilename,tmplink)
- os.rename(tmplink,linkpath)
-
rv = {}
- rv['permalink'] = reverse("files", args=[graceid, newFilename], request=request)
+ # XXX this seems wobbly.
+ longname = fdest.name
+ shortname = longname[longname.rfind(filename):]
+ rv['permalink'] = reverse(
+ "files", args=[graceid, shortname], request=request)
response = Response(rv, status=status.HTTP_201_CREATED)
+ except Exception, e:
+ # XXX This needs some thought.
+ response = Response(str(e), status=status.HTTP_400_BAD_REQUEST)
- elif os.path.isfile(filepath):
- # The thing is a file and not a symlink. We will not allow a put request to the file
- # resource (for now, anyway).
- response = HttpResponseForbidden("%s is a file. Versioning is not supported with legacy data. Please change your filename to avoid clobbering." % filename)
- elif not filename:
- # Not good. There's nothing we can do without a filename.
- response = HttpResponseBadRequest("Must have a filename for upload.")
- elif os.path.isdir(filepath):
- response = HttpResponseForbidden("%s is a directory" % filename)
- else:
- response = HttpResponseServerError("Should not happen.")
+ try:
+ description = "UPLOAD: {0}".format(filename)
+ issueAlertForUpdate(event, description, doxmpp=True)
+ except:
+ # XXX something should be done here.
+ pass
return response
diff --git a/gracedb/serialize/utils.py b/gracedb/serialize.py
similarity index 99%
rename from gracedb/serialize/utils.py
rename to gracedb/serialize.py
index 95e14ffd710b116ecace8e055cee9d058c35523c..f6f532128295391bf54394927e8c5575bc17a93b 100644
--- a/gracedb/serialize/utils.py
+++ b/gracedb/serialize.py
@@ -9,6 +9,8 @@ from glue.ligolw import ligolw
from glue.ligolw import table
from glue.ligolw import lsctables
+from utils.vfile import VersionedFile
+
##############################################################################
#
# useful variables
@@ -87,11 +89,11 @@ def write_output_files(root_dir, xmldoc, log_content, \
write the xml-format coinc tables and log file
"""
- f = open(root_dir+'/'+xml_fname,'w')
+ f = VersionedFile(root_dir+'/'+xml_fname,'w')
xmldoc.write(f)
f.close()
- f = open(root_dir+'/'+log_fname,'w')
+ f = VersionedFile(root_dir+'/'+log_fname,'w')
f.write(log_content)
f.close()
diff --git a/gracedb/serialize/__init__.py b/gracedb/serialize/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/gracedb/settings b/gracedb/settings
new file mode 120000
index 0000000000000000000000000000000000000000..1aea006e66684dac4f99a8b3134ae5e473aaae3b
--- /dev/null
+++ b/gracedb/settings
@@ -0,0 +1 @@
+/home/bmoe/gracedb/settings
\ No newline at end of file
diff --git a/gracedb/translator.py b/gracedb/translator.py
index 8f998ac05d4b7a97a7bec0df7c9c0893da49d4d4..80a57c1bf937884d65ee4fde451ebd7738e4aa2c 100644
--- a/gracedb/translator.py
+++ b/gracedb/translator.py
@@ -6,12 +6,13 @@ from models import EventLog
import glue
import glue.ligolw.utils
-from gracedb.serialize.utils import populate_inspiral_tables, \
+from gracedb.serialize import populate_inspiral_tables, \
populate_omega_tables, \
write_output_files
from VOEventLib.Vutil import parse, getWhereWhen
from utils import isoToGps
+from utils.vfile import VersionedFile
def handle_uploaded_data(event, datafilename,
log_filename='event.log',
@@ -370,7 +371,7 @@ class Translator(object):
def writeLogfile(self, path):
data = self.logData()
if data:
- f = open(path, 'w')
+ f = VersionedFile(path, 'w')
f.write(data)
f.close()
return True
diff --git a/gracedb/urls_rest.py b/gracedb/urls_rest.py
index f0e630872afd2d65894e84f282082ad7f996c84b..d915fe9e4996cdfb41625b04da04a7d4318aa751 100644
--- a/gracedb/urls_rest.py
+++ b/gracedb/urls_rest.py
@@ -10,7 +10,8 @@ from gracedb.api import Files, FileMeta
from gracedb.api import EventNeighbors, EventLabel
urlpatterns = patterns('gracedb.api',
- url (r'^$', GracedbRoot.as_view(), name="api-root"),
+ url (r'^/?$', GracedbRoot.as_view(), name="api-root"),
+
# Event Resources
# events/[{graceid}[/{version}]]
diff --git a/gracedb/views.py b/gracedb/views.py
index 793a2bd994125ab960ec247adbc959fa6fd5eb3f..7568abb00b160cc9caed0de8bd1c0bc33059d2b8 100644
--- a/gracedb/views.py
+++ b/gracedb/views.py
@@ -17,6 +17,8 @@ from translator import handle_uploaded_data
import urllib
+from utils.vfile import VersionedFile
+
import os
import re
from django.core.mail import mail_admins
@@ -222,7 +224,7 @@ def _createEventFromForm(request, form):
os.chmod( os.path.join(eventDir,"general"), 041777 )
f = request.FILES['eventFile']
uploadDestination = os.path.join(eventDir, "private", f.name)
- fdest = open(uploadDestination, 'w')
+ fdest = VersionedFile(uploadDestination, 'w')
# Save uploaded file into user private area.
for chunk in f.chunks():
fdest.write(chunk)
@@ -261,7 +263,7 @@ def _createEventFromForm(request, form):
def _saveUploadedFile(event, uploadedFile):
# XXX Hardcoding.
fname = os.path.join(GRACEDB_DATA_DIR, event.graceid(), "private", uploadedFile.name)
- f = open(fname, "w")
+ f = VersionedFile(fname, "w")
for chunk in uploadedFile.chunks():
f.write(chunk)
f.close()
@@ -324,6 +326,7 @@ def upload(request):
msg = "ERROR: Event '%s' does not exist" % graceid
else:
#event issuer comment
+ # XXX Note: filename or comment oughta have a version
log = EventLog(event=event,
issuer=request.ligouser,
filename=uploadedfile.name,
@@ -337,9 +340,8 @@ def upload(request):
# XXX
# Badnesses:
# Same hardcoded path in multiple places.
- # What if we're clobbering an existing file?
fname = os.path.join(GRACEDB_DATA_DIR, event.graceid(), "private", uploadedfile.name)
- f = open(fname, "w")
+ f = VersionedFile(fname, 'w')
for chunk in uploadedfile.chunks():
f.write(chunk)
f.close()
diff --git a/utils/vfile.py b/utils/vfile.py
new file mode 100644
index 0000000000000000000000000000000000000000..b733e9ce1a1074f79984f79d44a4c069f45862cb
--- /dev/null
+++ b/utils/vfile.py
@@ -0,0 +1,165 @@
+
+import os
+import tempfile
+import logging
+import errno
+import shutil
+
+
+class VersionedFile(file):
+ """Open a versioned file.
+
+ VersionedFile(name [, mode [, version [, OTHER FILE ARGS]]]) -> file object
+ """
+ def __init__(self, name, *args, **kwargs):
+
+ self.log = logging.getLogger('VersionedFile')
+
+ if ',' in name:
+ # XXX too strict.
+ raise IOError("versioned file name cannot contain a ','")
+
+ if len(args):
+ mode = args[0]
+ else:
+ mode = kwargs.get('mode', "")
+
+ if len(args) > 1:
+ version = args[1]
+ # Remove from arglist to prep for passing to parent class.
+ args = args[:2]
+ else:
+ version = None
+ if 'version' in kwargs:
+ version = kwargs['version']
+ # Remove from kwargs to prep for passing to parent class.
+ del kwargs['version']
+
+ self.writing = ('w' in mode) or ('a' in mode) or ('+' in mode)
+
+ absname = os.path.abspath(name)
+ self.absname = absname
+ fullname = name
+ self.fullname = fullname
+
+ # If we are merely reading, just open the file as requested.
+ # Easy Peasy.
+
+ if not self.writing:
+ actual_name = self._name_for_version(version)
+ self.log.debug(
+ "opening file '{0}' with mode '{1}'"
+ .format(actual_name, mode))
+ file.__init__(self, actual_name, *args, **kwargs)
+ return
+
+ # Otherwise...
+
+ # Specific version requsted? For writing?
+ # I don't think so.
+ if version is not None:
+ # XXX IOError appropriate here?
+ e = IOError(
+ "Cannot write to a specific version of a VersionedFile")
+ e.errno = errno.EINVAL
+ e.filename = fullname
+ raise e
+
+ # XXX No appending. Could conceivably copy the latest
+ # (or specified) version of the file, then open w/append.
+ if 'a' in mode:
+ # XXX IOError appropriate here?
+ e = IOError("Cannot (currently) append to a VersionedFile")
+ e.errno = errno.EINVAL
+ e.filename = fullname
+ raise e
+
+ version = max([-1] + self.known_versions()) + 1
+
+ if os.path.exists(fullname) and not os.path.islink(fullname):
+ # It is not versioned. Versionize it.
+ if version != 0:
+ raise IOError("VersionedFile symlink inconsistency.")
+ # XXX risky. race condition.
+ #os.rename(fullname, self._name_for_version(version))
+ shutil.move(fullname, self._name_for_version(version))
+ self._repoint_symlink()
+ version += 1
+
+ # Open file, which must not exist.
+
+ failedAttempts = 0
+
+ while failedAttempts < 5:
+ actual_name = self._name_for_version(version)
+ self.log.debug(
+ "opening file '{0}' with mode '{1}'"
+ .format(actual_name, mode))
+ try:
+ # os.O_EXCL causes the open to fail if the file already exists.
+ fd = os.open(actual_name,
+ os.O_WRONLY | os.O_CREAT | os.O_EXCL,
+ 0644)
+ # re-open
+ file.__init__(self, actual_name, *args, **kwargs)
+ # lose fd we used to ensure file creation.
+ os.close(fd)
+ break
+ except OSError, e:
+ if e.error != errno.EEXIST:
+ raise
+ version += 1
+ failedAttempts += 1
+
+ if failedAttempts >= 5:
+ raise IOError("Too many attempts to open file")
+
+ def _name_for_version(self, version):
+ if version is None:
+ return self.fullname
+ return "{0},{1}".format(self.fullname, version)
+
+ def _repoint_symlink(self):
+ # re-point symlink to latest version
+ last_version = max(self.known_versions())
+ # XXX Maybe ought to check that we are removing a symlink.
+ try:
+ # XXX Another race condition. File will not exist for a very brief time.
+ os.unlink(self.fullname)
+ except:
+ # Do not care if file does not exist.
+ pass
+ os.symlink(self._name_for_version(last_version), self.fullname)
+ return
+
+# XXX This fails when renaming/mv-ing across devices.
+ # XXX assumption: tempfile name will remain unique after closing
+ tmp = tempfile.NamedTemporaryFile(delete=True)
+ tmpname = tmp.name
+ tmp.close()
+ os.symlink(self._name_for_version(last_version), tmpname)
+ #os.rename(tmp.name, self.fullname)
+ shutil.move(tmp.name, self.fullname)
+
+ def known_versions(self):
+ path = self.absname
+ d = os.path.dirname(path) or '.'
+ name = os.path.basename(path)
+ # XXX what if stuff after ',' is not an int.
+ return [int(f.split(',')[1])
+ for f in os.listdir(d) if f.startswith(name + ',')]
+
+ def close(self):
+ if self.writing:
+ # no need to update symlink if we were only reading.
+ # can cause trouble if we were reading a non-versioned
+ # file -- trying to discover the lastest version fails
+ # painfully. (max(known_versions()) => max([]))
+ self._repoint_symlink()
+ if not self.closed:
+ file.close(self)
+
+ def __del__(self):
+ # XXX file does not have a __del__ method. Should we?
+ if not self.closed:
+ self.close()