[PATCH 2 of 2 bfiles] httpstore: implement put to HTTP/WebDAV storage

[Geoff Crossland]

# HG changeset patch
# User Geoff Crossland <gcrossland at linguamatics.com>
# Date 1309869536 -3600
# Node ID 07c64faf886e8a046f39d4dc200ee512a97b7519
# Parent  ebccf967f84db866f6f0d7a1ae9dd2c59c49d3d7
httpstore: implement put to HTTP/WebDAV storage

httpstore.put() uploads each new file with a PUT and, if necessary, MKCOLs to
create the required parent directories. Since the MKCOL response code doesn't
distinguish between 'this isn't a valid place for a collection' and 'there is
already a collection here', we only worry about errors from the PUTs.

diff --git a/bfiles/httpstore.py b/bfiles/httpstore.py
--- a/bfiles/httpstore.py
+++ b/bfiles/httpstore.py
@@ -3,6 +3,7 @@
 import urlparse
 import urllib
 import urllib2
+import new
 
 from mercurial import util, url as url_
 import bfutil, basestore
@@ -47,19 +48,127 @@
     def __init__(self, ui, repo, url):
         self.ui = ui
         self.repo = repo
-        self.url = url
-        (baseurl, authinfo) = url_.getauthinfo(self.url)
+        (baseurl, authinfo) = url_.getauthinfo(url)
+        self.baseurl = baseurl
         self.opener = url_.opener(self.ui, authinfo)
 
+    def __do_put(self, source, url):
+        '''Upload source to an HTTP server (as url) with a PUT
+        request.'''
+        fh = open(source, 'rb')
+        data = fh.read()
+        fh.close()
+
+        try:
+            request = urllib2.Request(url, data = data)
+            request.get_method = new.instancemethod(lambda self: 'PUT', request, request.__class__)
+            request.add_header('Content-Type', 'application/octet-stream')
+            self.opener.open(request)
+            rc = 200
+        except urllib2.HTTPError, err:
+            rc = err.code
+        return rc
+
+    def __do_mkcol(self, url):
+        '''Create a collection with the specified URL on a WevDAV
+        server.'''
+        try:
+            request = urllib2.Request(url)
+            request.get_method = new.instancemethod(lambda self: 'MKCOL', request, request.__class__)
+            self.opener.open(request)
+            rc = 200
+        except urllib2.HTTPError, err:
+            rc = err.code
+        return rc
+
+    # Ensure that there is a collection at the specified location.
+    def __makecols(self, url):
+        '''Try to ensure that a collection with the specified URL exists
+        (including making parent collections as necessary).'''
+        (scheme, host, path, query, frag) = urlparse.urlsplit(url, '', False)
+        assert scheme in ("http", "https")
+        assert query == ""
+        assert frag == ""
+        baseUrl = scheme + "://" + host + "/"
+        leafNames = [leafName for leafName in path.split("/") if len(leafName) != 0]
+
+        # Find how far down the collection tree we can go before we need
+        # to create the collections ourselves.
+        rootwardLimit = -1
+        leafwardLimit = len(leafNames)
+        while rootwardLimit < leafwardLimit:
+            mid = (rootwardLimit + leafwardLimit) / 2
+
+            # If there isn't a middle position (because rootwardLimit
+            # and leafwardLimit are next to each other), just bail out
+            # (and then try making collections from here onwards).
+            if rootwardLimit + 1 == leafwardLimit:
+                break
+
+            midUrl = baseUrl + "/".join(leafNames[0:mid + 1]) + "/"
+            # TODO: just check that the collection exists with PROPFIND?
+            rc = self.__do_mkcol(midUrl)
+            if rc == 201:
+                # We created a collection where there wasn't one before,
+                # so the search is done (and we can carry on making
+                # collections from here onwards).
+                break
+            elif rc == 409:
+                # We can't make this collection until we make its
+                # parents, so it's too leafward.
+                leafwardLimit = mid
+            elif rc == 405:
+                # We can't do an MKCOL on this URL. We assume that a
+                # collection already exists here, so it's too rootward.
+                rootwardLimit = mid
+            elif not (200 <= rc <= 299):
+                # We assume that something's gone wrong.
+                return
+
+        # Now that we've found the lowest collection on our path, make
+        # the rest of them.
+        for i in xrange(mid + 1, len(leafNames)):
+            midUrl = baseUrl + "/".join(leafNames[0:i + 1]) + "/"
+            rc = self.__do_mkcol(midUrl)
+            if not (200 <= rc <= 299):
+                # We assume that something's gone wrong.
+                return
+
+        return
+
     def put(self, source, filename, hash):
-        raise NotImplementedError('sorry, HTTP PUT not implemented yet')
+        parentUrl = urlparse.urljoin(self.baseurl, urllib.quote(filename))
+        url = parentUrl + '/' + hash
+
+        try:
+            # First, assume that the appropriate destination dir already
+            # exists (or that the server will automagically make any
+            # directories needed) and try the PUT.
+            rc = self.__do_put(source, url)
+            if 200 <= rc <= 299:
+                return
+
+            # Try to make the required directories and retry the PUT.
+            self.__makecols(parentUrl)
+            rc = self.__do_put(source, url)
+            if 200 <= rc <= 299:
+                return
+
+            # Give up.
+            detail = "HTTP error: PUT of " + source + " failed with response code " + str(rc)
+            raise basestore.StoreError(filename, hash, url, detail)
+        except urllib2.URLError, err:
+            # This usually indicates a connection problem, so don't
+            # keep trying with the other files... they will probably
+            # all fail too.
+            reason = err[0][1]      # assumes err[0] is a socket.error
+            raise util.Abort('%s: %s' % (self.baseurl, reason))
 
     def _verifyfile(self, cctx, cset, contents, standin, verified):
         raise NotImplementedError('sorry, verify via HTTP not implemented yet')
 
     def _getfile(self, tmpfile, filename, hash):
-        (baseurl, authinfo) = url_.getauthinfo(self.url)
-        url = urlparse.urljoin(baseurl,
+        url = urlparse.urljoin(self.baseurl,
                                urllib.quote(filename) + '/' + hash)
         try:
             infile = self.opener.open(url)
@@ -71,5 +180,5 @@
             # keep trying with the other files... they will probably
             # all fail too.
             reason = err[0][1]      # assumes err[0] is a socket.error
-            raise util.Abort('%s: %s' % (baseurl, reason))
+            raise util.Abort('%s: %s' % (self.baseurl, reason))
         return bfutil._copy_and_hash(bfutil._blockstream(infile), tmpfile)