D6890: sidedata: add a function to read sidedata from revlog raw text

[marmoute (Pierre-Yves David)]

marmoute updated this revision to Diff 16737.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D6890?vs=16657&id=16737

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D6890/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D6890

AFFECTED FILES
  mercurial/error.py
  mercurial/revlog.py
  mercurial/revlogutils/sidedata.py

CHANGE DETAILS

diff --git a/mercurial/revlogutils/sidedata.py b/mercurial/revlogutils/sidedata.py
--- a/mercurial/revlogutils/sidedata.py
+++ b/mercurial/revlogutils/sidedata.py
@@ -32,3 +32,32 @@
 """
 
 from __future__ import absolute_import
+
+import hashlib
+import struct
+
+from .. import error
+
+SIDEDATA_HEADER = struct.Struct('>H')
+SIDEDATA_ENTRY = struct.Struct('>HL20s')
+
+def sidedatareadprocessor(rl, text):
+    sidedata = {}
+    offset = 0
+    nbentry, = SIDEDATA_HEADER.unpack(text[:SIDEDATA_HEADER.size])
+    offset += SIDEDATA_HEADER.size
+    dataoffset = SIDEDATA_HEADER.size + (SIDEDATA_ENTRY.size * nbentry)
+    for i in range(nbentry):
+        nextoffset = offset + SIDEDATA_ENTRY.size
+        key, size, storeddigest = SIDEDATA_ENTRY.unpack(text[offset:nextoffset])
+        offset = nextoffset
+        # read the data associated with that entry
+        nextdataoffset = dataoffset + size
+        entrytext = text[dataoffset:nextdataoffset]
+        readdigest = hashlib.sha1(entrytext).digest()
+        if storeddigest != readdigest:
+            raise error.SidedataHashError(key, storeddigest, readdigest)
+        sidedata[key] = entrytext
+        dataoffset = nextdataoffset
+    text = text[dataoffset:]
+    return text, True, sidedata
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -1678,7 +1678,13 @@
             validatehash = flagutil.processflagsraw(self, rawtext, flags)
             text = rawtext
         else:
-            r = flagutil.processflagsread(self, rawtext, flags)
+            try:
+                r = flagutil.processflagsread(self, rawtext, flags)
+            except error.SidedataHashError as exc:
+                msg = _("integrity check failed on %s:%s sidedata key %d")
+                msg %= (self.indexfile, pycompat.bytestr(rev),
+                        exc.sidedatakey)
+                raise error.RevlogError(msg)
             text, validatehash, sidedata = r
         if validatehash:
             self.checkhash(text, node, rev=rev)
diff --git a/mercurial/error.py b/mercurial/error.py
--- a/mercurial/error.py
+++ b/mercurial/error.py
@@ -44,6 +44,13 @@
 class RevlogError(StorageError):
     __bytes__ = _tobytes
 
+class SidedataHashError(RevlogError):
+
+    def __init__(self, key, expected, got):
+        self.sidedatakey = key
+        self.expecteddigest = expected
+        self.actualdigest = got
+
 class FilteredIndexError(IndexError):
     __bytes__ = _tobytes
 



To: marmoute, durin42, indygreg, #hg-reviewers
Cc: mercurial-devel