[Request] [+--- ] D11520: dirstate-v2: Add support when Rust is not enabled
SimonSapin
phabricator at mercurial-scm.org
Fri Oct 1 07:20:26 UTC 2021
SimonSapin created this revision.
Herald added a reviewer: hg-reviewers.
Herald added a subscriber: mercurial-patches.
REVISION SUMMARY
This wires into `dirstatemap` the parser and serializer added in previous
changesets. The memory representation is still the same, with a flat `dict`
for `DirstateItem`s and another one for copy sources. Serialization always
creates a new dirstate-v2 data file and does not support (when Rust is not
enabled) appending to an existing one, since we don’t keep track of which
tree nodes are new or modified. Instead the tree is reconstructed during
serialization.
REPOSITORY
rHG Mercurial
BRANCH
default
REVISION DETAIL
https://phab.mercurial-scm.org/D11520
AFFECTED FILES
mercurial/dirstate.py
mercurial/dirstatemap.py
mercurial/localrepo.py
tests/test-dirstate-race.t
tests/test-dirstate-race2.t
tests/test-dirstate.t
tests/test-hgignore.t
tests/test-permissions.t
tests/test-purge.t
tests/test-status.t
tests/test-symlinks.t
CHANGE DETAILS
diff --git a/tests/test-symlinks.t b/tests/test-symlinks.t
--- a/tests/test-symlinks.t
+++ b/tests/test-symlinks.t
@@ -3,7 +3,6 @@
#testcases dirstate-v1 dirstate-v2
#if dirstate-v2
-#require rust
$ echo '[format]' >> $HGRCPATH
$ echo 'exp-dirstate-v2=1' >> $HGRCPATH
#endif
diff --git a/tests/test-status.t b/tests/test-status.t
--- a/tests/test-status.t
+++ b/tests/test-status.t
@@ -1,13 +1,6 @@
#testcases dirstate-v1 dirstate-v2
-#if no-rust
- $ hg init repo0 --config format.exp-dirstate-v2=1
- abort: dirstate v2 format requested by config but not supported (requires Rust extensions)
- [255]
-#endif
-
#if dirstate-v2
-#require rust
$ echo '[format]' >> $HGRCPATH
$ echo 'exp-dirstate-v2=1' >> $HGRCPATH
#endif
@@ -743,7 +736,7 @@
if also listing unknowns.
The tree-based dirstate and status algorithm fix this:
-#if symlink no-dirstate-v1
+#if symlink no-dirstate-v1 rust
$ cd ..
$ hg init issue6335
@@ -759,11 +752,11 @@
? bar/a
? foo
- $ hg status -c # incorrect output with `dirstate-v1`
+ $ hg status -c # incorrect output without the Rust implementation
$ hg status -cu
? bar/a
? foo
- $ hg status -d # incorrect output with `dirstate-v1`
+ $ hg status -d # incorrect output without the Rust implementation
! foo/a
$ hg status -du
! foo/a
@@ -910,7 +903,7 @@
I B.hs
I ignored-folder/ctest.hs
-#if dirstate-v2
+#if rust dirstate-v2
Check read_dir caching
diff --git a/tests/test-purge.t b/tests/test-purge.t
--- a/tests/test-purge.t
+++ b/tests/test-purge.t
@@ -1,7 +1,6 @@
#testcases dirstate-v1 dirstate-v2
#if dirstate-v2
-#require rust
$ echo '[format]' >> $HGRCPATH
$ echo 'exp-dirstate-v2=1' >> $HGRCPATH
#endif
diff --git a/tests/test-permissions.t b/tests/test-permissions.t
--- a/tests/test-permissions.t
+++ b/tests/test-permissions.t
@@ -3,7 +3,6 @@
#testcases dirstate-v1 dirstate-v2
#if dirstate-v2
-#require rust
$ echo '[format]' >> $HGRCPATH
$ echo 'exp-dirstate-v2=1' >> $HGRCPATH
#endif
diff --git a/tests/test-hgignore.t b/tests/test-hgignore.t
--- a/tests/test-hgignore.t
+++ b/tests/test-hgignore.t
@@ -1,7 +1,6 @@
#testcases dirstate-v1 dirstate-v2
#if dirstate-v2
-#require rust
$ echo '[format]' >> $HGRCPATH
$ echo 'exp-dirstate-v2=1' >> $HGRCPATH
#endif
@@ -397,9 +396,10 @@
#endif
-#if dirstate-v2
+#if dirstate-v2 rust
Check the hash of ignore patterns written in the dirstate
+This is an optimization that is only relevant when using the Rust extensions
$ hg status > /dev/null
$ cat .hg/testhgignore .hg/testhgignorerel .hgignore dir2/.hgignore dir1/.hgignore dir1/.hgignoretwo | $TESTDIR/f --sha1
diff --git a/tests/test-dirstate.t b/tests/test-dirstate.t
--- a/tests/test-dirstate.t
+++ b/tests/test-dirstate.t
@@ -1,7 +1,6 @@
#testcases dirstate-v1 dirstate-v2
#if dirstate-v2
-#require rust
$ echo '[format]' >> $HGRCPATH
$ echo 'exp-dirstate-v2=1' >> $HGRCPATH
#endif
diff --git a/tests/test-dirstate-race2.t b/tests/test-dirstate-race2.t
--- a/tests/test-dirstate-race2.t
+++ b/tests/test-dirstate-race2.t
@@ -1,7 +1,6 @@
#testcases dirstate-v1 dirstate-v2
#if dirstate-v2
-#require rust
$ echo '[format]' >> $HGRCPATH
$ echo 'exp-dirstate-v2=1' >> $HGRCPATH
#endif
diff --git a/tests/test-dirstate-race.t b/tests/test-dirstate-race.t
--- a/tests/test-dirstate-race.t
+++ b/tests/test-dirstate-race.t
@@ -1,7 +1,6 @@
#testcases dirstate-v1 dirstate-v2
#if dirstate-v2
-#require rust
$ echo '[format]' >> $HGRCPATH
$ echo 'exp-dirstate-v2=1' >> $HGRCPATH
#endif
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -917,9 +917,6 @@
# Start with all requirements supported by this file.
supported = set(localrepository._basesupported)
- if dirstate.SUPPORTS_DIRSTATE_V2:
- supported.add(requirementsmod.DIRSTATE_V2_REQUIREMENT)
-
# Execute ``featuresetupfuncs`` entries if they belong to an extension
# relevant to this ui instance.
modules = {m.__name__ for n, m in extensions.extensions(ui)}
@@ -1266,6 +1263,7 @@
requirementsmod.NODEMAP_REQUIREMENT,
bookmarks.BOOKMARKS_IN_STORE_REQUIREMENT,
requirementsmod.SHARESAFE_REQUIREMENT,
+ requirementsmod.DIRSTATE_V2_REQUIREMENT,
}
_basesupported = supportedformats | {
requirementsmod.STORE_REQUIREMENT,
@@ -3609,15 +3607,7 @@
# experimental config: format.exp-dirstate-v2
# Keep this logic in sync with `has_dirstate_v2()` in `tests/hghave.py`
if ui.configbool(b'format', b'exp-dirstate-v2'):
- if dirstate.SUPPORTS_DIRSTATE_V2:
- requirements.add(requirementsmod.DIRSTATE_V2_REQUIREMENT)
- else:
- raise error.Abort(
- _(
- b"dirstate v2 format requested by config "
- b"but not supported (requires Rust extensions)"
- )
- )
+ requirements.add(requirementsmod.DIRSTATE_V2_REQUIREMENT)
# experimental config: format.exp-use-copies-side-data-changeset
if ui.configbool(b'format', b'exp-use-copies-side-data-changeset'):
diff --git a/mercurial/dirstatemap.py b/mercurial/dirstatemap.py
--- a/mercurial/dirstatemap.py
+++ b/mercurial/dirstatemap.py
@@ -36,7 +36,111 @@
rangemask = 0x7FFFFFFF
-class dirstatemap(object):
+class dirstatemapcommon(object):
+ """
+ Methods that are idertical for both implementations of the dirstatemap
+ class, with and without Rust extensions enabled.
+ """
+
+ def __init__(self, ui, opener, root, nodeconstants, use_dirstate_v2):
+ self._ui = ui
+ self._opener = opener
+ self._root = root
+ self._filename = b'dirstate'
+ self._nodelen = 20 # Also update Rust code when changing this!
+ self._nodeconstants = nodeconstants
+ self._use_dirstate_v2 = use_dirstate_v2
+ self._parents = None
+ self._dirtyparents = False
+ self._docket = None
+
+ # for consistent view between _pl() and _read() invocations
+ self._pendingmode = None
+
+ def _opendirstatefile(self):
+ fp, mode = txnutil.trypending(self._root, self._opener, self._filename)
+ if self._pendingmode is not None and self._pendingmode != mode:
+ fp.close()
+ raise error.Abort(
+ _(b'working directory state may be changed parallelly')
+ )
+ self._pendingmode = mode
+ return fp
+
+ def _readdirstatefile(self, size=-1):
+ try:
+ with self._opendirstatefile() as fp:
+ return fp.read(size)
+ except IOError as err:
+ if err.errno != errno.ENOENT:
+ raise
+ # File doesn't exist, so the current state is empty
+ return b''
+
+ @property
+ def docket(self):
+ if not self._docket:
+ if not self._use_dirstate_v2:
+ raise error.ProgrammingError(
+ b'dirstate only has a docket in v2 format'
+ )
+ self._docket = docketmod.DirstateDocket.parse(
+ self._readdirstatefile(), self._nodeconstants
+ )
+ return self._docket
+
+ def parents(self):
+ if not self._parents:
+ if self._use_dirstate_v2:
+ self._parents = self.docket.parents
+ else:
+ read_len = self._nodelen * 2
+ st = self._readdirstatefile(read_len)
+ l = len(st)
+ if l == read_len:
+ self._parents = (
+ st[: self._nodelen],
+ st[self._nodelen : 2 * self._nodelen],
+ )
+ elif l == 0:
+ self._parents = (
+ self._nodeconstants.nullid,
+ self._nodeconstants.nullid,
+ )
+ else:
+ raise error.Abort(
+ _(b'working directory state appears damaged!')
+ )
+ return self._parents
+
+ def write_no_append(self, tr, st, meta, packed):
+ old_docket = self.docket
+ new_docket = docketmod.DirstateDocket.with_new_uuid(
+ self.parents(), len(packed), meta
+ )
+ data_filename = new_docket.data_filename()
+ if tr:
+ tr.add(data_filename, 0)
+ self._opener.write(data_filename, packed)
+ # Write the new docket after the new data file has been
+ # written. Because `st` was opened with `atomictemp=True`,
+ # the actual `.hg/dirstate` file is only affected on close.
+ st.write(new_docket.serialize())
+ st.close()
+ # Remove the old data file after the new docket pointing to
+ # the new data file was written.
+ if old_docket.uuid:
+ data_filename = old_docket.data_filename()
+ unlink = lambda _tr=None: self._opener.unlink(data_filename)
+ if tr:
+ category = b"dirstate-v2-clean-" + old_docket.uuid
+ tr.addpostclose(category, unlink)
+ else:
+ unlink()
+ self._docket = new_docket
+
+
+class dirstatemap(dirstatemapcommon):
"""Map encapsulating the dirstate's contents.
The dirstate contains the following state:
@@ -70,23 +174,6 @@
denormalized form that they appear as in the dirstate.
"""
- def __init__(self, ui, opener, root, nodeconstants, use_dirstate_v2):
- self._ui = ui
- self._opener = opener
- self._root = root
- self._filename = b'dirstate'
- self._nodelen = 20
- self._nodeconstants = nodeconstants
- assert (
- not use_dirstate_v2
- ), "should have detected unsupported requirement"
-
- self._parents = None
- self._dirtyparents = False
-
- # for consistent view between _pl() and _read() invocations
- self._pendingmode = None
-
@propertycache
def _map(self):
self._map = {}
@@ -351,46 +438,6 @@
def _alldirs(self):
return pathutil.dirs(self._map)
- def _opendirstatefile(self):
- fp, mode = txnutil.trypending(self._root, self._opener, self._filename)
- if self._pendingmode is not None and self._pendingmode != mode:
- fp.close()
- raise error.Abort(
- _(b'working directory state may be changed parallelly')
- )
- self._pendingmode = mode
- return fp
-
- def parents(self):
- if not self._parents:
- try:
- fp = self._opendirstatefile()
- st = fp.read(2 * self._nodelen)
- fp.close()
- except IOError as err:
- if err.errno != errno.ENOENT:
- raise
- # File doesn't exist, so the current state is empty
- st = b''
-
- l = len(st)
- if l == self._nodelen * 2:
- self._parents = (
- st[: self._nodelen],
- st[self._nodelen : 2 * self._nodelen],
- )
- elif l == 0:
- self._parents = (
- self._nodeconstants.nullid,
- self._nodeconstants.nullid,
- )
- else:
- raise error.Abort(
- _(b'working directory state appears damaged!')
- )
-
- return self._parents
-
def setparents(self, p1, p2, fold_p2=False):
self._parents = (p1, p2)
self._dirtyparents = True
@@ -411,19 +458,17 @@
self._opener.join(self._filename)
)
- try:
- fp = self._opendirstatefile()
- try:
- st = fp.read()
- finally:
- fp.close()
- except IOError as err:
- if err.errno != errno.ENOENT:
- raise
- return
+ if self._use_dirstate_v2:
+ if not self.docket.uuid:
+ return
+ st = self._opener.read(self.docket.data_filename())
+ else:
+ st = self._readdirstatefile()
+
if not st:
return
+ # TODO: adjust this estimate for dirstate-v2
if util.safehasattr(parsers, b'dict_new_presized'):
# Make an estimate of the number of files in the dirstate based on
# its size. This trades wasting some memory for avoiding costly
@@ -445,8 +490,14 @@
# parsing the dirstate.
#
# (we cannot decorate the function directly since it is in a C module)
- parse_dirstate = util.nogc(parsers.parse_dirstate)
- p = parse_dirstate(self._map, self.copymap, st)
+ if self._use_dirstate_v2:
+ p = self.docket.parents
+ meta = self.docket.tree_metadata
+ parse_dirstate = util.nogc(v2.parse_dirstate)
+ parse_dirstate(self._map, self.copymap, st, meta)
+ else:
+ parse_dirstate = util.nogc(parsers.parse_dirstate)
+ p = parse_dirstate(self._map, self.copymap, st)
if not self._dirtyparents:
self.setparents(*p)
@@ -455,11 +506,19 @@
self.__getitem__ = self._map.__getitem__
self.get = self._map.get
- def write(self, _tr, st, now):
- st.write(
- parsers.pack_dirstate(self._map, self.copymap, self.parents(), now)
- )
- st.close()
+ def write(self, tr, st, now):
+ if not self._use_dirstate_v2:
+ st.write(
+ parsers.pack_dirstate(
+ self._map, self.copymap, self.parents(), now
+ )
+ )
+ st.close()
+ self._dirtyparents = False
+ return
+
+ packed, meta = v2.pack_dirstate(self._map, self.copymap, now)
+ self.write_no_append(tr, st, meta, packed)
self._dirtyparents = False
@propertycache
@@ -476,23 +535,15 @@
return f
+# When Rust is enabled, define a different implementation of
+# the `dirstatemap` class.
if rustmod is not None:
- class dirstatemap(object):
+ class dirstatemap(dirstatemapcommon):
def __init__(self, ui, opener, root, nodeconstants, use_dirstate_v2):
- self._use_dirstate_v2 = use_dirstate_v2
- self._nodeconstants = nodeconstants
- self._ui = ui
- self._opener = opener
- self._root = root
- self._filename = b'dirstate'
- self._nodelen = 20 # Also update Rust code when changing this!
- self._parents = None
- self._dirtyparents = False
- self._docket = None
-
- # for consistent view between _pl() and _read() invocations
- self._pendingmode = None
+ super(dirstatemap, self).__init__(
+ ui, opener, root, nodeconstants, use_dirstate_v2
+ )
def addfile(
self,
@@ -693,28 +744,6 @@
# forward for python2,3 compat
iteritems = items
- def _opendirstatefile(self):
- fp, mode = txnutil.trypending(
- self._root, self._opener, self._filename
- )
- if self._pendingmode is not None and self._pendingmode != mode:
- fp.close()
- raise error.Abort(
- _(b'working directory state may be changed parallelly')
- )
- self._pendingmode = mode
- return fp
-
- def _readdirstatefile(self, size=-1):
- try:
- with self._opendirstatefile() as fp:
- return fp.read(size)
- except IOError as err:
- if err.errno != errno.ENOENT:
- raise
- # File doesn't exist, so the current state is empty
- return b''
-
def setparents(self, p1, p2, fold_p2=False):
self._parents = (p1, p2)
self._dirtyparents = True
@@ -754,43 +783,6 @@
)
return copies
- def parents(self):
- if not self._parents:
- if self._use_dirstate_v2:
- self._parents = self.docket.parents
- else:
- read_len = self._nodelen * 2
- st = self._readdirstatefile(read_len)
- l = len(st)
- if l == read_len:
- self._parents = (
- st[: self._nodelen],
- st[self._nodelen : 2 * self._nodelen],
- )
- elif l == 0:
- self._parents = (
- self._nodeconstants.nullid,
- self._nodeconstants.nullid,
- )
- else:
- raise error.Abort(
- _(b'working directory state appears damaged!')
- )
-
- return self._parents
-
- @property
- def docket(self):
- if not self._docket:
- if not self._use_dirstate_v2:
- raise error.ProgrammingError(
- b'dirstate only has a docket in v2 format'
- )
- self._docket = docketmod.DirstateDocket.parse(
- self._readdirstatefile(), self._nodeconstants
- )
- return self._docket
-
@propertycache
def _rustmap(self):
"""
@@ -853,30 +845,7 @@
st.write(docket.serialize())
st.close()
else:
- old_docket = self.docket
- new_docket = docketmod.DirstateDocket.with_new_uuid(
- self.parents(), len(packed), meta
- )
- data_filename = new_docket.data_filename()
- if tr:
- tr.add(data_filename, 0)
- self._opener.write(data_filename, packed)
- # Write the new docket after the new data file has been
- # written. Because `st` was opened with `atomictemp=True`,
- # the actual `.hg/dirstate` file is only affected on close.
- st.write(new_docket.serialize())
- st.close()
- # Remove the old data file after the new docket pointing to
- # the new data file was written.
- if old_docket.uuid:
- data_filename = old_docket.data_filename()
- unlink = lambda _tr=None: self._opener.unlink(data_filename)
- if tr:
- category = b"dirstate-v2-clean-" + old_docket.uuid
- tr.addpostclose(category, unlink)
- else:
- unlink()
- self._docket = new_docket
+ self.write_no_append(tr, st, meta, packed)
# Reload from the newly-written file
util.clearcachedproperty(self, b"_rustmap")
self._dirtyparents = False
diff --git a/mercurial/dirstate.py b/mercurial/dirstate.py
--- a/mercurial/dirstate.py
+++ b/mercurial/dirstate.py
@@ -39,8 +39,6 @@
parsers = policy.importmod('parsers')
rustmod = policy.importrust('dirstate')
-SUPPORTS_DIRSTATE_V2 = rustmod is not None
-
propertycache = util.propertycache
filecache = scmutil.filecache
_rangemask = dirstatemap.rangemask
To: SimonSapin, #hg-reviewers
Cc: mercurial-patches, mercurial-devel
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mercurial-scm.org/pipermail/mercurial-patches/attachments/20211001/fce3bc58/attachment-0001.html>
More information about the Mercurial-patches
mailing list