# HG changeset patch # User Kostas Bourantanis # Date 1784294831 -10800 # Fri Jul 17 16:27:11 2026 +0300 # Node ID 15d33eb902ece311eba710611e382101fc6b777c # Parent 976a1e77932ea7de09558989eb82182c9429ed8f hg: make MercurialRepository.revisions lazy to speed up large repositories diff -r 976a1e77932e -r 15d33eb902ec kallithea/lib/vcs/backends/hg/inmemory.py --- a/kallithea/lib/vcs/backends/hg/inmemory.py Wed Nov 20 21:04:00 2024 +0100 +++ b/kallithea/lib/vcs/backends/hg/inmemory.py Fri Jul 17 16:27:11 2026 +0300 @@ -96,6 +96,14 @@ commit_ctx._user = safe_bytes(author) commit_ctx._date = date + # Materialize the lazy revisions view *before* committing, so its + # snapshot does not yet contain the revision we are about to create. + # This keeps the cheap incremental append() below correct: without + # this, the first access would happen after commitctx() and capture + # the new revision in the snapshot, which append() would then add a + # second time (see MercurialRepository.LazyRevisions). + revisions = self.repository.revisions + # TODO: Catch exceptions! n = self.repository._repo.commitctx(commit_ctx) # Returns mercurial node @@ -103,7 +111,7 @@ # Update vcs repository object & recreate mercurial _repo # new_ctx = self.repository._repo[node] # new_tip = ascii_str(self.repository.get_changeset(new_ctx.hex())) - self.repository.revisions.append(ascii_str(mercurial.node.hex(n))) + revisions.append(ascii_str(mercurial.node.hex(n))) self._repo = self.repository._get_repo(create=False) self.repository.branches = self.repository._get_branches() tip = self.repository.get_changeset() diff -r 976a1e77932e -r 15d33eb902ec kallithea/lib/vcs/backends/hg/repository.py --- a/kallithea/lib/vcs/backends/hg/repository.py Wed Nov 20 21:04:00 2024 +0100 +++ b/kallithea/lib/vcs/backends/hg/repository.py Fri Jul 17 16:27:11 2026 +0300 @@ -9,6 +9,7 @@ :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak. """ +import bisect import datetime import logging import os @@ -55,6 +56,79 @@ log = logging.getLogger(__name__) +class LazyRevisions(object): + """Lazily evaluated, list-like view of all visible changeset hex ids. + + Behaves like the previous eagerly-built ``list`` of every visible changeset + hex id (ascending order), but without materializing the hex id of *every* + changeset up front. ``len()`` and ``index()`` are cheap (O(1) / O(log n)); + individual hex ids are computed on access. This keeps the landing page, + summary, changelog and changeset navigation fast on large repositories, + where building the full list was O(total changesets) in both time and + memory on every request. + """ + + def __init__(self, repo): + self._repo = repo + # ascending list of *visible* revision numbers - cheap (ints only) + self._revs = list(repo.filtered(b'visible').changelog.revs()) + # hex ids of changesets committed *after* this snapshot was taken (e.g. + # by InMemoryChangeset.commit); appended incrementally to avoid + # rebuilding the whole view on every commit + self._extra = [] + + def __len__(self): + return len(self._revs) + len(self._extra) + + def __bool__(self): + return bool(self._revs) or bool(self._extra) + + def _hex(self, rev): + return ascii_str(self._repo[rev].hex()) + + def __getitem__(self, key): + if isinstance(key, slice): + return [self[i] for i in range(*key.indices(len(self)))] + if key < 0: + key += len(self) + nrevs = len(self._revs) + if 0 <= key < nrevs: + return self._hex(self._revs[key]) + if nrevs <= key < nrevs + len(self._extra): + return self._extra[key - nrevs] + raise IndexError('revision index out of range') + + def __iter__(self): + for rev in self._revs: + yield self._hex(rev) + for hexid in self._extra: + yield hexid + + def __contains__(self, hexid): + try: + self.index(hexid) + except ValueError: + return False + return True + + def index(self, hexid): + try: + rev = self._repo[safe_bytes(hexid)].rev() + pos = bisect.bisect_left(self._revs, rev) + if pos != len(self._revs) and self._revs[pos] == rev: + return pos + except (mercurial.error.RepoError, mercurial.error.LookupError, LookupError, ValueError): + pass + # hex ids appended after construction (InMemoryChangeset) + try: + return len(self._revs) + self._extra.index(hexid) + except ValueError: + raise ValueError('%s not in revisions' % (hexid,)) + + def append(self, hexid): + self._extra.append(hexid) + + class MercurialRepository(BaseRepository): """ Mercurial repository backend @@ -91,9 +165,10 @@ """ Checks if repository is empty ie. without any changesets """ - # TODO: Following raises errors when using InMemoryChangeset... - # return len(self._repo.changelog) == 0 - return len(self.revisions) == 0 + # Cheap emptiness check that avoids materializing self.revisions, which + # builds the hex id of *every* changeset (O(total changesets) in time and + # memory). tiprev() is O(1) and returns nullrev (-1) for an empty repo. + return self._repo.filtered(b'visible').changelog.tiprev() == mercurial.node.nullrev @LazyProperty def revisions(self): @@ -240,7 +315,9 @@ )) def _get_all_revisions(self): - return [ascii_str(self._repo[x].hex()) for x in self._repo.filtered(b'visible').changelog.revs()] + # Lazy, list-like view: cheap len()/index()/slicing without building the + # hex id of every changeset up front (see LazyRevisions). + return LazyRevisions(self._repo) def get_diff(self, rev1, rev2, path='', ignore_whitespace=False, context=3): @@ -561,7 +638,13 @@ revspec = b'limit(%s, %d)' % (revspec, max_revisions) revisions = mercurial.scmutil.revrange(self._repo, [revspec]) else: - revisions = self.revisions + # No explicit filtering: use the visible changelog's revision numbers + # directly instead of materializing the hex id of *every* revision + # (previously O(total changesets) in time and memory on each request, + # e.g. ~2.5s / ~115MB for a 700k-changeset repo just to show one page). + # CollectionGenerator/get_changeset accept integer revision numbers, + # exactly like the mercurial.scmutil.revrange() branch above. + revisions = self._repo.filtered(b'visible').changelog.revs() # this is very much a hack to turn this into a list; a better solution # would be to get rid of this function entirely and use revsets