[PATCH V2] repo-scan: add option to scan nested repositories
Mads Kiilerich
mads at kiilerich.com
Wed Mar 2 15:53:13 UTC 2016
(It seems like I forgot to send this one?)
On 02/28/2016 11:49 AM, Angel Ezquerra wrote:
> # HG changeset patch
> # User Angel Ezquerra <angel.ezquerra at gmail.com>
> # Date 1455366858 -3600
> # Sat Feb 13 13:34:18 2016 +0100
> # Branch stable
> # Node ID 083d69152e6949ff25e2a311b19105667468fb7d
> # Parent 9621195d7b42a95ee84669ff888e551bf31e4148
> repo-scan: add option to scan nested repositories
>
> "nested repositories" are those that are found within other repositories.
perhaps clarify further "found within the working directory or URL
prefix of other repositories"
> An
> example of these nested repositories are mercurial repositories.
Mercurial _sub_ repositories.
> This change
> adds an option to the admin settings page to look for nested repositories when
> performing a repository scan.
What will happen when such repositories are found? Will they be added to
the Kallithea repo list? Will they work? Can they be renamed? Can they
be added in other ways? It seems like the answer is non-trivial and I
would expect it in the commit description.
> diff --git a/kallithea/controllers/admin/settings.py b/kallithea/controllers/admin/settings.py
> --- a/kallithea/controllers/admin/settings.py
> +++ b/kallithea/controllers/admin/settings.py
> @@ -196,8 +196,9 @@
> c.active = 'mapping'
> if request.POST:
> rm_obsolete = request.POST.get('destroy', False)
> + find_nested = request.POST.get('find_nested', False)
> install_git_hooks = request.POST.get('hooks', False)
> - overwrite_git_hooks = request.POST.get('hooks_overwrite', False);
> + overwrite_git_hooks = request.POST.get('hooks_overwrite', False)
> invalidate_cache = request.POST.get('invalidate', False)
> log.debug('rescanning repo location with destroy obsolete=%s, '
> 'install git hooks=%s and '
> @@ -208,7 +209,7 @@
> for repo in Repository.get_all():
> ScmModel().mark_for_invalidation(repo.repo_name, delete=True)
>
> - filesystem_repos = ScmModel().repo_scan()
> + filesystem_repos = ScmModel().repo_scan(find_nested=find_nested)
> added, removed = repo2db_mapper(filesystem_repos, rm_obsolete,
> install_git_hooks=install_git_hooks,
> user=c.authuser.username,
> diff --git a/kallithea/lib/utils.py b/kallithea/lib/utils.py
> --- a/kallithea/lib/utils.py
> +++ b/kallithea/lib/utils.py
> @@ -207,12 +207,15 @@
> sa.commit()
>
>
> -def get_filesystem_repos(path, recursive=False, skip_removed_repos=True):
> +def get_filesystem_repos(path, recursive=False, skip_removed_repos=True,
> + find_nested=False):
> """
> Scans given path for repos and return (name,(type,path)) tuple
>
> :param path: path to scan for repositories
> :param recursive: recursive search and return names with subdirs in front
> + :param find_nested: look for 'nested repositories' (e.g. subrepos).
> + find_nested is ignored if recursive is false.
> """
>
> # remove ending slash for better results
> @@ -239,16 +242,27 @@
> continue
>
> try:
> + # try to access the cur_path as an SCM (i.e. a repository)
> scm_info = get_scm(cur_path)
> + # cur_path is the root of a repository
> yield scm_info[1].split(path, 1)[-1].lstrip(os.sep), scm_info
> + if not find_nested:
> + # do not recurse into the current repository
> + continue
> + # potentially recurse into the current repository (if
> + # recursive is enabled), to look for nested repositories
> except VCSError:
> - if not recursive:
> - continue
> - #check if this dir containts other repos for recursive scan
> - rec_path = os.path.join(p, dirpath)
> - if not os.path.islink(rec_path) and os.path.isdir(rec_path):
> - for inner_scm in _get_repos(rec_path):
> - yield inner_scm
> + # cur_path is not a repository -> do we need to recurse?
> + pass
> + if not recursive:
> + continue
> + # recurse into the current directory (or repository), looking for
> + # repositories (or nested repositories) deeper down the directory
> + # hierarchy
> + rec_path = os.path.join(p, dirpath)
> + if not os.path.islink(rec_path) and os.path.isdir(rec_path):
> + for inner_scm in _get_repos(rec_path):
> + yield inner_scm
It seems like this both is a bit of refactoring and improved comments
... and the new functionality. It would be nice to get it separated.
>
> return _get_repos(path)
>
> diff --git a/kallithea/model/scm.py b/kallithea/model/scm.py
> --- a/kallithea/model/scm.py
> +++ b/kallithea/model/scm.py
> @@ -260,12 +260,13 @@
>
> return q.ui_value
>
> - def repo_scan(self, repos_path=None):
> + def repo_scan(self, repos_path=None, find_nested=False):
> """
> Listing of repositories in given path. This path should not be a
> repository itself. Return a dictionary of repository objects
>
> :param repos_path: path to directory containing repositories
> + :param find_nested: look for 'nested repositories' (e.g. subrepos).
> """
>
> if repos_path is None:
> @@ -276,7 +277,8 @@
> baseui = make_ui('db')
> repos = {}
>
> - for name, path in get_filesystem_repos(repos_path, recursive=True):
> + for name, path in get_filesystem_repos(repos_path, recursive=True,
> + find_nested=find_nested):
> # name need to be decomposed and put back together using the /
> # since this is internal storage separator for kallithea
> name = Repository.normalize_repo_name(name)
> diff --git a/kallithea/templates/admin/settings/settings_mapping.html b/kallithea/templates/admin/settings/settings_mapping.html
> --- a/kallithea/templates/admin/settings/settings_mapping.html
> +++ b/kallithea/templates/admin/settings/settings_mapping.html
> @@ -19,6 +19,12 @@
> <span class="help-block">${_('Check this to reload data and clear cache keys for all repositories.')}</span>
>
> <div class="checkbox">
> + ${h.checkbox('find_nested',False)}
> + <label for="find_nested">${_('Look for nested repositories (e.g. Mercurial subrepos)')}</label>
> + </div>
> + <span class="help-block">${_('Check this option to look for repositories within existing repositories (changing the default behavior which is to not recurse into repositories during a repository scan). This option can be useful, for example, when you use Mercurial subrepositories (which are normally found within the parent repository working directory, and thus will not be found unless you enable this option).')}</span>
> +
> + <div class="checkbox">
> ${h.checkbox('hooks',True)}
> <label for="hooks"> ${_('Install Git hooks')} </label>
> </div>
>
>
> _______________________________________________
> kallithea-general mailing list
> kallithea-general at sfconservancy.org
> http://lists.sfconservancy.org/mailman/listinfo/kallithea-general
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.sfconservancy.org/pipermail/kallithea-general/attachments/20160302/5f34e58c/attachment.html>
More information about the kallithea-general
mailing list