[PATCH] repo-scan: add option to scan nested repositories

[Angel Ezquerra]

# HG changeset patch
# User angel.ezquerra at gmail.com
# Date 1455366858 -3600
#      Sat Feb 13 13:34:18 2016 +0100
# Node ID 71ec71b81cea0ca1aaefa1e1c5b3063219c96756
# Parent  2f14b4db03626c6eb2fd9b6f541d74c96de0178a
repo-scan: add option to scan nested repositories

"nested repositories" are those that are found within other repositories. An
example of these nested repositories are mercurial repositories. This change
adds an option to the admin settings page to look for nested repositories when
performing a repository scan.

diff -r 2f14b4db0362 -r 71ec71b81cea kallithea/controllers/admin/settings.py
--- a/kallithea/controllers/admin/settings.py    Tue Feb 09 17:46:36 2016 +0100
+++ b/kallithea/controllers/admin/settings.py    Sat Feb 13 13:34:18 2016 +0100
@@ -167,8 +167,9 @@
         c.active = 'mapping'
         if request.POST:
             rm_obsolete = request.POST.get('destroy', False)
+            find_nested = request.POST.get('find_nested', False)
             install_git_hooks = request.POST.get('hooks', False)
-            overwrite_git_hooks = request.POST.get('hooks_overwrite', False);
+            overwrite_git_hooks = request.POST.get('hooks_overwrite', False)
             invalidate_cache = request.POST.get('invalidate', False)
             log.debug('rescanning repo location with destroy obsolete=%s, '
                       'install git hooks=%s and '
@@ -179,7 +180,7 @@
                 for repo in Repository.get_all():
                     ScmModel().mark_for_invalidation(repo.repo_name)

-            filesystem_repos = ScmModel().repo_scan()
+            filesystem_repos = ScmModel().repo_scan(find_nested=find_nested)
             added, removed = repo2db_mapper(filesystem_repos, rm_obsolete,

install_git_hooks=install_git_hooks,
                                             user=c.authuser.username,
diff -r 2f14b4db0362 -r 71ec71b81cea kallithea/lib/utils.py
--- a/kallithea/lib/utils.py    Tue Feb 09 17:46:36 2016 +0100
+++ b/kallithea/lib/utils.py    Sat Feb 13 13:34:18 2016 +0100
@@ -206,12 +206,15 @@
         sa.commit()


-def get_filesystem_repos(path, recursive=False, skip_removed_repos=True):
+def get_filesystem_repos(path, recursive=False, skip_removed_repos=True,
+                         find_nested=False):
     """
     Scans given path for repos and return (name,(type,path)) tuple

     :param path: path to scan for repositories
     :param recursive: recursive search and return names with subdirs in front
+    :param find_nested: look for 'nested repositories' (e.g. subrepos).
+                        find_nested is ignored if recursive is false.
     """

     # remove ending slash for better results
@@ -240,14 +243,17 @@
             try:
                 scm_info = get_scm(cur_path)
                 yield scm_info[1].split(path, 1)[-1].lstrip(os.sep), scm_info
+                if not find_nested:
+                    continue
             except VCSError:
-                if not recursive:
-                    continue
-                #check if this dir containts other repos for recursive scan
-                rec_path = os.path.join(p, dirpath)
-                if not os.path.islink(rec_path) and os.path.isdir(rec_path):
-                    for inner_scm in _get_repos(rec_path):
-                        yield inner_scm
+                pass
+            if not recursive:
+                continue
+            #check if this dir containts other repos for recursive scan
+            rec_path = os.path.join(p, dirpath)
+            if not os.path.islink(rec_path) and os.path.isdir(rec_path):
+                for inner_scm in _get_repos(rec_path):
+                    yield inner_scm

     return _get_repos(path)

diff -r 2f14b4db0362 -r 71ec71b81cea kallithea/model/scm.py
--- a/kallithea/model/scm.py    Tue Feb 09 17:46:36 2016 +0100
+++ b/kallithea/model/scm.py    Sat Feb 13 13:34:18 2016 +0100
@@ -260,12 +260,13 @@

         return q.ui_value

-    def repo_scan(self, repos_path=None):
+    def repo_scan(self, repos_path=None, find_nested=False):
         """
         Listing of repositories in given path. This path should not be a
         repository itself. Return a dictionary of repository objects

         :param repos_path: path to directory containing repositories
+        :param find_nested: look for 'nested repositories' (e.g. subrepos).
         """

         if repos_path is None:
@@ -276,7 +277,8 @@
         baseui = make_ui('db')
         repos = {}

-        for name, path in get_filesystem_repos(repos_path, recursive=True):
+        for name, path in get_filesystem_repos(repos_path, recursive=True,
+                                               find_nested=find_nested):
             # name need to be decomposed and put back together using the /
             # since this is internal storage separator for kallithea
             name = Repository.normalize_repo_name(name)
diff -r 2f14b4db0362 -r 71ec71b81cea
kallithea/templates/admin/settings/settings_mapping.html
--- a/kallithea/templates/admin/settings/settings_mapping.html    Tue
Feb 09 17:46:36 2016 +0100
+++ b/kallithea/templates/admin/settings/settings_mapping.html    Sat
Feb 13 13:34:18 2016 +0100
@@ -19,6 +19,12 @@
                     <span class="help-block">${_('Check this to
reload data and clear cache keys for all repositories.')}</span>

                     <div class="checkbox">
+                        ${h.checkbox('find_nested',False)}
+                        <label for="find_nested">${_('Look for nested
repositories (e.g. mercurial subrepos)')}</label>
+                    </div>
+                    <span class="help-block">${_('Check this option
to look for mercurial subrepos within mercurial
repositories.')}</span>
+
+                    <div class="checkbox">
                         ${h.checkbox('hooks',True)}
                         <label for="hooks"> ${_('Install Git hooks')} </label>
                     </div>