[PATCH 09 of 17 v3] cli: convert 'gearbox cleanup-repos' into 'kallithea-cli repo-purge-deleted'
Thomas De Schampheleire
patrickdepinguin at gmail.com
Thu Oct 18 20:49:32 UTC 2018
# HG changeset patch
# User Thomas De Schampheleire <thomas.de_schampheleire at nokia.com>
# Date 1539547124 -7200
# Sun Oct 14 21:58:44 2018 +0200
# Node ID 1053ff6d2643aa2851466c4a4d57bde5c80e7d8d
# Parent 41448a482e0d03d701e8acd0fa03a15cfe3ba2aa
cli: convert 'gearbox cleanup-repos' into 'kallithea-cli repo-purge-deleted'
Some changes to user-facing text are done.
diff --git a/docs/usage/general.rst b/docs/usage/general.rst
--- a/docs/usage/general.rst
+++ b/docs/usage/general.rst
@@ -8,18 +8,18 @@ General Kallithea usage
Repository deletion
-------------------
-Currently when an admin or owner deletes a repository, Kallithea does
+When an admin or owner deletes a repository, Kallithea does
not physically delete said repository from the filesystem, but instead
renames it in a special way so that it is not possible to push, clone
or access the repository.
There is a special command for cleaning up such archived repositories::
- gearbox cleanup-repos --older-than=30d -c my.ini
+ kallithea-cli repo-purge-deleted --older-than=30d my.ini
This command scans for archived repositories that are older than
30 days, displays them, and asks if you want to delete them (unless given
-the ``--dont-ask`` flag). If you host a large amount of repositories with
+the ``--no-ask`` flag). If you host a large amount of repositories with
forks that are constantly being deleted, it is recommended that you run this
command via crontab.
diff --git a/kallithea/bin/kallithea_cli_repo.py b/kallithea/bin/kallithea_cli_repo.py
--- a/kallithea/bin/kallithea_cli_repo.py
+++ b/kallithea/bin/kallithea_cli_repo.py
@@ -23,9 +23,14 @@ import click
from kallithea.bin.kallithea_cli_base import cli
import kallithea.bin.kallithea_cli_util as cli_util
-from kallithea.lib.utils import repo2db_mapper
-from kallithea.lib.utils2 import safe_unicode
-from kallithea.model.db import Repository
+import datetime
+import os
+import re
+import shutil
+
+from kallithea.lib.utils import repo2db_mapper, REMOVED_REPO_PAT
+from kallithea.lib.utils2 import safe_unicode, safe_str
+from kallithea.model.db import Repository, Ui
from kallithea.model.meta import Session
from kallithea.model.scm import ScmModel
@@ -86,3 +91,90 @@ def repo_update_cache(config_file, repos
click.echo('Updated repository cache for following %s repositories' % (len(repo_list)))
click.echo('\n'.join(repo.repo_name for repo in repo_list))
+
+ at cli.command()
+ at cli_util.auto_setup_app()
+ at click.option('--ask/--no-ask', default=True, help='Ask for confirmation or not. Default is --ask.')
+ at click.option('--older-than',
+ help="""Only purge repositories that have been removed at least the given time ago.
+ For example, '--older-than=30d' purges repositories deleted 30 days ago or longer.
+ Possible suffixes: d (days), h (hours), m (minutes), s (seconds).""")
+def repo_purge_deleted(config_file, ask, older_than):
+ """Purge backups of deleted repositories.
+
+ When a repository is deleted via the Kallithea web interface, the actual
+ data is still present on the filesystem but set aside using a special name.
+ This command allows to delete these files permanently.
+ """
+ def _parse_older_than(val):
+ regex = re.compile(r'((?P<days>\d+?)d)?((?P<hours>\d+?)h)?((?P<minutes>\d+?)m)?((?P<seconds>\d+?)s)?')
+ parts = regex.match(val)
+ if not parts:
+ return
+ parts = parts.groupdict()
+ time_params = {}
+ for (name, param) in parts.iteritems():
+ if param:
+ time_params[name] = int(param)
+ return datetime.timedelta(**time_params)
+
+ def _extract_date(name):
+ """
+ Extract the date part from rm__<date> pattern of removed repos,
+ and convert it to datetime object
+
+ :param name:
+ """
+ date_part = name[4:19] # 4:19 since we don't parse milliseconds
+ return datetime.datetime.strptime(date_part, '%Y%m%d_%H%M%S')
+
+ repos_location = Ui.get_repos_location()
+ to_remove = []
+ for dn_, dirs, f in os.walk(safe_str(repos_location)):
+ alldirs = list(dirs)
+ del dirs[:]
+ if ('.hg' in alldirs or
+ '.git' in alldirs or
+ '.svn' in alldirs or
+ 'objects' in alldirs and ('refs' in alldirs or 'packed-refs' in f)):
+ continue
+ for loc in alldirs:
+ if REMOVED_REPO_PAT.match(loc):
+ to_remove.append([os.path.join(dn_, loc),
+ _extract_date(loc)])
+ else:
+ dirs.append(loc)
+ if dirs:
+ click.echo('Scanning: %s' % dn_)
+
+ # filter older than (if present)!
+ now = datetime.datetime.now()
+ if older_than:
+ to_remove_filtered = []
+ older_than_date = _parse_older_than(older_than)
+ for name, date_ in to_remove:
+ repo_age = now - date_
+ if repo_age > older_than_date:
+ to_remove_filtered.append([name, date_])
+
+ to_remove = to_remove_filtered
+ click.echo('Purging %s deleted repos older than %s (%s)'
+ % (len(to_remove), older_than, older_than_date))
+ else:
+ click.echo('Purging all %s deleted repos' % len(to_remove))
+
+ if not ask or not to_remove:
+ # don't ask just remove !
+ remove = True
+ else:
+ remove = cli_util.ask_ok('The following repositories will be removed completely:\n%s\n'
+ 'Do you want to proceed? [y/n] '
+ % '\n'.join(['%s deleted on %s' % (safe_str(x[0]), safe_str(x[1]))
+ for x in to_remove]))
+
+ if remove:
+ for path, date_ in to_remove:
+ click.echo('Purging repository %s' % path)
+ shutil.rmtree(path)
+ else:
+ click.echo('Nothing done, exiting...')
diff --git a/kallithea/bin/kallithea_cli_util.py b/kallithea/bin/kallithea_cli_util.py
--- a/kallithea/bin/kallithea_cli_util.py
+++ b/kallithea/bin/kallithea_cli_util.py
@@ -56,3 +56,15 @@ def full_cmd_name():
cmd.insert(0, ctx.parent.info_name)
ctx = ctx.parent
return ' '.join(cmd)
+
+def ask_ok(prompt, retries=4, complaint='Yes or no please!'):
+ while True:
+ ok = raw_input(prompt)
+ if ok in ('y', 'ye', 'yes'):
+ return True
+ if ok in ('n', 'no', 'nop', 'nope'):
+ return False
+ retries = retries - 1
+ if retries < 0:
+ raise IOError
+ click.echo(complaint)
diff --git a/kallithea/lib/paster_commands/cleanup.py b/kallithea/lib/paster_commands/cleanup.py
deleted file mode 100644
--- a/kallithea/lib/paster_commands/cleanup.py
+++ /dev/null
@@ -1,141 +0,0 @@
-# -*- coding: utf-8 -*-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-"""
-kallithea.lib.paster_commands.cleanup
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-cleanup-repos gearbox command for Kallithea
-
-
-This file was forked by the Kallithea project in July 2014.
-Original author and date, and relevant copyright and licensing information is below:
-:created_on: Jul 14, 2012
-:author: marcink
-:copyright: (c) 2013 RhodeCode GmbH, and others.
-:license: GPLv3, see LICENSE.md for more details.
-"""
-
-
-import os
-import re
-import shutil
-import datetime
-
-from kallithea.lib.paster_commands.common import ask_ok, BasePasterCommand
-from kallithea.lib.utils import REMOVED_REPO_PAT
-from kallithea.lib.utils2 import safe_str
-from kallithea.model.db import Ui
-
-
-class Command(BasePasterCommand):
- """Kallithea: Cleanup of backup files of deleted repositories"""
-
- def _parse_older_than(self, val):
- regex = re.compile(r'((?P<days>\d+?)d)?((?P<hours>\d+?)h)?((?P<minutes>\d+?)m)?((?P<seconds>\d+?)s)?')
- parts = regex.match(val)
- if not parts:
- return
- parts = parts.groupdict()
- time_params = {}
- for (name, param) in parts.iteritems():
- if param:
- time_params[name] = int(param)
- return datetime.timedelta(**time_params)
-
- def _extract_date(self, name):
- """
- Extract the date part from rm__<date> pattern of removed repos,
- and convert it to datetime object
-
- :param name:
- """
- date_part = name[4:19] # 4:19 since we don't parse milliseconds
- return datetime.datetime.strptime(date_part, '%Y%m%d_%H%M%S')
-
- def take_action(self, args):
- repos_location = Ui.get_repos_location()
- to_remove = []
- for dn_, dirs, f in os.walk(safe_str(repos_location)):
- alldirs = list(dirs)
- del dirs[:]
- if ('.hg' in alldirs or
- '.git' in alldirs or
- '.svn' in alldirs or
- 'objects' in alldirs and ('refs' in alldirs or 'packed-refs' in f)):
- continue
- for loc in alldirs:
- if REMOVED_REPO_PAT.match(loc):
- to_remove.append([os.path.join(dn_, loc),
- self._extract_date(loc)])
- else:
- dirs.append(loc)
- if dirs:
- print 'Scanning: %s' % dn_
-
- # filter older than (if present)!
- now = datetime.datetime.now()
- older_than = args.older_than
- if older_than:
- to_remove_filtered = []
- older_than_date = self._parse_older_than(older_than)
- for name, date_ in to_remove:
- repo_age = now - date_
- if repo_age > older_than_date:
- to_remove_filtered.append([name, date_])
-
- to_remove = to_remove_filtered
- print 'Removing %s deleted repos older than %s (%s)' \
- % (len(to_remove), older_than, older_than_date)
- else:
- print 'Removing all %s deleted repos' % len(to_remove)
- if args.dont_ask or not to_remove:
- # don't ask just remove !
- remove = True
- else:
- remove = ask_ok('the following repositories will be deleted completely:\n%s\n'
- 'are you sure you want to remove them [y/n]?'
- % '\n'.join(['%s removed on %s' % (safe_str(x[0]), safe_str(x[1]))
- for x in to_remove]))
-
- if remove:
- for path, date_ in to_remove:
- print 'Removing repository %s' % path
- shutil.rmtree(path)
- else:
- print 'Nothing done, exiting...'
-
- def get_parser(self, prog_name):
- parser = super(Command, self).get_parser(prog_name)
-
- parser.add_argument(
- '--older-than',
- action='store',
- dest='older_than',
- help=("only remove repos that have been removed "
- "at least given time ago. "
- "The default is to remove all removed repositories. "
- "Possible suffixes: "
- "d (days), h (hours), m (minutes), s (seconds). "
- "For example --older-than=30d deletes repositories "
- "removed more than 30 days ago.")
- )
-
- parser.add_argument(
- '--dont-ask',
- action="store_true",
- dest="dont_ask",
- help="remove repositories without asking for confirmation."
- )
-
- return parser
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -158,7 +158,6 @@ setuptools.setup(
main = kallithea.config.middleware:make_app
[gearbox.commands]
- cleanup-repos=kallithea.lib.paster_commands.cleanup:Command
install-iis=kallithea.lib.paster_commands.install_iis:Command
make-index=kallithea.lib.paster_commands.make_index:Command
make-rcext=kallithea.lib.paster_commands.make_rcextensions:Command
More information about the kallithea-general
mailing list