Issue #337: index creation fails on git repo with submodules in subdirectory (conservancy/kallithea)
Thomas De Schampheleire
issues-reply at bitbucket.org
Wed Mar 27 21:01:30 UTC 2019
New issue 337: index creation fails on git repo with submodules in subdirectory
https://bitbucket.org/conservancy/kallithea/issues/337/index-creation-fails-on-git-repo-with
Thomas De Schampheleire:
(reported on IRC by ayleph)
Steps to reproduce:
1. Clone git repo http://source.salmonlabs.net/dotfiles into kallithea 0.4.0rc1
2. `kallithea-cli index-create -c development.ini --index-only dotfiles`
Error:
```
Traceback (most recent call last):
File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/bin/kallithea-cli", line 11, in <module>
load_entry_point('Kallithea', 'console_scripts', 'kallithea-cli')()
File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/lib/python2.7/site-packages/click/core.py", line 764, in __call__
return self.main(*args, **kwargs)
File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/lib/python2.7/site-packages/click/core.py", line 717, in main
rv = self.invoke(ctx)
File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/lib/python2.7/site-packages/click/core.py", line 1137, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/lib/python2.7/site-packages/click/core.py", line 956, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/lib/python2.7/site-packages/click/core.py", line 555, in invoke
return callback(*args, **kwargs)
File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/bin/kallithea_cli_base.py", line 52, in runtime_wrapper
return annotated(*args, **kwargs)
File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/bin/kallithea_cli_index.py", line 59, in index_create
.run(full_index=full_index)
File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/indexers/daemon.py", line 461, in run
self.update_indexes()
File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/indexers/daemon.py", line 453, in update_indexes
self.update_file_index()
File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/indexers/daemon.py", line 394, in update_file_index
for path in self.get_paths(repo):
File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/indexers/daemon.py", line 136, in get_paths
for _topnode, _dirs, files in cs.walk('/'):
File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/backends/base.py", line 652, in walk
for tup in self.walk(dirnode.path):
File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/backends/base.py", line 652, in walk
for tup in self.walk(dirnode.path):
File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/backends/base.py", line 650, in walk
yield (topnode, topnode.dirs, topnode.files)
File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/utils/lazy.py", line 44, in __get__
value = self._func(obj)
File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/nodes.py", line 511, in dirs
return sorted((node for node in self.nodes if node.is_dir()))
File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/utils/lazy.py", line 44, in __get__
value = self._func(obj)
File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/nodes.py", line 499, in nodes
nodes = self.changeset.get_nodes(self.path)
File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/backends/git/changeset.py", line 418, in get_nodes
cf = ConfigFile.from_file(BytesIO(self.repository._repo.get_object(tree['.gitmodules'][1]).data))
File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/lib/python2.7/site-packages/dulwich/objects.py", line 936, in __getitem__
return self._entries[name]
KeyError: '.gitmodules'
```
Investigation of the problem yields to the following info:
- The root of the repo does contain a .gitmodules file, with following contents:
```
[submodule "dircolors-solarized"]
path = dircolors-solarized
url = https://github.com/seebi/dircolors-solarized.git
[submodule "vim/bundle/vim-colors-solarized"]
path = vim/bundle/vim-colors-solarized
url = https://github.com/altercation/vim-colors-solarized
[submodule "vim/bundle/VOoM"]
path = vim/bundle/VOoM
url = https://github.com/vim-voom/VOoM.git
[submodule "mintty-colors-solarized"]
path = mintty-colors-solarized
url = https://github.com/karlin/mintty-colors-solarized
```
The problem happens with the VOoM submodule.
The code is traversing the files in the repo, determines the 'git tree' for it, then trying to read the .gitmodules file from that git tree.
The 'git tree' corresponds to a directory tree, and in the case of VOoM the 'tree' is rooted at 'vim/bundle', not at the root of the repo. As a result, in that subtree there is no '.gitmodules' file.
Following change will start from the root tree:
```
diff --git a/kallithea/lib/vcs/backends/git/changeset.py b/kallithea/lib/vcs/backends/git/changeset.py
--- a/kallithea/lib/vcs/backends/git/changeset.py
+++ b/kallithea/lib/vcs/backends/git/changeset.py
@@ -411,7 +412,8 @@ class GitChangeset(BaseChangeset):
als = self.repository.alias
for name, stat, id in tree.iteritems():
if objects.S_ISGITLINK(stat):
- cf = ConfigFile.from_file(BytesIO(self.repository._repo.get_object(tree['.gitmodules'][1]).data))
+ root_tree = self.repository._repo[self._tree_id]
+ cf = ConfigFile.from_file(BytesIO(self.repository._repo.get_object(root_tree['.gitmodules'][1]).data))
url = cf.get(('submodule', name), 'url')
dirnodes.append(SubModuleNode(name, url=url, changeset=id,
alias=als))
```
and then the error becomes:
```
Traceback (most recent call last):
File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/bin/kallithea-cli", line 11, in <module>
load_entry_point('Kallithea', 'console_scripts', 'kallithea-cli')()
File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/lib/python2.7/site-packages/click/core.py", line 764, in __call__
return self.main(*args, **kwargs)
File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/lib/python2.7/site-packages/click/core.py", line 717, in main
rv = self.invoke(ctx)
File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/lib/python2.7/site-packages/click/core.py", line 1137, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/lib/python2.7/site-packages/click/core.py", line 956, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/lib/python2.7/site-packages/click/core.py", line 555, in invoke
return callback(*args, **kwargs)
File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/bin/kallithea_cli_base.py", line 52, in runtime_wrapper
return annotated(*args, **kwargs)
File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/bin/kallithea_cli_index.py", line 59, in index_create
.run(full_index=full_index)
File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/indexers/daemon.py", line 461, in run
self.update_indexes()
File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/indexers/daemon.py", line 453, in update_indexes
self.update_file_index()
File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/indexers/daemon.py", line 394, in update_file_index
for path in self.get_paths(repo):
File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/indexers/daemon.py", line 136, in get_paths
for _topnode, _dirs, files in cs.walk('/'):
File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/backends/base.py", line 652, in walk
for tup in self.walk(dirnode.path):
File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/backends/base.py", line 652, in walk
for tup in self.walk(dirnode.path):
File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/backends/base.py", line 650, in walk
yield (topnode, topnode.dirs, topnode.files)
File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/utils/lazy.py", line 44, in __get__
value = self._func(obj)
File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/nodes.py", line 511, in dirs
return sorted((node for node in self.nodes if node.is_dir()))
File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/utils/lazy.py", line 44, in __get__
value = self._func(obj)
File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/nodes.py", line 499, in nodes
nodes = self.changeset.get_nodes(self.path)
File "/home/tdescham/repo/contrib/kallithea/kallithea-incoming/kallithea/lib/vcs/backends/git/changeset.py", line 417, in get_nodes
url = cf.get(('submodule', name), 'url')
File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/lib/python2.7/site-packages/dulwich/config.py", line 240, in get
return self._values[(section[0],)][name]
File "/home/tdescham/repo/contrib/kallithea/venv/kallithea-incoming/lib/python2.7/site-packages/dulwich/config.py", line 87, in __getitem__
return super(CaseInsensitiveDict, self).__getitem__(key)
KeyError: ('submodule',)
```
We now found the .gitmodules file, but we try to find the submodule called 'VOoM' (based on the directory name) which does not exist.
More changes are thus necessary to correctly map a file path on the right entry in .gitmodules...
More information about the kallithea-general
mailing list