Access to repository times out/hangs - From browser and from git client

[Timothy Harris]

We have a repository hosted on our Bitbucket server (8.9.4) that suddenly is not reachable.

All the other repos are reachable and seem fine. So the problem seems to be just this one. But its kinda an important one...

• When trying to reach it through the UI the proxy out front just times out and we get a 504. 
• Git client commands also just hang.

For example when doing a git pull it just hangs and the logs on the Bitbucker server show.

SSH - git-upload-pack '/nem/sitecore-configs.git' c.a.s.i.s.g.p.s.SshCommandExitHandler git upload-pack --strict --timeout=1795 /var/atlassian/application-data/bitbucket/shared/data/repositories/144: (Exit code: 0)

And nothing happens...

Anyone have any ideas? 

Also see a lot git upload-pack processes for exactly this problem repo.

bitbuck+ 21 7 0 Jan16 ? 00:00:00 \_ bash /opt/atlassian/bitbucket/bin/start-bitbucket.sh -fg
bitbuck+ 211 21 0 Jan16 ? 00:00:00 \_ bash /opt/atlassian/bitbucket/bin/_start-webapp.sh run
bitbuck+ 220 211 7 Jan16 ? 01:08:04 \_ /opt/java/openjdk/bin/java -classpath /opt/atlassian/bitbucket/app -Datlassian.standalone=BITBUCKET -Dbitbucket.home=/var/atlassian/application-data/bitbucket -Dbitbucket.install=/opt/atlassian/bitbucket -Xms4096m -Xmx8192m -XX:+UseG1
bitbuck+ 414 220 1 Jan16 ? 00:14:29 \_ /opt/java/openjdk/bin/java -Xms768m -Xmx768m -XX:+UseG1GC -Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8 -Djava.io.tmpdir=/var/atlassian/application-data/bitbucket/mesh/tmp -Dlayout.caches-dir=/var/atlassian/application-data/bitbu
bitbuck+ 79490 414 0 09:10 ? 00:00:00 \_ /usr/bin/git repack -A -d -l -n --unpack-unreachable=72.hours.ago
bitbuck+ 80141 414 0 09:15 ? 00:00:00 \_ /usr/bin/git upload-pack --strict --timeout=1795 /var/atlassian/application-data/bitbucket/shared/data/repositories/22
bitbuck+ 80194 414 0 09:16 ? 00:00:00 \_ /usr/bin/git http-backend /var/atlassian/application-data/bitbucket/shared/data/repositories/22
bitbuck+ 80195 80194 0 09:16 ? 00:00:00 | \_ /usr/lib/git-core/git-http-backend /var/atlassian/application-data/bitbucket/shared/data/repositories/22
bitbuck+ 80196 80195 0 09:16 ? 00:00:00 | \_ /usr/lib/git-core/git upload-pack --stateless-rpc .
bitbuck+ 80407 414 0 09:17 ? 00:00:00 \_ /usr/bin/git http-backend /var/atlassian/application-data/bitbucket/shared/data/repositories/22
bitbuck+ 80408 80407 0 09:17 ? 00:00:00 | \_ /usr/lib/git-core/git-http-backend /var/atlassian/application-data/bitbucket/shared/data/repositories/22
bitbuck+ 80409 80408 0 09:17 ? 00:00:00 | \_ /usr/lib/git-core/git upload-pack --stateless-rpc .
bitbuck+ 80543 414 0 09:18 ? 00:00:00 \_ /usr/bin/git http-backend /var/atlassian/application-data/bitbucket/shared/data/repositories/22
bitbuck+ 80544 80543 0 09:18 ? 00:00:00 | \_ /usr/lib/git-core/git-http-backend /var/atlassian/application-data/bitbucket/shared/data/repositories/22
bitbuck+ 80545 80544 0 09:18 ? 00:00:00 | \_ /usr/lib/git-core/git upload-pack --stateless-rpc .
bitbuck+ 81018 414 0 09:22 ? 00:00:00 \_ /usr/bin/git cat-file -t refs/heads/master:
bitbuck+ 81019 414 0 09:22 ? 00:00:00 \_ /usr/bin/git cat-file -t refs/heads/master:
bitbuck+ 81243 414 0 09:23 ? 00:00:00 \_ /usr/bin/git cat-file -t refs/heads/master:
bitbuck+ 81254 414 0 09:23 ? 00:00:00 \_ /usr/bin/git upload-pack --strict --timeout=1795 /var/atlassian/application-data/bitbucket/shared/data/repositories/22
bitbuck+ 81269 414 0 09:24 ? 00:00:00 \_ /usr/bin/git cat-file -t refs/heads/master:
bitbuck+ 81565 414 0 09:25 ? 00:00:00 \_ /usr/bin/git rev-list --format=%H%x02%P%x02%aN%x02%aE%x02%at%x02%cN%x02%cE%x02%ct --no-walk=unsorted 3536c3d0bb5b30c93490c80a6808fc9d9893e5a2 3536c3d0bb5b30c93490c80a6808fc9d9893e5a2^@ --
bitbuck+ 81603 414 0 09:25 ? 00:00:00 \_ /usr/bin/git rev-list --format=%H%x02%P%x02%aN%x02%aE%x02%at%x02%cN%x02%cE%x02%ct --topo-order --no-min-parents --stdin --do-walk --
bitbuck+ 81618 414 0 09:26 ? 00:00:00 \_ /usr/bin/git upload-pack --strict --timeout=1795 /var/atlassian/application-data/bitbucket/shared/data/repositories/22
bitbuck+ 81697 414 0 09:26 ? 00:00:00 \_ /usr/bin/git http-backend /var/atlassian/application-data/bitbucket/shared/data/repositories/22
bitbuck+ 81699 81697 0 09:26 ? 00:00:00 | \_ /usr/lib/git-core/git-http-backend /var/atlassian/application-data/bitbucket/shared/data/repositories/22
bitbuck+ 81701 81699 0 09:26 ? 00:00:00 | \_ /usr/lib/git-core/git upload-pack --stateless-rpc .
bitbuck+ 81730 414 0 09:26 ? 00:00:00 \_ /usr/bin/git rev-list --format=%H%x02%P%x02%aN%x02%aE%x02%at%x02%cN%x02%cE%x02%ct --topo-order --no-min-parents --stdin --do-walk --
bitbuck+ 81802 414 0 09:28 ? 00:00:00 \_ /usr/bin/git http-backend /var/atlassian/application-data/bitbucket/shared/data/repositories/22
bitbuck+ 81803 81802 0 09:28 ? 00:00:00 | \_ /usr/lib/git-core/git-http-backend /var/atlassian/application-data/bitbucket/shared/data/repositories/22
bitbuck+ 81804 81803 0 09:28 ? 00:00:00 | \_ /usr/lib/git-core/git upload-pack --stateless-rpc .
bitbuck+ 82012 414 0 09:29 ? 00:00:00 \_ /usr/bin/git upload-pack --strict --timeout=1795 /var/atlassian/application-data/bitbucket/shared/data/repositories/22
bitbuck+ 82130 414 0 09:31 ? 00:00:00 \_ /usr/bin/git http-backend /var/atlassian/application-data/bitbucket/shared/data/repositories/22
bitbuck+ 82131 82130 0 09:31 ? 00:00:00 | \_ /usr/lib/git-core/git-http-backend /var/atlassian/application-data/bitbucket/shared/data/repositories/22
bitbuck+ 82132 82131 0 09:31 ? 00:00:00 | \_ /usr/lib/git-core/git upload-pack --stateless-rpc .
bitbuck+ 82339 414 0 09:31 ? 00:00:00 \_ /usr/bin/git cat-file -t refs/heads/master:
bitbuck+ 82447 414 0 09:32 ? 00:00:00 \_ /usr/bin/git upload-pack --strict --timeout=1795 /var/atlassian/application-data/bitbucket/shared/data/repositories/22
bitbuck+ 82492 414 0 09:32 ? 00:00:00 \_ /usr/bin/git cat-file -t refs/heads/master:
bitbuck+ 82541 414 0 09:33 ? 00:00:00 \_ /usr/bin/git cat-file -t refs/heads/master:
bitbuck+ 82562 414 0 09:33 ? 00:00:00 \_ /usr/bin/git cat-file -t refs/heads/master:
bitbuck+ 82566 414 0 09:33 ? 00:00:00 \_ /usr/bin/git cat-file -t refs/heads/master:
bitbuck+ 82589 414 0 09:34 ? 00:00:00 \_ /usr/bin/git cat-file -t refs/heads/master:
bitbuck+ 82649 414 0 09:34 ? 00:00:00 \_ /usr/bin/git cat-file -t refs/heads/master:
bitbuck+ 82701 414 0 09:34 ? 00:00:00 \_ /usr/bin/git cat-file -t refs/heads/master:
bitbuck+ 82714 414 0 09:35 ? 00:00:00 \_ /usr/bin/git http-backend /var/atlassian/application-data/bitbucket/shared/data/repositories/22
bitbuck+ 82715 82714 0 09:35 ? 00:00:00 | \_ /usr/lib/git-core/git-http-backend /var/atlassian/application-data/bitbucket/shared/data/repositories/22
bitbuck+ 82716 82715 0 09:35 ? 00:00:00 | \_ /usr/lib/git-core/git upload-pack --stateless-rpc .
bitbuck+ 82735 414 0 09:35 ? 00:00:00 \_ /usr/bin/git cat-file -t refs/heads/master:
bitbuck+ 82805 414 0 09:35 ? 00:00:00 \_ /usr/bin/git cat-file -t refs/heads/master:
bitbuck+ 82822 414 0 09:35 ? 00:00:00 \_ /usr/bin/git cat-file -t refs/heads/master:
bitbuck+ 82925 414 0 09:36 ? 00:00:00 \_ /usr/bin/git http-backend /var/atlassian/application-data/bitbucket/shared/data/repositories/22
bitbuck+ 82927 82925 0 09:36 ? 00:00:00 | \_ /usr/lib/git-core/git-http-backend /var/atlassian/application-data/bitbucket/shared/data/repositories/22
bitbuck+ 82929 82927 0 09:36 ? 00:00:00 | \_ /usr/lib/git-core/git upload-pack --stateless-rpc .
bitbuck+ 82962 414 0 09:36 ? 00:00:00 \_ /usr/bin/git cat-file -t refs/heads/master:
bitbuck+ 82999 414 0 09:37 ? 00:00:00 \_ /usr/bin/git upload-pack --strict --timeout=1795 /var/atlassian/application-data/bitbucket/shared/data/repositories/22
bitbuck+ 83251 414 0 09:37 ? 00:00:00 \_ /usr/bin/git cat-file -t refs/heads/master:
bitbuck+ 83368 414 0 09:38 ? 00:00:00 \_ /usr/bin/git cat-file -t refs/heads/master:
bitbuck+ 83412 414 0 09:39 ? 00:00:00 \_ /usr/bin/git cat-file -t refs/heads/master:
bitbuck+ 83471 414 0 09:39 ? 00:00:00 \_ /usr/bin/git cat-file -t refs/heads/master:
bitbuck+ 83558 414 0 09:40 ? 00:00:00 \_ /usr/bin/git diff-tree -C -r --always --format=commit %H%n%H%x02%P%x02%aN%x02%aE%x02%at%x02%cN%x02%cE%x02%ct%n%B%n%x03END%x04 --root --stdin --
bitbuck+ 83567 414 0 09:40 ? 00:00:00 \_ /usr/bin/git cat-file -t refs/heads/master:
bitbuck+ 83596 414 0 09:40 ? 00:00:00 \_ /usr/bin/git http-backend /var/atlassian/application-data/bitbucket/shared/data/repositories/22
bitbuck+ 83597 83596 0 09:40 ? 00:00:00 | \_ /usr/lib/git-core/git-http-backend /var/atlassian/application-data/bitbucket/shared/data/repositories/22
bitbuck+ 83598 83597 0 09:40 ? 00:00:00 | \_ /usr/lib/git-core/git upload-pack --stateless-rpc .
bitbuck+ 83798 414 0 09:40 ? 00:00:00 \_ /usr/bin/git cat-file -t refs/heads/master:
bitbuck+ 83853 414 0 09:41 ? 00:00:00 \_ /usr/bin/git cat-file -t refs/heads/master:
bitbuck+ 83937 414 0 09:41 ? 00:00:00 \_ /usr/bin/git cat-file -t refs/heads/master: