gitlab报错503的情况很多,我记录一下这一次我的排错经历
首先,常规查看gitlab是哪个进程出错了
[root@gitlab ~]# gitlab-ctl status
run: alertmanager: (pid 1424) 3121s; run: log: (pid 1423) 3121s
down: gitaly: 0s, normally up, want up; run: log: (pid 1392) 3121s
run: gitlab-exporter: (pid 1410) 3121s; run: log: (pid 1409) 3121s
run: gitlab-workhorse: (pid 1405) 3121s; run: log: (pid 1403) 3121s
run: grafana: (pid 1422) 3121s; run: log: (pid 1421) 3121s
run: logrotate: (pid 1415) 3121s; run: log: (pid 1414) 3121s
run: nginx: (pid 1416) 3121s; run: log: (pid 1406) 3121s
run: node-exporter: (pid 1408) 3121s; run: log: (pid 1407) 3121s
run: postgres-exporter: (pid 1420) 3121s; run: log: (pid 1419) 3121s
run: postgresql: (pid 1404) 3121s; run: log: (pid 1402) 3121s
run: prometheus: (pid 1426) 3121s; run: log: (pid 1418) 3121s
run: redis: (pid 1390) 3121s; run: log: (pid 1388) 3121s
run: redis-exporter: (pid 1425) 3121s; run: log: (pid 1417) 3121s
run: sidekiq: (pid 1413) 3121s; run: log: (pid 1412) 3121s
run: unicorn: (pid 1395) 3121s; run: log: (pid 1394) 3121s
可以看到是gitaly挂了,那继续查gitaly的日志,查gitlab-shell没问题
[root@gitlab ~]# tail -f /var/log/gitlab/gitaly/gitlab-shell.log
{"method":"POST","url":"http://127.0.0.1:8080/api/v4/internal/allowed","duration":0.129717868,"gitaly_embedded":true,"pid":11574,"level":"info","msg":"finished HTTP request","time":"2019-12-08T22:07:42-05:00"}
{"method":"POST","url":"http://127.0.0.1:8080/api/v4/internal/pre_receive","duration":0.099115893,"gitaly_embedded":true,"pid":11574,"level":"info","msg":"finished HTTP request","time":"2019-12-08T22:07:42-05:00"}
{"method":"POST","url":"http://127.0.0.1:8080/api/v4/internal/post_receive","duration":0.070931364,"gitaly_embedded":true,"pid":11596,"level":"info","msg":"finished HTTP request","time":"2019-12-08T22:07:43-05:00"}
^C
继续,查到current的时候发现问题了
[root@gitlab ~]# cd /var/log/gitlab/gitaly/
[root@gitlab gitaly]# ll
total 22680
-rwxr--r-- 1 root root 2712 Dec 7 02:11 @400000005deb50b42308ae6c.s
-rwxr--r-- 1 root root 1311 Dec 8 02:12 @400000005deca25c2724495c.s
-rw-r--r-- 1 root root 9464 Dec 8 19:20 @400000005deda34c12795ccc.u
-rw-r--r-- 1 root root 148193 Dec 9 00:20 @400000005deddfb10aa81c7c.u
-rw-r--r-- 1 root root 1448 Dec 9 00:46 @400000005dedffdb23a8da54.u
-rw-r--r-- 1 root root 13167183 Dec 9 14:00 @400000005deef40c346c18ec.u
-rw-r--r-- 1 root root 1799877 Dec 9 21:51 @400000005def08f20691d4d4.u
-rwxr--r-- 1 root root 296533 Dec 15 20:31 @400000005df6de5f17d3b17c.s
-rw-r--r-- 1 root root 30 Dec 6 01:59 config
-rw-r--r-- 1 root root 1478904 Dec 15 21:42 current
-rw-r--r-- 1 git git 0 Dec 8 22:07 gitaly_hooks.log
-rw-r--r-- 1 git git 639 Dec 8 22:07 gitlab-shell.log
-rw------- 1 root root 0 Dec 6 01:59 lock
-rw-r--r-- 1 root root 0 Dec 15 20:31 state
[root@gitlab gitaly]# tail -f current
{"error":"strconv.Atoi: parsing \"\": invalid syntax","level":"fatal","msg":"find gitaly","time":"2019-12-16T02:42:17Z","wrapper":25383}
{"level":"info","msg":"Wrapper started","time":"2019-12-16T02:42:18Z","wrapper":25394}
{"level":"info","msg":"finding gitaly","pid_file":"/var/opt/gitlab/gitaly/gitaly.pid","time":"2019-12-16T02:42:18Z","wrapper":25394}
{"error":"strconv.Atoi: parsing \"\": invalid syntax","level":"fatal","msg":"find gitaly","time":"2019-12-16T02:42:18Z","wrapper":25394}
{"level":"info","msg":"Wrapper started","time":"2019-12-16T02:42:19Z","wrapper":25401}
{"level":"info","msg":"finding gitaly","pid_file":"/var/opt/gitlab/gitaly/gitaly.pid","time":"2019-12-16T02:42:19Z","wrapper":25401}
{"error":"strconv.Atoi: parsing \"\": invalid syntax","level":"fatal","msg":"find gitaly","time":"2019-12-16T02:42:19Z","wrapper":25401}
{"level":"info","msg":"Wrapper started","time":"2019-12-16T02:42:20Z","wrapper":25410}
{"level":"info","msg":"finding gitaly","pid_file":"/var/opt/gitlab/gitaly/gitaly.pid","time":"2019-12-16T02:42:20Z","wrapper":25410}
{"error":"strconv.Atoi: parsing \"\": invalid syntax","level":"fatal","msg":"find gitaly","time":"2019-12-16T02:42:20Z","wrapper":25410}
{"level":"info","msg":"Wrapper started","time":"2019-12-16T02:42:21Z","wrapper":25419}
{"level":"info","msg":"finding gitaly","pid_file":"/var/opt/gitlab/gitaly/gitaly.pid","time":"2019-12-16T02:42:21Z","wrapper":25419}
{"error":"strconv.Atoi: parsing \"\": invalid syntax","level":"fatal","msg":"find gitaly","time":"2019-12-16T02:42:21Z","wrapper":25419}
{"level":"info","msg":"Wrapper started","time":"2019-12-16T02:42:22Z","wrapper":25426}
{"level":"info","msg":"finding gitaly","pid_file":"/var/opt/gitlab/gitaly/gitaly.pid","time":"2019-12-16T02:42:22Z","wrapper":25426}
{"error":"strconv.Atoi: parsing \"\": invalid syntax","level":"fatal","msg":"find gitaly","time":"2019-12-16T02:42:22Z","wrapper":25426}
{"level":"info","msg":"Wrapper started","time":"2019-12-16T02:42:23Z","wrapper":25433}
{"level":"info","msg":"finding gitaly","pid_file":"/var/opt/gitlab/gitaly/gitaly.pid","time":"2019-12-16T02:42:23Z","wrapper":25433}
{"error":"strconv.Atoi: parsing \"\": invalid syntax","level":"fatal","msg":"find gitaly","time":"2019-12-16T02:42:23Z","wrapper":25433}
{"level":"info","msg":"Wrapper started","time":"2019-12-16T02:42:24Z","wrapper":25440}
{"level":"info","msg":"finding gitaly","pid_file":"/var/opt/gitlab/gitaly/gitaly.pid","time":"2019-12-16T02:42:24Z","wrapper":25440}
{"error":"strconv.Atoi: parsing \"\": invalid syntax","level":"fatal","msg":"find gitaly","time":"2019-12-16T02:42:24Z","wrapper":25440}
^C
看到报错没
{"error":"strconv.Atoi: parsing \"\": invalid syntax","level":"fatal","msg":"find gitaly","time":"2019-12-16T02:42:24Z","wrapper":25440}
搜索这个报错,都说删除这个进程对应的pid文件,照做
[root@gitlab gitaly]# cd /var/opt/gitlab/gitaly/
[root@gitlab gitaly]# ll
total 8
-rw-r----- 1 root git 781 Dec 6 01:59 config.toml
-rw------- 1 git git 0 Dec 9 00:46 gitaly.pid
srwxr-xr-x 1 git git 0 Dec 9 00:46 gitaly.socket
drwx------ 2 git root 6 Dec 6 01:59 internal_sockets
-rw-r--r-- 1 root root 23 Dec 6 01:59 VERSION
[root@gitlab gitaly]# rm -rf gitaly.pid
重启gitlab正常
[root@gitlab gitaly]# gitlab-ctl restart
ok: run: alertmanager: (pid 26304) 0s
ok: run: gitaly: (pid 26315) 0s
ok: run: gitlab-exporter: (pid 26334) 1s
ok: run: gitlab-workhorse: (pid 26337) 0s
ok: run: grafana: (pid 26356) 0s
ok: run: logrotate: (pid 26368) 1s
ok: run: nginx: (pid 26375) 0s
ok: run: node-exporter: (pid 26385) 1s
ok: run: postgres-exporter: (pid 26391) 0s
ok: run: postgresql: (pid 26477) 1s
ok: run: prometheus: (pid 26486) 0s
ok: run: redis: (pid 26497) 0s
ok: run: redis-exporter: (pid 26502) 1s
ok: run: sidekiq: (pid 26509) 1s
ok: run: unicorn: (pid 26520) 0s
再看一次,一切正常了
[root@gitlab gitaly]# gitlab-ctl status
run: alertmanager: (pid 26304) 14s; run: log: (pid 1423) 3337s
run: gitaly: (pid 26315) 13s; run: log: (pid 1392) 3337s
run: gitlab-exporter: (pid 26334) 13s; run: log: (pid 1409) 3337s
run: gitlab-workhorse: (pid 26337) 12s; run: log: (pid 1403) 3337s
run: grafana: (pid 26356) 12s; run: log: (pid 1421) 3337s
run: logrotate: (pid 26368) 12s; run: log: (pid 1414) 3337s
run: nginx: (pid 26375) 12s; run: log: (pid 1406) 3338s
run: node-exporter: (pid 26385) 12s; run: log: (pid 1407) 3338s
run: postgres-exporter: (pid 26391) 11s; run: log: (pid 1419) 3338s
run: postgresql: (pid 26477) 11s; run: log: (pid 1402) 3338s
run: prometheus: (pid 26486) 10s; run: log: (pid 1418) 3338s
run: redis: (pid 26497) 10s; run: log: (pid 1388) 3338s
run: redis-exporter: (pid 26502) 10s; run: log: (pid 1417) 3338s
run: sidekiq: (pid 26509) 9s; run: log: (pid 1412) 3338s
run: unicorn: (pid 26520) 7s; run: log: (pid 1394) 3338s