gitlab迁移到kubernetes集群

2,911次阅读

没有评论

之前公司代码管理采用的 docker compose 搭建的 gitlab 管理。为完成资源优化，高可用。现在将环境迁移到 kubernetes 集群环境中

内容隐藏

2.3.2 pgsql-deploy-svc

2.4 gitlab 搭建

2.4.1 gitlab-pvc

2.4.2 gitlab-deploy-svc

原有配置

cat > docker-compose.yaml << EOF
version: "3"
 
services:
  gitlab:
    image: 'gitlab/gitlab-ce:14.1.5-ce.0'
    container_name: gitlab-ce
    restart: always
    #network_mode: host
    hostname: 'xxxxxxxxx.com'
    environment:
      GITLAB_OMNIBUS_CONFIG: |
        # Add any other gitlab.rb configuration here, each on its own line
 
        external_url 'https://xxxxxxxx.com/'
 
        gitlab_rails['time_zone'] = 'Asia/Shanghai'
        gitlab_rails['smtp_enable'] = true
        gitlab_rails['smtp_address'] = "xxxxxx.net"
        gitlab_rails['smtp_port'] = 587
        gitlab_rails['smtp_user_name'] = "ldap_mail"
        gitlab_rails['smtp_password'] = "xxxxxxxx"
        gitlab_rails['smtp_authentication'] = "login"
 
        gitlab_rails['gitlab_email_from'] = 'xxxxx@xxxx.com'
        nginx['enable'] = true
        nginx['redirect_http_to_https'] = true
        nginx['ssl_certificate'] = '/etc/gitlab/ssl/xxxxxx.pem'
        nginx['ssl_certificate_key'] = '/etc/gitlab/ssl/xxxxxx.key'
 
        gitlab_rails['ldap_enabled'] = true
        gitlab_rails['prevent_ldap_sign_in'] = false
        gitlab_rails['ldap_servers'] = YAML.load <<-'EOS'
          main: # 'main' is the GitLab 'provider ID' of this LDAP server
            label: 'LDAP'
            host: 'xxxxxx.com'
            port: 389
            uid: 'sAMAccountName'
            bind_dn: 'xxxxxxx'
            password: 'xxxxxxx'
            encryption: 'plain' # "start_tls" or "simple_tls" or "plain"
            verify_certificates: false
            allow_username_or_email_login: true
            block_auto_created_users: false
            active_directory: false
            user_filter: '(xxxxxxxxx)'
            base: 'xxxxxxxxxxx'
            attributes:
              username: ['sAMAccountName']
              email: ['mail']
              name: 'givenName'
              #first_name: 'givenName'
 
            #smartcard_auth: false
            #lowercase_usernames: false
        EOS
 
        gitlab_rails['rack_attack_git_basic_auth'] = {
         'enabled' => false,
         'ip_whitelist' => ["127.0.0.1", "xxxxxxxxx"],
         'maxretry' => 1000000,
         'findtime' => 60,
         'bantime' => 0
        }
 
        node_exporter['enable'] = false
        prometheus_monitoring['enable'] = false
        registry_external_url 'https://xxxxxxxxxxxx.com'
        gitlab_rails['registry_enabled'] = true
        registry_nginx['ssl_certificate'] = "/etc/gitlab/ssl/xxxxx.pem"
        registry_nginx['ssl_certificate_key'] = "/etc/gitlab/ssl/xxxxxx.key"
 
        gitlab_rails['omniauth_enabled'] = false
 
        prometheus_monitoring['enable'] = true
        node_exporter['enable'] = true
        grafana['enable'] = true
 
 
    ports:
      - '80:80'
      - '443:443'
      - '22:22'
    volumes:
      - './config:/etc/gitlab'
      - './ssl:/etc/gitlab/ssl'
      - './logs:/var/log/gitlab'
      - './data:/var/opt/gitlab'
EOF

gitlab 主要涉及 Redis、Postgresql、Gitlab 三个应用，现在基于 kubernetes 先搭建 gitlab 初始化环境：

环境搭建

前期准备

共享存储

个人存储使用了 rook 的 ceph，所以持久化存储都采用了 rook ceph.

postgreSQL

这里采用使用的 kuberenetes 搭建 pgsql，也可使用现有的

Redis 搭建

pvc 声明

cat > redis-pvc.yaml << EOF
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: gitlab-redis
  namespace: devops
  labels:
    app: redis
spec:
  storageClassName: gitlab-storage
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi
EOF
kubectl apply -f redis-pvc.yaml

deploy-svc

cat > redis-deploy-svc.yaml << EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: redis
  namespace: devops
  labels:
    name: redis
spec:
  selector:
    matchLabels:
      name: redis
  template:
    metadata:
      name: redis
      labels:
        name: redis
    spec:
      containers:
      - name: redis
        image: sameersbn/redis
        imagePullPolicy: IfNotPresent
        ports:
        - name: redis
          containerPort: 6379
        volumeMounts:
        - mountPath: /var/lib/redis
          name: gitlab-redis
        livenessProbe:
          exec:
            command:
            - redis-cli
            - ping
          initialDelaySeconds: 30
          timeoutSeconds: 5
        readinessProbe:
          exec:
            command:
            - redis-cli
            - ping
          initialDelaySeconds: 5
          timeoutSeconds: 1
      volumes:
      - name: gitlab-redis
        persistentVolumeClaim:
          claimName: gitlab-redis
---
apiVersion: v1
kind: Service
metadata:
  name: redis
  namespace: devops
  labels:
    name: redis
spec:
  ports:
    - name: redis
      port: 6379
      targetPort: redis
  selector:
    name: redis
EOF
kubectl apply -f redis-deploy-svc.yaml

postgreSQL 搭建

pgsql-pvc

cat > postgresql-pvc.yaml << EOF
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: gitlab-postgresql
  namespace: devops
  labels:
    name: postgresql
spec:
  storageClassName: gitlab-storage
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi
EOF

pgsql-deploy-svc

cat > postgresql-deploy-svc.yaml << EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: postgresql
  namespace: devops
  labels:
    name: postgresql
spec:
spec:
  selector:
    matchLabels:
      name: postgresql
  template:
    metadata:
      name: postgresql
      labels:
        name: postgresql
    spec:
      containers:
      - name: postgresql
        image: postgres:12.13
        imagePullPolicy: IfNotPresent
        env:
        - name: DB_USER
          value: gitlab
        - name: DB_PASS
          value: passw0rd
        - name: DB_NAME
          value: gitlab_production
        - name: DB_EXTENSION
          value: pg_trgm
        ports:
        - name: postgres
          containerPort: 5432
        volumeMounts:
        - mountPath: /var/lib/postgresql
          name: gitlab-postgresql
        livenessProbe:
          exec:
            command:
            - pg_isready
            - -h
            - localhost
            - -U
            - postgres
          initialDelaySeconds: 30
          timeoutSeconds: 5
        readinessProbe:
          exec:
            command:
            - pg_isready
            - -h
            - localhost
            - -U
            - postgres
          initialDelaySeconds: 5
          timeoutSeconds: 1
      volumes:
      - name: gitlab-postgresql
        persistentVolumeClaim:
          claimName: gitlab-storage

---
apiVersion: v1
kind: Service
metadata:
  name: postgresql
  namespace: devops
  labels:
    name: postgresql
spec:
  ports:
    - name: postgres
      port: 5432
      targetPort: postgres
  selector:
    name: postgresql
EOF

gitlab 搭建

gitlab-pvc

cat > gitlab-pvc.yaml << EOF
# gitlab-config
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: gitlab-config
  labels:
    app: gitlab
  namespace: devops
spec:
  storageClassName: gitlab-storage
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 5Gi
---
# gitlab-data
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: gitlab-data
  labels:
    app: gitlab
  namespace: devops
spec:
  storageClassName: gitlab-storage
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 200Gi
EOF

注意：这里采用挂在配置目录的方式，所以需要将 gitlab.rb 拷贝进去，gitlab.rb 配置如下：

external_url 'http://xxxxxxxx.com/'
gitlab_rails['time_zone'] = 'Asia/Shanghai'
gitlab_rails['smtp_enable'] = true
gitlab_rails['smtp_address'] = "xxxxxxxxx"
gitlab_rails['smtp_port'] = 587
gitlab_rails['smtp_user_name'] = "ldap_mail"
gitlab_rails['smtp_password'] = "xxxxxxxx"
gitlab_rails['smtp_authentication'] = "login"
gitlab_rails['gitlab_email_from'] = 'xxxxxxxxx'
gitlab_rails['ldap_enabled'] = true
gitlab_rails['prevent_ldap_sign_in'] = false
gitlab_rails['ldap_servers'] = YAML.load <<-'EOS'
    label: 'LDAP'
    host: 'xxxxxxxx'
    port: 389
    uid: 'sAMAccountName'
    bind_dn: 'xxxxxxxx'
    password: 'xxxxxxxx'
    verify_certificates: false
    active_directory: false
    allow_username_or_email_login: true
    block_auto_created_users: false
    base: 'xxxxxxxxx'
    user_filter: 'xxxxxxxxx'
    attributes:
      username: ['sAMAccountName']
      email: ['mail']
      name: 'givenName'
EOS
gitlab_rails['db_adapter'] = "postgresql"
gitlab_rails['db_encoding'] = "utf8"
gitlab_rails['db_database'] = "gitlabhq_production"
gitlab_rails['db_username'] = "gitlab"
gitlab_rails['db_password'] = "xxxxxxxxxx"
gitlab_rails['db_host'] = "postgresql"
gitlab_rails['db_port'] = 5432
gitlab_rails['redis_host'] = "redis"
gitlab_rails['redis_port'] = 6379
registry_external_url 'https://harbor.xxxxx.com'
gitlab_rails['registry_enabled'] = true
registry_nginx['ssl_certificate'] = "xxxxxx.pem"
registry_nginx['ssl_certificate_key'] = "xxxxxxxx.key"
#sidekiq['max_concurrency'] = 3
#sidekiq['min_concurrency'] = 1
postgresql['enable'] = true
redis['enable'] = true
node_exporter['enable'] = true
prometheus_monitoring['enable'] = true
grafana['enable'] = true
letsencrypt['enable'] = false

gitlab-deploy-svc

cat > gitlab-deploy-svc.yaml << EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: gitlab
  namespace: devops
  labels:
    name: gitlab
spec:
  selector:
    matchLabels:
      name: gitlab
  template:
    metadata:
      name: gitlab
      labels:
        name: gitlab
    spec:
      containers:
      - name: gitlab
        image: gitlab/gitlab-ce:14.1.5-ce.0
        imagePullPolicy: IfNotPresent
        ports:
        - name: http
          containerPort: 80
        - name: ssh
          containerPort: 22
        volumeMounts:
        - mountPath: /var/opt/gitlab
          name: gitlab-data
        - mountPath: /etc/gitlab
          name: gitlab-config
        livenessProbe:
          httpGet:
            path: /
            port: 80
          initialDelaySeconds: 180
          timeoutSeconds: 5
        readinessProbe:
          httpGet:
            path: /
            port: 80
          initialDelaySeconds: 5
          timeoutSeconds: 1
      volumes:
      - name: gitlab-data
        persistentVolumeClaim:
          claimName: gitlab-data
      - name: gitlab-config
        persistentVolumeClaim:
          claimName: gitlab-config
---
apiVersion: v1
kind: Service
metadata:
  name: gitlab
  namespace: devops
  labels:
    name: gitlab
spec:
  ports:
    - name: http
      port: 80
      targetPort: http
      nodePort: 30080
    - name: ssh
      port: 22
      targetPort: ssh
      nodePort: 30022
  type: NodePort
  selector:
    name: gitlab
EOF

(选填) 待所有服务都就绪后，给服务配置一个 ingress

cat > gitlab-ingress.yaml << EOF
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: gitlab-ingress
  namespace: devops
  annotations:
    kubernetes.io/ingress.class: "public-internal-nginx"
spec:
  rules:
    - host: xxxxxxx.com
      http:
        paths:
          - backend:
              service:
                name: gitlab
                port:
                  number: 80
            path: /
            pathType: ImplementationSpecific
  tls:
  - secretName: xxxxxxx
    hosts:
    - xxxxxx.com
EOF

或者使用公有云的 loadbalancer

正文完