Cloud runners timeout with size 4x/8x

[Peter Meijer]

Hello,

We are trying to get the 4x / 8x size runners working. But every time we enable them for a runner the pipeline simply ends in a timeout. We are running the pipelines in the bitbucket cloud. We have a standard product plan, so we should be able to use this option.

Can somebody please explain what the issue is or how to debug this problem? I have been stripping the bitbucket-pipelines.yml config in order to test but no luck so far.. Without any output its hard to see what is causing the issue. 

If you need more information, please let me know.

Thanks in advance!

This is our configuration that is failing:

bitbucket-pipelines.yml

image: thecodingmachine/php:8.3-v4-apache-node14

clone:
  depth: full

definitions:
  services:
    mysql:
      image: mysql:8.0
      environment:
        MYSQL_DATABASE: 'database'
        MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
        MYSQL_USER: 'homestead'
        MYSQL_PASSWORD: 'secret'
    docker-extended-memory:
      memory: 5120
      type: docker
    docker:
      memory: 2048
  caches:
    npm: .npm-cache
    sonar: ~/.sonar/cache
    pint: .pint
  steps:
    - parallel: &common_steps_build
        fail-fast: true
        steps:
          - step:
              name: PHP Build
              script:
                - php -v
                - php -m
                - cp .env.pipelines .env
                - composer install --prefer-dist --no-ansi --no-interaction --no-progress --no-plugins
              services:
                - mysql
              artifacts:
                - .env
                - vendor/**
              caches:
                - composer
          - step:
              name: NPM Build
              script:
                - npm --version
                - npm ci --cache .npm-cache
                - npm run production
              artifacts:
                - public/**
              caches:
                - npm

    - parallel: &common_steps_test
        fail-fast: true
        steps:
          - step:
              name: PHP Test
              script:
                - php artisan migrate:fresh --seed
                - ./vendor/bin/phpunit -c ./phpunit.xml --coverage-clover phpunit.coverage.xml --log-junit phpunit.report.xml -d memory_limit=-1  ./tests/Unit
              artifacts:
                - storage/logs/**
                - phpunit.coverage.xml
                - phpunit.report.xml
              services:
                - mysql
                - docker-extended-memory
              size: 8x
              max-time: 15
          - step:
              name: PHP Check code style
              script:
                - git config --global --add safe.directory /opt/atlassian/pipelines/agent/build
                - ./vendor/bin/pint --test
              caches:
                - pint
          - step:
              name: PHP Analyze
              script:
                - ./vendor/bin/phpstan analyse --no-progress --memory-limit=4G --configuration phpstan.bitbucket.neon

    - parallel: &common_steps_scan
        fail-fast: true
        steps:
          - step:
              services: [ docker-extended-memory ]
              size: 8x
              caches:
                - sonar
              name: SonarCloud Scan
              script:
                - pipe: sonarsource/sonarcloud-scan:2.0.0
                  #- pipe: sonarsource/sonarcloud-quality-gate:0.1.6
                  variables:
                    SONAR_TOKEN: ${SONAR_TOKEN}
          - step:
              name: Dependency Check
              script:
                - composer validate
                - composer audit
                - npm audit || true

    - step: &build-docker-image
        name: Docker Build image
        image: amazon/aws-cli:latest
        services:
          - docker
        caches:
          - docker
        script:
          - yum install --assumeyes jq
          - source bin/bitbucket_build_docker_image.bash --ecr-repo hc-app-shared --dockerfile Dockerfile --target prod
          - echo "Pushing to Shared ECR"
          - pipe: atlassian/aws-ecr-push-image:2.2.0
            variables:
              AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID_SHARED
              AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY_SHARED
              AWS_DEFAULT_REGION: $AWS_REGION
              IMAGE_NAME: hc-app-shared
              TAGS: $TAGS_INLINE
          - echo "Image has been built and uploaded to Shared ECR successfully. Image = hc-app-shared:$BITBUCKET_BUILD_NUMBER"

    - step: &build-docker-base-image
        name: Docker Build base image
        image: amazon/aws-cli:latest
        services:
          - docker
        caches:
          - docker
        script:
          - yum install --assumeyes jq
          - source bin/bitbucket_build_docker_image.bash --ecr-repo hc-app-base-image-shared --dockerfile Dockerfile --target base_image
          - echo "Pushing to Shared ECR"
          - pipe: atlassian/aws-ecr-push-image:2.2.0
            variables:
              AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID_SHARED
              AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY_SHARED
              AWS_DEFAULT_REGION: $AWS_REGION
              IMAGE_NAME: hc-app-base-image-shared
              TAGS: $TAGS_INLINE
          - echo "Image has been built and uploaded to Shared ECR successfully. Image = hc-app-base-image-shared:$BITBUCKET_BUILD_NUMBER"

    - step: &deploy-docker-image
        name: Push and Deploy Docker Image
        image: amazon/aws-cli:latest
        services:
          - docker
        caches:
          - docker
        script:
          - yum install --assumeyes jq
          - source bin/bitbucket_pull_image_from_shared_account.bash
          - echo "Pushing to Target ECR, Tags=$TAGS_INLINE"
          - pipe: atlassian/aws-ecr-push-image:2.2.0
            variables:
              AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID
              AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY
              AWS_DEFAULT_REGION: $AWS_REGION
              IMAGE_NAME: hc-app-$DEPLOY_ENV
              TAGS: $TAGS_INLINE
          - echo "Image has been uploaded to $DEPLOY_ENV ECR successfully. Image = hc-app-$DEPLOY_ENV"
          - pipe: atlassian/slack-notify:2.2.0
            variables:
              WEBHOOK_URL: $SLACK_NOTIFY_WEBHOOK_URL
              MESSAGE: 'Updating app ENV ${DEPLOY_ENV} BRANCH ${BITBUCKET_BRANCH} DOWN ${ENABLE_MAINTENANCE_MODE} MIGRATE ${RUN_MIGRATIONS}'
          - bin/oh_dear_maintenance.bash $OHDEAR_TOKEN $OHDEAR_URL start
          - bin/aws_ecs_enable_maintenance_mode.bash $DEPLOY_ENV
          - bin/aws_ecs_run_migrations_fresh_seed.bash $DEPLOY_ENV
          - bin/aws_ecs_run_migrations.bash $DEPLOY_ENV
          - echo "Updating ECS services"
          - pipe: atlassian/aws-ecs-deploy:1.11.0
            variables:
              AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID
              AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY
              AWS_DEFAULT_REGION: $AWS_REGION
              CLUSTER_NAME: hc-$DEPLOY_ENV
              SERVICE_NAME: hc-app-$DEPLOY_ENV
              FORCE_NEW_DEPLOYMENT: 'true'
          - pipe: atlassian/aws-ecs-deploy:1.11.0
            variables:
              AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID
              AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY
              AWS_DEFAULT_REGION: $AWS_REGION
              CLUSTER_NAME: hc-$DEPLOY_ENV
              SERVICE_NAME: hc-worker-$DEPLOY_ENV
              FORCE_NEW_DEPLOYMENT: 'true'
          - pipe: atlassian/aws-ecs-deploy:1.11.0
            variables:
              AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID
              AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY
              AWS_DEFAULT_REGION: $AWS_REGION
              CLUSTER_NAME: hc-$DEPLOY_ENV
              SERVICE_NAME: hc-scheduled-tasks-$DEPLOY_ENV
              FORCE_NEW_DEPLOYMENT: 'true'
          - bin/aws_ecs_disable_maintenance_mode.bash $DEPLOY_ENV
# needs more time, defaults to 1 hour - bin/oh_dear_maintenance.bash $OHDEAR_TOKEN $OHDEAR_URL stop

    - step: &create-temp-ondemand-environment
        name: Create temporary ondemand environment
        image: amazon/aws-cli:latest
        script:
          - yum install --assumeyes jq
          - source bin/bitbucket_create_od_env.bash
        artifacts:
          - ARTIFACT_ENV_VARS.sh

    - step: &destroy-temp-ondemand-environment
        max-time: 15
        name: Destroy temporary ondemand environment
        image: atlassian/default-image:4
        script:
          - source bin/bitbucket_destroy_od_env.bash

pipelines:
  branches:
    'master':
      - parallel: *common_steps_build
      - parallel: *common_steps_test
      - step: *build-docker-image
    'develop':
      - parallel: *common_steps_build
      - parallel: *common_steps_test
      - parallel: *common_steps_scan
      - step: *destroy-temp-ondemand-environment
    '{versions/test}':
      - parallel: *common_steps_build
      - parallel: *common_steps_test
      - parallel: *common_steps_scan
      - step: *build-docker-image
      - step:
          <<: *deploy-docker-image
          deployment: test
          trigger: manual
          name: Push and Deploy Docker Image (test)
    '{versions/demo}':
      - parallel: *common_steps_build
      - parallel: *common_steps_test
      - parallel: *common_steps_scan
      - step: *build-docker-image
      - step:
          <<: *deploy-docker-image
          deployment: demo
          name: Push and Deploy Docker Image (demo)
    '{versions/acc}':
      - parallel: *common_steps_build
      - parallel: *common_steps_test
      - parallel: *common_steps_scan
      - step: *build-docker-image
      - step:
          <<: *deploy-docker-image
          trigger: manual
          deployment: accept
          name: Push and Deploy Docker Image (accept)
    '**-od-**':
      - parallel: *common_steps_build
      - parallel: *common_steps_test
      - parallel: *common_steps_scan
      - step:
          <<: *create-temp-ondemand-environment
          name: Create env (ondemand short lived)
      - step: *build-docker-image
      - step:
          <<: *deploy-docker-image
          name: Push and Deploy Docker Image (ondemand short lived)
  pull-requests:
    'hotfix/*':
      - parallel: *common_steps_build
      - parallel: *common_steps_test
      - parallel: *common_steps_scan
    'feature/sc-*':
      - parallel: *common_steps_build
      - parallel: *common_steps_test
      - parallel: *common_steps_scan
  tags:
    '*':
      - step: *build-docker-image

  custom:

    build-docker-image:
      - step: *build-docker-image

    build-test-scan:
      - parallel: *common_steps_build
      - parallel: *common_steps_test
      - parallel: *common_steps_scan

    deploy-to-test:
      - variables:
          - name: IMAGE_TAG
            default: "latest"
          - name: RUN_MIGRATIONS
            default: "true"
            allowed-values:
              - "true"
              - "false"
          - name: ENABLE_MAINTENANCE_MODE
            default: "true"
            allowed-values:
              - "true"
              - "false"
      - step:
          <<: *deploy-docker-image
          deployment: test
          name: Push and Deploy Docker Image (test)

    deploy-to-demo:
      - variables:
          - name: IMAGE_TAG
            default: "latest"
          - name: RUN_MIGRATIONS
            default: "true"
            allowed-values:
              - "true"
              - "false"
          - name: ENABLE_MAINTENANCE_MODE
            default: "true"
            allowed-values:
              - "true"
              - "false"
      - step:
          <<: *deploy-docker-image
          deployment: demo
          name: Push and Deploy Docker Image (demo)

    deploy-to-accept:
      - variables:
          - name: IMAGE_TAG
            default: "latest"
          - name: RUN_MIGRATIONS
            default: "true"
            allowed-values:
              - "true"
              - "false"
          - name: ENABLE_MAINTENANCE_MODE
            default: "true"
            allowed-values:
              - "true"
              - "false"
      - step:
          <<: *deploy-docker-image
          deployment: accept
          name: Push and Deploy Docker Image (accept)

    deploy-to-prod:
      - variables:
          - name: IMAGE_TAG
            default: "latest"
          - name: RUN_MIGRATIONS
            default: "true"
            allowed-values:
              - "true"
              - "false"
          - name: ENABLE_MAINTENANCE_MODE
            default: "true"
            allowed-values:
              - "true"
              - "false"
      - step:
          <<: *deploy-docker-image
          deployment: production
          name: Push and Deploy Docker Image (production)

    deploy-to-ondemand:
      - variables:
          - name: IMAGE_TAG
            default: "latest"
          - name: RUN_MIGRATIONS
            default: "true"
            allowed-values:
              - "true"
              - "false"
          - name: ENABLE_MAINTENANCE_MODE
            default: "true"
            allowed-values:
              - "true"
              - "false"
      - step:
          <<: *deploy-docker-image
          deployment: ondemand
          name: Push and Deploy Docker Image (ondemand)

    deploy-to-ondemand-short-lived:
      - variables:
          - name: IMAGE_TAG
            default: "latest"
          - name: OD_ENV_NAME
            default: "od-env-name"
          - name: RUN_MIGRATIONS
            default: "true"
            allowed-values:
              - "true"
              - "false"
          - name: ENABLE_MAINTENANCE_MODE
            default: "true"
            allowed-values:
              - "true"
              - "false"
      - step:
          <<: *deploy-docker-image
          deployment: ondemand
          name: Push and Deploy Docker Image (ondemand short lived)

    deploy-to-all-nonprod:
      - variables:
          - name: IMAGE_TAG
            default: "latest"
          - name: RUN_MIGRATIONS
            default: "true"
            allowed-values:
              - "true"
              - "false"
          - name: ENABLE_MAINTENANCE_MODE
            default: "true"
            allowed-values:
              - "true"
              - "false"
      - step:
          <<: *deploy-docker-image
          deployment: test
          name: Push and Deploy Docker Image (test)
      - step:
          <<: *deploy-docker-image
          deployment: accept
          name: Push and Deploy Docker Image (accept)
    #      - step:
    #          <<: *deploy-docker-image
    #          deployment: demo
    #          name: Push and Deploy Docker Image (demo)

    build-test-scan-docker-build:
      - parallel: *common_steps_build
      - parallel: *common_steps_test
      - parallel: *common_steps_scan
      - step: *build-docker-image

    build-test-scan-docker-build-deploy-to-accept:
      - parallel: *common_steps_build
      - parallel: *common_steps_test
      - parallel: *common_steps_scan
      - step: *build-docker-image
      - step:
          <<: *deploy-docker-image
          deployment: accept
          name: Push and Deploy Docker Image (accept)

    build-hc-app-base-image:
      - step:
          <<: *build-docker-base-image

[Peter Meijer]

For other people running into the same issue:

We contacted support, apparently It's a known issue: https://jira.atlassian.com/browse/BCLOUD-23326