generate artifacts in parallel

[maiermic]

I'm trying to generate files in parallel steps and commit them to a new (shared) branch.

image: atlassian/default-image:4.20230906

pipelines:
  custom:
    test-parallel-creation-of-artifacts:
      - step:
          name: Create Shared Files
          runs-on:
            - self.hosted
            - linux
          artifacts:
            - generated-artifacts/**
          script:
            - mkdir generated-artifacts
            - echo "shared" > generated-artifacts/shared-1.txt
            - echo "shared" > generated-artifacts/shared-2.txt
            - export NEW_BRANCH=${BITBUCKET_BRANCH}_test-artifacts_${BITBUCKET_BUILD_NUMBER}
            - git checkout -b ${NEW_BRANCH}
            - git add generated-artifacts/*
            - git status
            - git commit -m "Create shared files"
            - git push origin ${NEW_BRANCH}
      - parallel:
          steps:
            - step:
                name: Create file 1 and modify shared file 1
                runs-on:
                  - self.hosted
                  - linux
                artifacts:
                  - generated-artifacts/**
                script:
                  - touch generated-artifacts/1.txt
                  - echo "parallel-1" >> generated-artifacts/shared-1.txt
                  - export NEW_BRANCH=${BITBUCKET_BRANCH}_test-artifacts_${BITBUCKET_BUILD_NUMBER}
                  - git fetch origin
                  - git branch -a
                  - git checkout -b ${NEW_BRANCH}
                  - git pull
                  - git status
                  - git show HEAD -q
                  - git add generated-artifacts/*
                  - git status
                  - git commit -m "Update (part 1)"
                  - git push origin ${NEW_BRANCH}
            - step:
                name: Create file 2 and modify shared file 2
                runs-on:
                  - self.hosted
                  - linux
                artifacts:
                  - generated-artifacts/**
                script:
                  - touch generated-artifacts/2.txt
                  - echo "parallel-2" >> generated-artifacts/shared-2.txt
                  - export NEW_BRANCH=${BITBUCKET_BRANCH}_test-artifacts_${BITBUCKET_BUILD_NUMBER}
                  - git fetch origin
                  - git branch -a
                  - git checkout -b ${NEW_BRANCH}
                  - git pull
                  - git status
                  - git show HEAD -q
                  - git add generated-artifacts/*
                  - git status
                  - git commit -m "Update (part 2)"
                  - git push origin ${NEW_BRANCH}

As you can see, I create the branch in the first step with a new commit. However, if I checkout the branch and pull in one of the parallel steps, it claims

Already up to date.

but HEAD is still the commit before the commit added by "Create Shared Files". If I look at the branches in Bitbucket (https://bitbucket.org/org/proj/branches/), it shows the branch with the commit "Create shared files". Why can't the parallel steps pull this commit?

[maiermic · Accepted solution]

I dind't know/consider that it is a shallow clone of the Git repository that has to be treated differently (see more explicit fetch and rebase commands). Besides, I removed the artifacts, since the exchange is done using Git. Further, the steps have to pull the changes of the other step(s) that might have finished earlier and rebase, i.e.

image: atlassian/default-image:4.20230906

pipelines:
  custom:
    test-parallel-creation-of-artifacts:
      - step:
          name: Create Shared Files
          runs-on:
            - self.hosted
            - linux
          script:
            - mkdir generated-artifacts
            - echo "shared" > generated-artifacts/shared-1.txt
            - echo "shared" > generated-artifacts/shared-2.txt
            - export NEW_BRANCH=${BITBUCKET_BRANCH}_test-artifacts_${BITBUCKET_BUILD_NUMBER}
            - git checkout -b ${NEW_BRANCH}
            - git add generated-artifacts/*
            - git status
            - git commit -m "Create shared files"
            - git push origin ${NEW_BRANCH}
      - parallel:
          steps:
            - step:
                name: Create file 1 and modify shared file 1
                runs-on:
                  - self.hosted
                  - linux
                script:
                  - export NEW_BRANCH=${BITBUCKET_BRANCH}_test-artifacts_${BITBUCKET_BUILD_NUMBER}
                  - git fetch origin ${NEW_BRANCH}:${NEW_BRANCH}
                  - git branch -a
                  - git checkout ${NEW_BRANCH}
                  # pull changes of previous (not parallel) steps
                  - git pull
                  - git status
                  - git show HEAD -q

                  # do changes
                  - touch generated-artifacts/1.txt
                  - echo "parallel-1" >> generated-artifacts/shared-1.txt

                  # create commit
                  - git add generated-artifacts/*
                  - git status
                  - git commit -m "Update (part 1)"
                  - source ./util.sh
                  # push changes and pull changes of other parallel steps
                  - |
                    retry2 --cmd "git push origin ${NEW_BRANCH}" \
                           --max-attempts 5 \
                           --before-retry "sleep 1 && git fetch origin ${NEW_BRANCH}:origin/${NEW_BRANCH} && git rebase origin/${NEW_BRANCH}"

            - step:
                name: Create file 2 and modify shared file 2
                runs-on:
                  - self.hosted
                  - linux
                script:
                  - export NEW_BRANCH=${BITBUCKET_BRANCH}_test-artifacts_${BITBUCKET_BUILD_NUMBER}
                  - git fetch origin ${NEW_BRANCH}:${NEW_BRANCH}
                  - git branch -a
                  - git checkout ${NEW_BRANCH}
                  # pull changes of previous (not parallel) steps
                  - git pull
                  - git status
                  - git show HEAD -q

                  # do changes
                  - touch generated-artifacts/2.txt
                  - echo "parallel-2" >> generated-artifacts/shared-2.txt

                  # create commit
                  - git add generated-artifacts/*
                  - git status
                  - git commit -m "Update (part 2)"
                  - source ./util.sh
                  # push changes and pull changes of other parallel steps
                  - |
                    retry2 --cmd "git push origin ${NEW_BRANCH}" \
                           --max-attempts 5 \
                           --before-retry "sleep 1 && git fetch origin ${NEW_BRANCH}:origin/${NEW_BRANCH} && git rebase origin/${NEW_BRANCH}"

with util.sh containing

#!/bin/bash

# Execute a command (--cmd) multiple times until it succeeds or the
# maximum number of attempts (--max-attempts) is reached.
# Before each retry, another command (--before-retry) may be executed.
#
# Example:
#   retry2 --cmd "git push origin ${NEW_BRANCH}" \
#          --max-attempts 2 \
#          --before-retry "git pull --rebase"
#
retry2() {
    local max_attempts=1
    local before_retry=""
    local cmd=""

    while (( "$#" )); do
        case "$1" in
            --max-attempts)
            max_attempts=$2
            shift 2
            ;;
            --before-retry)
            before_retry=$2
            shift 2
            ;;
            --cmd)
            cmd=$2
            shift 2
            ;;
        esac
    done

    local attempt_num=1
    local exit_code=0

    while [[ $attempt_num -le $max_attempts ]]
    do
        echo "Attempt $attempt_num/$max_attempts: $cmd"
        # script should not exit if command fails,
        # even if errexit is set
        local errexit_enabled=${-//[^e]/}
        echo "disable errexit (errexit_enabled=${errexit_enabled})"
        set +o errexit

        eval "$cmd"
        exit_code="$?"

        if [[ -z "${errexit_enabled}" ]]; then
          echo "disable errexit (errexit_enabled=${errexit_enabled})"
          set +o errexit;
        else
          echo "enable errexit (errexit_enabled=${errexit_enabled})"
          set -o errexit;
        fi

        if [[ $exit_code -eq 0 ]]; then
            echo "Attempt $attempt_num/$max_attempts was successful!"
            break
        fi

        echo "Attempt $attempt_num/$max_attempts failed (exit code $exit_code)!" >&2

        if [[ $attempt_num -lt $max_attempts ]] && [[ "$before_retry" != "" ]]; then
            echo "Before retry: $before_retry"
            eval "$before_retry"
        fi

        let attempt_num++
    done

    if [[ $exit_code -ne 0 ]]; then
        echo "Max retries of $max_attempts reached. Failed to run: $cmd" >&2
    fi

    return $exit_code
}