Finally, I implemented interactive workflow:
---
name: 'BackstopJS test'
on:
pull_request:
types:
- edited
- opened
- synchronize
branches:
- 'develop'
env:
AWS_ACCOUNT_ID: '12345678'
AWS_REGION: 'us-east-1'
AWS_BUCKET_NAME: 'bucket_name'
AWS_BUCKET_PATH: 'bucket_folder'
AWS_BUCKET_KEY: 'bitmaps_archive.zip'
defaults:
run:
shell: bash
working-directory: backstop_test
jobs:
test:
# yamllint disable rule:line-length
if: ${{ (github.event.action != 'edited' ) || contains(github.event.pull_request.body, 'approve ') }}
name: 'BackstopJS test'
runs-on: ubuntu-latest
permissions:
id-token: write
contents: read
pull-requests: read
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Get last commit message (only 100 commits in PR are acceptable)
if: ${{ github.event.action != 'edited' }}
env:
COMMITS_URL: ${{ github.event.pull_request.commits_url }}
run: |
if [ "${COMMITS_URL}x" != "x" ]; then
echo "COMMIT_MSG=$(curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" "${COMMITS_URL}?per_page=100" | jq -r .[-1].commit.message)" >> "${GITHUB_ENV}"
else
echo '::warning ::Cannot get commits list URL'
echo 'COMMIT_MSG=' >> "${GITHUB_ENV}"
fi
- name: Search for approve directives in PR body or commit message
shell: python
env:
PR_MSG: ${{ github.event.pull_request.body }}
run: |
from os import environ as env
from sys import exit
file_path = env.get('GITHUB_ENV', None)
if file_path is None:
raise OSError('Environ file not found')
autoapprove = False
approve_only = False
commit_message = env.get('COMMIT_MSG', '')
pr_message = env.get('PR_MSG', '')
with open(file_path, 'a') as gh_envs:
if '[cancel test]' in commit_message.lower() or '[skip test]' in commit_message.lower():
gh_envs.write('SKIP_TEST=1\n')
print("::warning ::Test is skipped by commit tag")
exit(0)
elif 'cancel test' in pr_message.lower() or 'skip test' in pr_message.lower():
gh_envs.write('SKIP_TEST=1\n')
print("::warning ::Test is skipped by tag in PR message")
exit(0)
else:
gh_envs.write('SKIP_TEST=0\n')
if '[approve me]' in commit_message.lower():
autoapprove = True
approve_only = True
print("Reference bitmaps will be approved by commit message")
else:
print("Last commit message:", commit_message)
if '${{ github.event.action }}' == 'edited':
approve_only = True
pr_message = pr_message.split('\n')
last_commit_id = '${{ github.event.pull_request.head.sha }}'
commit_id = None
for line in pr_message:
if line.startswith('approve '):
commit_id = line.split(' ')[-1].rstrip('\n\r')
break
if commit_id:
if last_commit_id.startswith(commit_id):
autoapprove = True
else:
print(
"::warning ::approved commit sha and last commit sha are missmatched:",
commit_id,
"/",
last_commit_id
)
else:
print("Auto approvment disabled")
with open(file_path, 'a') as gh_envs:
if autoapprove:
gh_envs.write('AUTOAPPROVE=1\n')
else:
gh_envs.write('AUTOAPPROVE=0\n')
if approve_only and autoapprove:
gh_envs.write('APPROVE_ONLY=1\n')
else:
gh_envs.write('APPROVE_ONLY=0\n')
- name: Configure AWS credentials
if: ${{ env.SKIP_TEST != 1 }}
uses: aws-actions/configure-aws-credentials@v1
with:
role-to-assume: arn:aws:iam::${{ env.AWS_ACCOUNT_ID }}:role/github-iam-role
aws-region: ${{ env.AWS_REGION }}
role-session-name: backstopjs_test_runner
- name: Download and extract reference bitmaps
if: ${{ env.SKIP_TEST != 1 }}
run: |
aws s3 cp "s3://${AWS_BUCKET_NAME}/${AWS_BUCKET_PATH}/${AWS_BUCKET_KEY}" "./backup_${AWS_BUCKET_KEY}" && unzip -od ./backstop_data "backup_${AWS_BUCKET_KEY}" || echo "::warning file=${AWS_BUCKET_KEY}::No reference bitmaps archive found"
- name: Run tests
if: ${{ env.SKIP_TEST != 1 }}
run: |
if [ "${AUTOAPPROVE}" == "1" ] || [ "${AUTOAPPROVE^^}" == "TRUE" ] || [ "${AUTOAPPROVE^^}" == "YES" ]; then
echo "**Autoapprove is activated. Reference images will be renewed** " >> "${GITHUB_STEP_SUMMARY}"
else
{
echo "**Autoapprove is not active. Current reference images will be used** ";
echo "";
echo "Add \`approve ${{ github.event.pull_request.head.sha }}\` line to PR description";
echo "and test JOB will be automatically re-run to approve new reference bitmaps";
echo "";
echo "＊ if you will do new commit into PR, \`sha\` of current approvment should be updated too ";
echo "";
echo "Also \`[approve me]\` tag may be added inside commit message to renew bitmaps automatically ";
} >> "${GITHUB_STEP_SUMMARY}"
fi
{
echo "";
echo "Test may be cancelled by using \`[skip test]\` (or \`[cancel test]\`) tag inside commit message ";
echo "or by using \`skip test\` (or \`cancel test\`) code word inside PR message ";
echo "";
echo "---";
} >> "${GITHUB_STEP_SUMMARY}"
# Run tests here.
# If AUTOAPPROVE=1 then backstop test → backstop approve → backstop test will be run (both reports will be saved)
# If AUTOAPPROVE=1 and APPROVE_ONLY=1 then backstop reference → backstop test will be run
# ...
# Set Job to fail or success depends on tests exit status code
# In this example tests are not included, and status will be always failed if autoapprove is 0
if [ "${AUTOAPPROVE}x" == "1x" ]; then
echo "IS_FAILED=0" >> "${GITHUB_ENV}"
else
echo "IS_FAILED=1" >> "${GITHUB_ENV}"
echo "Error: test \`BLAHBLAHBLAH\` failed with status code: \`1\`" >> "${GITHUB_STEP_SUMMARY}"
fi
- name: Upload new reference ritmaps to S3 bucket
if: ${{ env.AUTOAPPROVE == 1 && env.IS_FAILED == 0 && env.SKIP_TEST != 1 }}
run: |
cd backstop_data && zip -ur "../${AWS_BUCKET_KEY}" bitmaps_reference && cd .. && \
aws s3 cp "${AWS_BUCKET_KEY}" "s3://${AWS_BUCKET_NAME}/${AWS_BUCKET_PATH}/${AWS_BUCKET_KEY}"
if [ -f "backup_${AWS_BUCKET_KEY}" ]; then
aws s3 cp "backup_${AWS_BUCKET_KEY}" "s3://${AWS_BUCKET_NAME}/${AWS_BUCKET_PATH}/backup_${AWS_BUCKET_KEY}"
fi
- name: Save HTML reports
if: ${{ env.SKIP_TEST != 1 }}
uses: actions/upload-artifact@v3
with:
name: html_reports
path: backstop_test/report
- name: Save logs (only if failed)
if: ${{ env.IS_FAILED == 1 && env.SKIP_TEST != 1 }}
uses: actions/upload-artifact@v3
with:
name: test_logs
path: backstop_test/logs
- name: Set to fail
if: ${{ env.IS_FAILED == 1 && env.SKIP_TEST != 1 }}
uses: actions/github-script@v3
with:
script: |
core.setFailed('Some of regression tests failed. Check Summary for detailed info')
Flow runs on:
- PR contents updates (
updated)
- New commits inside PR (
synchronize)
- New PR created (
opened)
If PRs body has line skip test (cancel test) or commits message has tag [skip test] ([cancel test]), then test will be skipped
If PRs body has line approve commit-SHA (where commit-SHA is a sha of a last commit in PR) or commits message has tag [approve me], then new reference bitmaps will be created
- If
approve line is present in PR, only one test with new reference images will be run
- If
approve tag is present in commit message, two tests will be run (before and after approvement) and two reports will be saved
Reference images are uploaded/downloaded/stored from/to/in S3 bucket
