pull_metadata.yml

# This is a workflow to pull all of the metadata from all of the modules into a single file
# each time something is merged to main, this workflow will run and rebuild the metadata file.

name: Process Module Metadata


# Controls when the workflow will run
on:
  # Triggers the workflow on push or pull request events but only for the "main" branch
  push:
    branches: "main" #CHANGE TO MAIN BEFORE MERGING
    paths-ignore:
      - 'assets/**'
      - '_for_authors/**'
      - '_module_templates/**'

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:

  run_script:
    name: Pull module metadata
    runs-on: ubuntu-latest
    # Map a step output to a job output
    # outputs:
    #   changes: ${{ steps.check_if_changed.outputs.changes }}
    steps:
      - uses: actions/checkout@v4
      # Process all the metadata using the script for that
      - name: Process Metadata
        run: |
          bash .github/scripts/process_metadata.sh

      - name: Upload json file as an artifact
        uses: actions/upload-artifact@v4
        with:
          name: json_artifact
          path: module_data.json

        
  check_for_changes:
    name: Has metadata changed?
    runs-on: ubuntu-latest
    needs: run_script
    # Map a step output to a job output
    outputs:
      changes: ${{ steps.check_if_changed.outputs.changes }}
    steps:
      - name: Download artifact
        uses: actions/download-artifact@v4
        with:
          name: json_artifact

      # Check if processing the data has changed the python file at all

      - name: Check if anything changed
        id: check_if_changed 
        run: |
          wget https://raw.githubusercontent.com/arcus/education_modules/metadata_workflow/assets/metadata/module_data.json -O current_data_file.txt
          wc current_data_file.txt

          changes=$(diff module_data.json current_data_file.txt | wc -l)

          echo  there are $changes changed lines
          changes=$([ $changes -gt 0 ] && echo "true" || echo "false")
          echo the changes variable is now $changes
          echo "changes=$changes" >> "$GITHUB_OUTPUT"

  # This message is for me to check that things ran correctly even if nothing actually updated
  nothing_to_update_message:
    name: No - nothing to update
    runs-on: ubuntu-latest
    needs: check_for_changes
    if: needs.check_for_changes.outputs.changes != 'true'  
    steps:
      - env:
          OUTPUT3: ${{needs.check_for_changes.outputs.changes}}
        name: Nothing was committed
        run: echo "Nothing was committed because the any_change variable was $OUTPUT3"

  # This message is for me to check that things ran correctly even if nothing actually updated
  metadata_changed_message:
    name: Yes - updates needed
    runs-on: ubuntu-latest
    needs: check_for_changes
    if: needs.check_for_changes.outputs.changes == 'true'  
    steps:
      - env:
          OUTPUT3: ${{needs.check_for_changes.outputs.changes}}
        name: Metadata has changed
        run: echo "The any_change variable was $OUTPUT3 so now the changes need to be propagated"
  

  # If changes were made to the json file, push those changes to the metadata_workflow branch

  update_metadata_workflow_branch:
    name: Module repo - Push to branch metadata_workflow 
    runs-on: ubuntu-latest
    needs: metadata_changed_message
    steps:
      - env:
          OUTPUT3: ${{needs.run_script.outputs.changes}}
        run: echo "The changes variable is $OUTPUT3"
      
      - uses: actions/checkout@v4
      
      - name: Download json artifact
        uses: actions/download-artifact@v4
        with:
          name: json_artifact

      - name: Check that artifacts downloaded
        run: |
          wc module_data.json

      - name: Commit newly updated files
        run: |
          git config --local user.name actions-user

          git config --local user.email "actions@github.com" 
 
          git fetch

          git checkout metadata_workflow

          mv -f module_data.json assets/metadata/

          git add assets/metadata/module_data.json

          git commit -am "update metadata records" 
 
          git push origin metadata_workflow 

  # The module_discovery repository automatically checks weekly to see if the module data has changed, but a message here informs users how to manually run that workflow.
  update_module_discovery_repository:
    name: App - Sun. cron job or manual update
    runs-on: ubuntu-latest
    needs: metadata_changed_message 
    steps:
      - env:
          OUTPUT3: ${{needs.run_script.outputs.changes}}
        run: echo "The changes variable is $OUTPUT3"
      
      - name: Print module_discovery instructions
        run: |
          echo "The module_discovery repository will automatically update on Sunday morning. If you have changed metadata that needs to be updated there immediately, please manually activate that workflow at https://github.com/arcus/module_discovery/actions/workflows/update_module_data.yml"


  # If the metadata changed, generate the list of modules for arcus.github.io/education_modules
  generate_webpage_markdown:
    name: Website - Generate page
    runs-on: ubuntu-latest
    needs: metadata_changed_message
    steps:
      - uses: actions/checkout@v4

      - name: Build webpage markdown file
        run: |
          bash .github/scripts/public_list_of_modules.sh
      
      - name: Upload webpage file as an artifact
        uses: actions/upload-artifact@v4
        with:
          name: webpage_artifact
          path: new_list_of_modules.md

  # It is possible for the metadata to change without any of the things displayed on the public facing website actually needing to be updated.
  check_list_of_modules_for_changes:
    name: Website - Did it change?
    runs-on: ubuntu-latest
    needs: generate_webpage_markdown
    # Map a step output to a job output
    outputs:
      changes: ${{ steps.check_if_website_changed.outputs.changes }}
    steps:
      - name: Download artifact
        uses: actions/download-artifact@v4
        with:
          name: webpage_artifact

      # Check if processing the data has changed the webpage file at all

      - name: Check if anything changed
        id: check_if_website_changed 
        run: |
          wget https://raw.githubusercontent.com/arcus/education_modules/gh-pages/docs/list_of_modules.md -O current_webpage.txt
          wc current_webpage.txt

          changes=$(diff new_list_of_modules.md current_webpage.txt | wc -l)

          echo  there are $changes changed lines
          changes=$([ $changes -gt 0 ] && echo "true" || echo "false")
          echo the changes variable is now $changes
          echo "changes=$changes" >> "$GITHUB_OUTPUT"

  # Maybe the public facing website doesn't need updates
  no_update_to_gh-pages:
    name: Website - No changes
    runs-on: ubuntu-latest
    needs: check_list_of_modules_for_changes
    if: needs.check_list_of_modules_for_changes.outputs.changes != 'true'  
    steps:
      - env:
          OUTPUT3: ${{needs.check_list_of_modules_for_changes.outputs.changes}}
        name: Nothing was committed
        run: echo "Nothing was committed because the any_change variable was $OUTPUT3"

  # The public facing website does need updates. 
  update_gh-pages:
    name: Update website
    runs-on: ubuntu-latest
    needs: check_list_of_modules_for_changes
    if: needs.check_list_of_modules_for_changes.outputs.changes == 'true'  
    steps:      
      - uses: actions/checkout@v4
      
      - name: Download webpage artifact
        uses: actions/download-artifact@v4
        with:
          name: webpage_artifact

      - name: Check that artifacts downloaded
        run: |
          wc new_list_of_modules.md


      - name: Commit newly updated file
        run: |
          git config --local user.name actions-user

          git config --local user.email "actions@github.com" 
 
          git fetch

          git checkout gh-pages

          mv -f new_list_of_modules.md docs/list_of_modules.md

          git add docs/list_of_modules.md

          git commit -am "update webpage records" 
 
          git push origin gh-pages