Sync fork

.github/workflows/build_deploy_dev.yml

@@ -2,13 +2,16 @@ name: Build and deploy to dev
 
 permissions:
   contents: read
+  pull-requests: write
+  pages: write
+  id-token: write
 
 on:
   pull_request:
     types: [opened, reopened, synchronize, labeled]
     branches:
       - 'main'
-      - 'develop'
+      - 'releases/*'
 
 # Cancel any existing runs of this workflow on the same branch/pr
 # We always want to build/deploy/test a new commit over an older one
@@ -30,7 +33,8 @@ jobs:
           predicate-quantifier: "every"
           filters: |
             not-docs:
-              - '!docs/**'  
+              - '!docs/**'
+              - '!assets/**'  
               - '!**/*.md'
               - '!.github/workflows/build_deploy_dev.yml'
               - '!.github/workflows/test_on_deploy.yml'

.github/workflows/build_deploy_prod.yml

@@ -2,6 +2,9 @@ name: Build and deploy release
 
 permissions:
   contents: read
+  pull-requests: write
+  pages: write
+  id-token: write
 
 on:
   release:
@@ -18,10 +21,16 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
+  run_unit_tests:
+    name: Run Unit Tests
+    uses: ./.github/workflows/tests_unit.yml
+    secrets: inherit
+
   # Check that the version defined in the github release is valid
   check-version:
     name: Check Code Version
     runs-on: mdb-dev
+    needs: [run_unit_tests]
     if: github.actor != 'mindsdbadmin'
     steps:
       - uses: actions/checkout@v4
@@ -38,7 +47,7 @@ jobs:
   deploy_to_pypi:
     name: Publish to PyPI
     runs-on: mdb-dev
-    needs: check-version
+    needs: [check-version, run_unit_tests]
     if: github.actor != 'mindsdbadmin'
     steps:
       - uses: actions/checkout@v4
@@ -66,7 +75,7 @@ jobs:
   build:
     name: Build Docker Images
     runs-on: mdb-dev
-    needs: [check-version]
+    needs: [check-version, run_unit_tests]
     if: github.actor != 'mindsdbadmin'
     steps:
       - uses: actions/checkout@v4
@@ -96,15 +105,15 @@ jobs:
           cache-only: true
 
   # Call our deployment workflow
-  # deploy:
-  #   name: Deploy to Prod
-  #   needs: [build]
-  #   uses: ./.github/workflows/deploy.yml
-  #   with:
-  #     deploy-envs: '["prod"]'
-  #     image-tag: ${{ github.event.release.tag_name }}
-  #     prod: true
-  #   secrets: inherit
+  deploy:
+    name: Deploy to Prod
+    needs: [build]
+    uses: ./.github/workflows/deploy.yml
+    with:
+      deploy-envs: '["prod"]'
+      image-tag: ${{ github.event.release.tag_name }}
+      prod: true
+    secrets: inherit
 
   # Trigger private repo to deploy the docker desktop extension
   trigger_dd_extension_release:
@@ -124,63 +133,74 @@ jobs:
           client_payload: '{"image-tag": "${{ env.CI_REF_NAME }}"}'
 
   # Run integration tests
-  # TODO: Run these against the deployed environment
-  # run_tests:
-  #   name: Run Integration Tests
-  #   needs: [deploy]
-  #   concurrency:
-  #     group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}-tests
-  #     cancel-in-progress: true
-  #   uses: ./.github/workflows/test_on_deploy.yml
-  #   with:
-  #     deploy-env: prod
-  #   secrets: inherit
+  run_integration_tests:
+    name: Run Integration Tests
+    needs: [deploy]
+    concurrency:
+      group: deploy-prod
+      cancel-in-progress: false
+    uses: ./.github/workflows/tests_integration.yml
+    with:
+      git-sha: ${{ github.event.release.tag_name }}
+      deploy-env: prod
+      runs-on: mdb-prod
+    secrets: inherit
+
+  tests_completed:
+    name: All Tests Succeeded
+    needs: [run_unit_tests, run_integration_tests]
+    runs-on: mdb-dev
+    steps:
+      - name: fail if tests failed or didnt run
+        if: ${{ needs.run_unit_tests.result != 'success' || needs.run_integration_tests.result != 'success'}}
+        run: exit 1
+      - run: echo "Tests ran successfully"
 
-  # slack_message:
-  #   if: failure() && !cancelled()
-  #   name: Notify Slack
-  #   # Every previous job needs to be in here, because failure() will only return true if the job that failed is in 'needs'
-  #   needs: [check-version, deploy_to_pypi, build, build-cache, deploy, trigger_dd_extension_release, run_tests]
-  #   runs-on: mdb-dev
-  #   steps:
-  #     - name: Notify of failing tests
-  #       uses: slackapi/slack-github-action@v1.26.0
-  #       with:
-  #         channel-id: ${{ secrets.SLACK_ENG_CHANNEL_ID }}
-  #         payload: |
-  #           {
-  #             "attachments": [
-  #               {
-  #                 "color": "#FF4444",
-  #                 "blocks": [
-  #                   {
-  #                     "type": "header",
-  #                     "text": {
-  #                       "type": "plain_text",
-  #                       "text": "TEST RUN FAILED ON RELEASE",
-  #                       "emoji": true
-  #                     }
-  #                   },
-  #                   {
-  #                     "type": "section",
-  #                     "text": {
-  #                       "type": "mrkdwn",
-  #                       "text": " "
-  #                     },
-  #                     "fields": [
-  #                       {
-  #                         "type": "mrkdwn",
-  #                         "text": "*Commit*\n<${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}>"
-  #                       },
-  #                       {
-  #                         "type": "mrkdwn",
-  #                         "text": "*Workflow Run*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}>"
-  #                       }
-  #                     ]
-  #                   }
-  #                 ]
-  #               }
-  #             ]
-  #           }
-  #       env:
-  #         SLACK_BOT_TOKEN: ${{ secrets.GH_ACTIONS_SLACK_BOT_TOKEN }}
+  slack_message:
+    if: failure() && !cancelled()
+    name: Notify Slack
+    # Every previous job needs to be in here, because failure() will only return true if the job that failed is in 'needs'
+    needs: [check-version, run_unit_tests, deploy_to_pypi, build, build-cache, deploy, trigger_dd_extension_release, run_integration_tests, tests_completed]
+    runs-on: mdb-dev
+    steps:
+      - name: Notify of failing tests
+        uses: slackapi/slack-github-action@v1.26.0
+        with:
+          channel-id: ${{ secrets.SLACK_ENG_CHANNEL_ID }}
+          payload: |
+            {
+              "attachments": [
+                {
+                  "color": "#FF4444",
+                  "blocks": [
+                    {
+                      "type": "header",
+                      "text": {
+                        "type": "plain_text",
+                        "text": "TEST RUN FAILED ON RELEASE ${{ github.event.release.tag_name }}",
+                        "emoji": true
+                      }
+                    },
+                    {
+                      "type": "section",
+                      "text": {
+                        "type": "mrkdwn",
+                        "text": " "
+                      },
+                      "fields": [
+                        {
+                          "type": "mrkdwn",
+                          "text": "*Commit*\n<${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}>"
+                        },
+                        {
+                          "type": "mrkdwn",
+                          "text": "*Workflow Run*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}>"
+                        }
+                      ]
+                    }
+                  ]
+                }
+              ]
+            }
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.GH_ACTIONS_SLACK_BOT_TOKEN }}

.github/workflows/build_deploy_staging.yml

@@ -11,8 +11,7 @@ on:
       - closed
     branches:
       - 'main'
-      - 'develop'
-      - 'release/*'
+      - 'releases/*'
 
 concurrency:
   group: ${{ github.workflow_ref }}

.github/workflows/tests_integration.yml

@@ -13,6 +13,10 @@ on:
       deploy-env:
         required: true
         type: string
+      runs-on:
+        required: false
+        type: string
+        default: "mdb-dev"
     secrets:
       OPENAI_API_KEY:
         required: true
@@ -25,6 +29,10 @@ on:
       deploy-env:
         required: true
         type: string
+      runs-on:
+        required: false
+        type: string
+        default: "mdb-dev"
     secrets:
       OPENAI_API_KEY:
         required: true
@@ -43,7 +51,7 @@ jobs:
       name: ${{ inputs.deploy-env }}
       url: ${{ vars.MDB_ENV_URL }}
     name: Run integration tests on deploy
-    runs-on: mdb-dev
+    runs-on: ${{ inputs.runs-on }}
     steps:
     - uses: actions/checkout@v4
       with:
@@ -58,11 +66,9 @@ jobs:
     - name: Install dependencies
       run: |
         uv pip install -r requirements/requirements-test.txt
-
     - name: Run Integration Tests on Deploy
       run: |
           make integration_tests_slow
       env:
         OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         INTERNAL_URL: ${{ vars.MINDSDB_INTERNAL_URL }}
-