firecrawl/.github/workflows/test-server.yml

name: Server Test Suite

on:
  pull_request:
    branches:
      - main

env:
  BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
  HOST: ${{ secrets.HOST }}
  POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
  POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }}
  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
  PORT: ${{ secrets.PORT }}
  REDIS_URL: ${{ secrets.REDIS_URL }}
  SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}
  SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
  SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
  SUPABASE_REPLICA_URL: ${{ secrets.SUPABASE_REPLICA_URL }}
  INDEX_SUPABASE_SERVICE_TOKEN: ${{ secrets.INDEX_SUPABASE_SERVICE_TOKEN }}
  INDEX_SUPABASE_ANON_TOKEN: ${{ secrets.INDEX_SUPABASE_ANON_TOKEN }}
  INDEX_SUPABASE_URL: ${{ secrets.INDEX_SUPABASE_URL }}
  TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
  TEST_API_KEY_CONCURRENCY: ${{ secrets.TEST_API_KEY_CONCURRENCY }}
  FIRE_ENGINE_BETA_URL: ${{ secrets.FIRE_ENGINE_BETA_URL }}
  USE_DB_AUTHENTICATION: true
  SERPER_API_KEY: ${{ secrets.SERPER_API_KEY }}
  ENV: ${{ secrets.ENV }}
  RUNPOD_MU_POD_ID: ${{ secrets.RUNPOD_MU_POD_ID }}
  RUNPOD_MUV2_POD_ID: ${{ secrets.RUNPOD_MUV2_POD_ID }}
  RUNPOD_MU_API_KEY: ${{ secrets.RUNPOD_MU_API_KEY }}
  GCS_CREDENTIALS: ${{ secrets.GCS_CREDENTIALS }}
  GCS_BUCKET_NAME: ${{ secrets.GCS_BUCKET_NAME }}
  GCS_INDEX_BUCKET_NAME: ${{ secrets.GCS_INDEX_BUCKET_NAME }}
  GOOGLE_GENERATIVE_AI_API_KEY: ${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }}
  GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
  ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
  VERTEX_CREDENTIALS: ${{ secrets.VERTEX_CREDENTIALS }}
  USE_GO_MARKDOWN_PARSER: true
  SENTRY_ENVIRONMENT: dev
  IDMUX_URL: ${{ secrets.IDMUX_URL }}

jobs:
  test:
    name: Run tests
    runs-on: ubuntu-latest
    services:
      redis:
        image: redis
        ports:
          - 6379:6379
    steps:
      - uses: actions/checkout@v3
      - name: Tailscale
        uses: tailscale/github-action@v3
        with:
          oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}
          oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}
          tags: tag:ci
          use-cache: 'true'
      - name: Install pnpm
        uses: pnpm/action-setup@v4
        with:
          version: 10
      - name: Set up Node.js
        uses: actions/setup-node@v3
        with:
          node-version: "20"
          cache: "pnpm"
          cache-dependency-path: './apps/api/pnpm-lock.yaml'
      - name: Install dependencies
        run: pnpm install
        working-directory: ./apps/api
      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version: '1.19'
          cache-dependency-path: ./apps/api/sharedLibs/go-html-to-md/go.sum
      - name: Set up Rust
        uses: actions-rust-lang/setup-rust-toolchain@v1
      - name: Build go-html-to-md
        run: |
          go mod tidy
          go build -o html-to-markdown.so -buildmode=c-shared html-to-markdown.go
          chmod +x html-to-markdown.so
        working-directory: ./apps/api/sharedLibs/go-html-to-md
      - name: Build html-transformer
        run: |
          cargo build --release
          chmod +x target/release/libhtml_transformer.so
        working-directory: ./apps/api/sharedLibs/html-transformer
      - name: Build pdf-parser
        run: |
          cargo build --release
          chmod +x target/release/libpdf_parser.so
        working-directory: ./apps/api/sharedLibs/pdf-parser
      - name: Start the application
        run: npm run start:production > api.log 2>&1 &
        working-directory: ./apps/api
        id: start_app
      - name: Start worker
        run: npm run workers > worker.log 2>&1 &
        working-directory: ./apps/api
        id: start_workers
      - name: Start index worker
        run: npm run index-worker > index-worker.log 2>&1 &
        working-directory: ./apps/api
        id: start_index_worker
      - name: Wait for API
        run: pnpx wait-on tcp:3002 -t 60s
      - name: Run snippet tests
        run: |
          npm run test:snips
        working-directory: ./apps/api
      - name: Kill instances
        if: always()
        run: pkill -9 node
      # - uses: actions/upload-artifact@v4
      #   if: always()
      #   with:
      #     name: Logs
      #     path: |
      #       ./apps/api/api.log
      #       ./apps/api/worker.log
      #       ./apps/api/index-worker.log