mirror of
https://github.com/microsoft/autogen.git
synced 2025-12-30 08:38:20 +00:00
* Added spark support for parallel training. * Added tests and fixed a bug * Added more tests and updated docs * Updated setup.py and docs * Added customize_learner and tests * Update spark tests and setup.py * Update docs and verbose * Update logging, fix issue in cloud notebook * Update github workflow for spark tests * Update github workflow * Remove hack of handling _choice_ * Allow for failures * Fix tests, update docs * Update setup.py * Update Dockerfile for Spark * Update tests, remove some warnings * Add test for notebooks, update utils * Add performance test for Spark * Fix lru_cache maxsize * Fix test failures on some platforms * Fix coverage report failure * resovle PR comments * resovle PR comments 2nd round * resovle PR comments 3rd round * fix lint and rename test class * resovle PR comments 4th round * refactor customize_learner to broadcast_code
117 lines
5.0 KiB
YAML
117 lines
5.0 KiB
YAML
# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
|
|
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
|
|
|
|
name: Build
|
|
|
|
on:
|
|
push:
|
|
branches: ['main']
|
|
pull_request:
|
|
branches: ['main']
|
|
|
|
jobs:
|
|
build:
|
|
|
|
runs-on: ${{ matrix.os }}
|
|
strategy:
|
|
matrix:
|
|
os: [ubuntu-latest, macos-latest, windows-2019]
|
|
python-version: ["3.7", "3.8", "3.9", "3.10"]
|
|
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
- name: Set up Python ${{ matrix.python-version }}
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
- name: On mac + python 3.10, install libomp to facilitate lgbm and xgboost install
|
|
if: matrix.os == 'macOS-latest' && matrix.python-version == '3.10'
|
|
run: |
|
|
# remove libomp version constraint after xgboost works with libomp>11.1.0 on python 3.10
|
|
wget https://raw.githubusercontent.com/Homebrew/homebrew-core/679923b4eb48a8dc7ecc1f05d06063cd79b3fc00/Formula/libomp.rb -O $(find $(brew --repository) -name libomp.rb)
|
|
brew unlink libomp
|
|
brew install libomp
|
|
export CC=/usr/bin/clang
|
|
export CXX=/usr/bin/clang++
|
|
export CPPFLAGS="$CPPFLAGS -Xpreprocessor -fopenmp"
|
|
export CFLAGS="$CFLAGS -I/usr/local/opt/libomp/include"
|
|
export CXXFLAGS="$CXXFLAGS -I/usr/local/opt/libomp/include"
|
|
export LDFLAGS="$LDFLAGS -Wl,-rpath,/usr/local/opt/libomp/lib -L/usr/local/opt/libomp/lib -lomp"
|
|
- name: On Linux, install Spark stand-alone cluster and PySpark
|
|
if: matrix.os == 'ubuntu-latest'
|
|
run: |
|
|
sudo apt-get update && sudo apt-get install -y --allow-downgrades --allow-change-held-packages --no-install-recommends ca-certificates-java ca-certificates openjdk-17-jdk-headless && sudo apt-get clean && sudo rm -rf /var/lib/apt/lists/*
|
|
wget --progress=dot:giga "https://www.apache.org/dyn/closer.lua/spark/spark-3.3.0/spark-3.3.0-bin-hadoop2.tgz?action=download" -O - | tar -xzC /tmp; archive=$(basename "spark-3.3.0/spark-3.3.0-bin-hadoop2.tgz") bash -c "sudo mv -v /tmp/\${archive/%.tgz/} /spark"
|
|
pip install --no-cache-dir pyspark>=3.0
|
|
export SPARK_HOME=/spark
|
|
export PYTHONPATH=/spark/python/lib/py4j-0.10.9.5-src.zip:/spark/python
|
|
export PATH=$PATH:$SPARK_HOME/bin
|
|
- name: Install packages and dependencies
|
|
run: |
|
|
python -m pip install --upgrade pip wheel
|
|
pip install -e .
|
|
python -c "import flaml"
|
|
pip install -e .[test]
|
|
- name: If linux, install ray 2
|
|
if: matrix.os == 'ubuntu-latest'
|
|
run: |
|
|
pip install ray[tune]
|
|
- name: If mac, install ray
|
|
if: matrix.os == 'macOS-latest'
|
|
run: |
|
|
pip install -e .[ray]
|
|
- name: If linux or mac, install prophet on python < 3.9
|
|
if: (matrix.os == 'macOS-latest' || matrix.os == 'ubuntu-latest') && matrix.python-version != '3.9' && matrix.python-version != '3.10'
|
|
run: |
|
|
pip install -e .[forecast]
|
|
- name: Install vw on python < 3.10
|
|
if: matrix.python-version != '3.10'
|
|
run: |
|
|
pip install -e .[vw]
|
|
- name: Lint with flake8
|
|
run: |
|
|
# stop the build if there are Python syntax errors or undefined names
|
|
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
|
|
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
|
|
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
|
|
- name: Test with pytest
|
|
if: (matrix.python-version != '3.7' || matrix.os == 'macos-latest') && matrix.python-version != '3.10'
|
|
run: |
|
|
pytest test
|
|
- name: Coverage
|
|
if: (matrix.python-version == '3.7') && matrix.os != 'macos-latest' || matrix.python-version == '3.10'
|
|
run: |
|
|
pip install coverage
|
|
coverage run -a -m pytest test
|
|
coverage xml
|
|
- name: Upload coverage to Codecov
|
|
if: (matrix.python-version == '3.7') && matrix.os != 'macos-latest' || matrix.python-version == '3.10'
|
|
uses: codecov/codecov-action@v3
|
|
with:
|
|
file: ./coverage.xml
|
|
flags: unittests
|
|
|
|
# docs:
|
|
|
|
# runs-on: ubuntu-latest
|
|
|
|
# steps:
|
|
# - uses: actions/checkout@v3
|
|
# - name: Setup Python
|
|
# uses: actions/setup-python@v4
|
|
# with:
|
|
# python-version: '3.8'
|
|
# - name: Compile documentation
|
|
# run: |
|
|
# pip install -e .
|
|
# python -m pip install sphinx sphinx_rtd_theme
|
|
# cd docs
|
|
# make html
|
|
# - name: Deploy to GitHub pages
|
|
# if: ${{ github.ref == 'refs/heads/main' }}
|
|
# uses: JamesIves/github-pages-deploy-action@3.6.2
|
|
# with:
|
|
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
# BRANCH: gh-pages
|
|
# FOLDER: docs/_build/html
|
|
# CLEAN: true |