love me some deps

postgresml · montanalow · Dec 19, 2023 · Dec 19, 2023 · Dec 19, 2023 · Dec 19, 2023
commit 39ccb685cfb9f880ec53491cddc0e50ebb21ebf8
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -33,7 +33,7 @@ jobs:
               python3-pip \
               python3 \
               lld
-          sudo pip3 install -r requirements.txt
+          sudo pip3 install -r requirements.linux.txt
       - name: Cache dependencies
         uses: buildjet/cache@v3
         if: steps.pgml_extension_changed.outputs.PGML_EXTENSION_CHANGED_FILES != '0'

diff --git a/packages/postgresml-python/DEBIAN/postinst b/packages/postgresml-python/DEBIAN/postinst
@@ -7,6 +7,5 @@ set -e
 # Setup virtualenv
 virtualenv /var/lib/postgresml-python/pgml-venv
 source "/var/lib/postgresml-python/pgml-venv/bin/activate"
-python -m pip install -r "/etc/postgresml-python/requirements.txt"
-python -m pip install -r "/etc/postgresml-python/requirements-xformers.txt" --no-dependencies
+python -m pip install -r "/etc/postgresml-python/requirements.linux.txt"
 deactivate
diff --git a/packages/postgresml-python/build.sh b/packages/postgresml-python/build.sh
@@ -28,19 +28,12 @@ rm "$deb_dir/release.sh"
 (cat ${SCRIPT_DIR}/DEBIAN/prerm | envsubst '${PGVERSION}') > "$deb_dir/DEBIAN/prerm"
 (cat ${SCRIPT_DIR}/DEBIAN/postrm | envsubst '${PGVERSION}') > "$deb_dir/DEBIAN/postrm"
 
-cp ${SCRIPT_DIR}/../../pgml-extension/requirements.txt "$deb_dir/etc/postgresml-python/requirements.txt"
+cp ${SCRIPT_DIR}/../../pgml-extension/requirements.linux.txt "$deb_dir/etc/postgresml-python/requirements.linux.txt"
 
 virtualenv --python="python$PYTHON_VERSION" "$deb_dir/var/lib/postgresml-python/pgml-venv"
 source "$deb_dir/var/lib/postgresml-python/pgml-venv/bin/activate"
 
-python -m pip install -r "${deb_dir}/etc/postgresml-python/requirements.txt"
-
-# No source included, can't build on non x86 platforms
-set +e
-python -m pip install -r "${deb_dir}/etc/postgresml-python/requirements-autogptq.txt"
-set -e
-
-python -m pip install -r "${deb_dir}/etc/postgresml-python/requirements-xformers.txt" --no-dependencies
+python -m pip install -r "${deb_dir}/etc/postgresml-python/requirements.linux.txt"
 
 deactivate
 

diff --git a/pgml-cms/docs/resources/developer-docs/installation.md b/pgml-cms/docs/resources/developer-docs/installation.md
@@ -63,16 +63,15 @@ To install the necessary Python packages into a virtual environment, use the `vi
 ```bash
 virtualenv pgml-venv && \
 source pgml-venv/bin/activate && \
-pip install -r requirements.txt && \
-pip install -r requirements-xformers.txt --no-dependencies
+pip install -r requirements.linux.txt
 ```
 {% endtab %}
 
 {% tab title="Globally" %}
 Installing Python packages globally can cause issues with your system. If you wish to proceed nonetheless, you can do so:
 
 ```bash
-pip3 install -r requirements.txt
+pip3 install -r requirements.linux.txt
 ```
 {% endtab %}
 {% endtabs %}

diff --git a/pgml-extension/Cargo.lock b/pgml-extension/Cargo.lock
diff --git a/pgml-extension/Cargo.toml b/pgml-extension/Cargo.toml
@@ -26,7 +26,6 @@ blas = { version = "0.22" }
 blas-src = { version = "0.9", features = ["openblas"] }
 indexmap = { version = "1.0", features = ["serde"] }
 itertools = "0.11"
-pyo3 = { version = "0.17", features = ["auto-initialize"], optional = true }
 heapless = "0.7"
 lightgbm = { git = "https://github.com/postgresml/lightgbm-rs", branch = "main" }
 linfa = { path = "deps/linfa" }
@@ -42,6 +41,7 @@ ndarray-stats = "0.5.1"
 parking_lot = "0.12"
 pgrx = "=0.11.2"
 pgrx-pg-sys = "=0.11.2"
+pyo3 = { version = "0.20.0", features = ["auto-initialize"], optional = true }
 rand = "0.8"
 rmp-serde = { version = "1.1" }
 signal-hook = "0.3"

diff --git a/pgml-extension/build.rs b/pgml-extension/build.rs
@@ -9,7 +9,7 @@ fn main() {
     // of symbols into the previous opened .so file, but the others will use a
     // relative offset in pgml.so, and will cause a null-pointer crash.
     //
-    // hid all symbol to avoid symbol conflicts.
+    // hide all symbol to avoid symbol conflicts.
     //
     // append mode (link-args) only works with clang ld (lld)
     println!(

diff --git a/pgml-extension/requirements.base.txt b/pgml-extension/requirements.base.txt
diff --git a/pgml-extension/requirements.linux.txt b/pgml-extension/requirements.linux.txt
@@ -0,0 +1,123 @@
+accelerate==0.25.0
+aiohttp==3.9.1
+aiosignal==1.3.1
+annotated-types==0.6.0
+anyio==4.2.0
+async-timeout==4.0.3
+attrs==23.1.0
+auto-gptq==0.6.0
+bitsandbytes==0.41.3.post2
+catboost==1.2.2
+certifi==2023.11.17
+charset-normalizer==3.3.2
+click==8.1.7
+colorama==0.4.6
+coloredlogs==15.0.1
+contourpy==1.2.0
+ctransformers==0.2.27
+cycler==0.12.1
+dataclasses-json==0.6.3
+datasets==2.15.0
+deepspeed==0.12.5
+dill==0.3.7
+einops==0.7.0
+exceptiongroup==1.2.0
+filelock==3.13.1
+fonttools==4.47.0
+frozenlist==1.4.1
+fsspec==2023.10.0
+gekko==1.0.6
+graphviz==0.20.1
+greenlet==3.0.2
+hjson==3.1.0
+huggingface-hub==0.19.4
+humanfriendly==10.0
+idna==3.6
+InstructorEmbedding==1.0.1
+Jinja2==3.1.2
+joblib==1.3.2
+jsonpatch==1.33
+jsonpointer==2.4
+kiwisolver==1.4.5
+langchain==0.0.351
+langchain-community==0.0.4
+langchain-core==0.1.1
+langsmith==0.0.72
+lightgbm==4.1.0
+lxml==4.9.3
+MarkupSafe==2.1.3
+marshmallow==3.20.1
+matplotlib==3.8.2
+mpmath==1.3.0
+multidict==6.0.4
+multiprocess==0.70.15
+mypy-extensions==1.0.0
+networkx==3.2.1
+ninja==1.11.1.1
+nltk==3.8.1
+numpy==1.26.2
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.18.1
+nvidia-nvjitlink-cu12==12.3.101
+nvidia-nvtx-cu12==12.1.105
+optimum==1.16.1
+orjson==3.9.10
+packaging==23.2
+pandas==2.1.4
+peft==0.7.1
+Pillow==10.1.0
+plotly==5.18.0
+portalocker==2.8.2
+protobuf==4.25.1
+psutil==5.9.7
+py-cpuinfo==9.0.0
+pyarrow==11.0.0
+pyarrow-hotfix==0.6
+pydantic==2.5.2
+pydantic_core==2.14.5
+pynvml==11.5.0
+pyparsing==3.1.1
+python-dateutil==2.8.2
+pytz==2023.3.post1
+PyYAML==6.0.1
+regex==2023.10.3
+requests==2.31.0
+rouge==1.0.1
+sacrebleu==2.4.0
+sacremoses==0.1.1
+safetensors==0.4.1
+scikit-learn==1.3.2
+scipy==1.11.4
+sentence-transformers==2.2.2
+sentencepiece==0.1.99
+six==1.16.0
+sniffio==1.3.0
+SQLAlchemy==2.0.23
+sympy==1.12
+tabulate==0.9.0
+tenacity==8.2.3
+threadpoolctl==3.2.0
+tokenizers==0.15.0
+torch==2.1.2
+torchaudio==2.1.2
+torchvision==0.16.2
+tqdm==4.66.1
+transformers==4.36.2
+transformers-stream-generator==0.0.4
+triton==2.1.0
+typing-inspect==0.9.0
+typing_extensions==4.9.0
+tzdata==2023.3
+urllib3==2.1.0
+xformers==0.0.23.post1
+xgboost==2.0.2
+xxhash==3.4.1
+yarl==1.9.4