diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 6c60e6d5d..000000000 --- a/.dockerignore +++ /dev/null @@ -1,8 +0,0 @@ -/.git/ -/dist/ -/results/ -/tmp_check/ -/sql/vector--?.?.?.sql -regression.* -*.o -*.so diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 27c989d29..50e65a816 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,27 +8,29 @@ jobs: fail-fast: false matrix: include: + - postgres: 19 + os: ubuntu-24.04 - postgres: 18 os: ubuntu-24.04 - postgres: 17 os: ubuntu-24.04 - postgres: 16 - os: ubuntu-22.04 + os: ubuntu-24.04-arm - postgres: 15 os: ubuntu-22.04 - postgres: 14 - os: ubuntu-20.04 + os: ubuntu-22.04-arm - postgres: 13 - os: ubuntu-20.04 + os: ubuntu-22.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: ankane/setup-postgres@v1 with: postgres-version: ${{ matrix.postgres }} dev-files: true - run: make env: - PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare + PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-sign-compare ${{ matrix.postgres >= 18 && '-Wno-missing-field-initializers' || '' }} - run: | export PG_CONFIG=`which pg_config` sudo --preserve-env=PG_CONFIG make install @@ -46,18 +48,18 @@ jobs: fail-fast: false matrix: include: - - postgres: 17 - os: macos-15 + - postgres: 18 + os: macos-26 - postgres: 14 - os: macos-13 + os: macos-15-intel steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: ankane/setup-postgres@v1 with: postgres-version: ${{ matrix.postgres }} - run: make env: - PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter + PG_CFLAGS: -DUSE_ASSERT_CHECKING -Wall -Wextra -Werror -Wno-unused-parameter -Wno-unknown-warning-option ${{ matrix.postgres >= 18 && '-Wno-missing-field-initializers' || '' }} - run: make install - run: make installcheck - if: ${{ failure() }} @@ -70,27 +72,35 @@ jobs: tar xf $TAG.tar.gz mv postgres-$TAG postgres env: - TAG: ${{ matrix.postgres == 17 && 'REL_17_2' || 'REL_14_15' }} + TAG: ${{ matrix.postgres == 18 && 'REL_18_0' || 'REL_14_19' }} - run: make prove_installcheck PROVE_FLAGS="-I ./postgres/src/test/perl -I ./test/perl" env: PERL5LIB: /Users/runner/perl5/lib/perl5 - run: make clean && $(brew --prefix llvm@$LLVM_VERSION)/bin/scan-build --status-bugs make env: - LLVM_VERSION: ${{ matrix.os == 'macos-15' && 18 || 15 }} + LLVM_VERSION: ${{ matrix.os == 'macos-26' && 20 || 18 }} PG_CFLAGS: -DUSE_ASSERT_CHECKING windows: - runs-on: windows-latest + runs-on: ${{ matrix.os }} if: ${{ !startsWith(github.ref_name, 'mac') }} + strategy: + fail-fast: false + matrix: + include: + - postgres: 17 + os: windows-2025 + - postgres: 14 + os: windows-2022 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: ankane/setup-postgres@v1 with: - postgres-version: 14 + postgres-version: ${{ matrix.postgres }} - run: | call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^ nmake /NOLOGO /F Makefile.win && ^ nmake /NOLOGO /F Makefile.win install && ^ - nmake /NOLOGO /F Makefile.win installcheck && ^ + nmake /NOLOGO /F Makefile.win installcheck ${{ matrix.postgres != 17 && 'PG_REGRESS=$(PGROOT)\bin\pg_regress' || '' }} && ^ nmake /NOLOGO /F Makefile.win clean && ^ nmake /NOLOGO /F Makefile.win uninstall shell: cmd @@ -123,10 +133,10 @@ jobs: if: ${{ !startsWith(github.ref_name, 'mac') && !startsWith(github.ref_name, 'windows') }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: ankane/setup-postgres-valgrind@v1 with: - postgres-version: 17 + postgres-version: 18 check-ub: yes - run: make OPTFLAGS="" - run: sudo --preserve-env=PG_CONFIG make install diff --git a/CHANGELOG.md b/CHANGELOG.md index 757f998d0..8f74ea89f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## 0.8.2 (unreleased) + +- Improved `install` target on Windows +- Fixed `Index Searches` in `EXPLAIN` output for Postgres 18 + +## 0.8.1 (2025-09-04) + +- Added support for Postgres 18 rc1 +- Improved performance of `binary_quantize` function + ## 0.8.0 (2024-10-30) - Added support for iterative index scans diff --git a/Dockerfile b/Dockerfile index 936440928..7e06759d5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,11 @@ +# syntax=docker/dockerfile:1 + ARG PG_MAJOR=17 -FROM postgres:$PG_MAJOR +ARG DEBIAN_CODENAME=bookworm +FROM postgres:$PG_MAJOR-$DEBIAN_CODENAME ARG PG_MAJOR -COPY . /tmp/pgvector +ADD https://github.com/pgvector/pgvector.git#v0.8.1 /tmp/pgvector RUN apt-get update && \ apt-mark hold locales && \ diff --git a/META.json b/META.json index b9a68f62a..343518d1c 100644 --- a/META.json +++ b/META.json @@ -2,7 +2,7 @@ "name": "vector", "abstract": "Open-source vector similarity search for Postgres", "description": "Supports L2 distance, inner product, and cosine distance", - "version": "0.8.0", + "version": "0.8.1", "maintainer": [ "Andrew Kane " ], @@ -20,7 +20,7 @@ "vector": { "file": "sql/vector.sql", "docfile": "README.md", - "version": "0.8.0", + "version": "0.8.1", "abstract": "Open-source vector similarity search for Postgres" } }, diff --git a/Makefile b/Makefile index 7a4b88caf..d98d73b8c 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ EXTENSION = vector -EXTVERSION = 0.8.0 +EXTVERSION = 0.8.1 MODULE_big = vector DATA = $(wildcard sql/*--*--*.sql) @@ -76,4 +76,9 @@ docker: .PHONY: docker-release docker-release: - docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) -t pgvector/pgvector:pg$(PG_MAJOR) -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR) . + docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) --build-arg DEBIAN_CODENAME=bookworm -t pgvector/pgvector:pg$(PG_MAJOR) -t pgvector/pgvector:pg$(PG_MAJOR)-bookworm -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR) -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR)-bookworm . + +.PHONY: docker-release-trixie + +docker-release-trixie: + docker buildx build --push --pull --no-cache --platform linux/amd64,linux/arm64 --build-arg PG_MAJOR=$(PG_MAJOR) --build-arg DEBIAN_CODENAME=trixie -t pgvector/pgvector:pg$(PG_MAJOR)-trixie -t pgvector/pgvector:$(EXTVERSION)-pg$(PG_MAJOR)-trixie . diff --git a/Makefile.win b/Makefile.win index 8c62f9d5f..9d4cbe842 100644 --- a/Makefile.win +++ b/Makefile.win @@ -1,5 +1,5 @@ EXTENSION = vector -EXTVERSION = 0.8.0 +EXTVERSION = 0.8.1 DATA_built = sql\$(EXTENSION)--$(EXTVERSION).sql OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj @@ -31,6 +31,9 @@ LIBDIR = $(PGROOT)\lib PKGLIBDIR = $(PGROOT)\lib SHAREDIR = $(PGROOT)\share +# Use $(PGROOT)\bin\pg_regress for Postgres < 17 +PG_REGRESS = $(LIBDIR)\pgxs\src\test\regress\pg_regress + CFLAGS = /nologo /I"$(INCLUDEDIR_SERVER)\port\win32_msvc" /I"$(INCLUDEDIR_SERVER)\port\win32" /I"$(INCLUDEDIR_SERVER)" /I"$(INCLUDEDIR)" CFLAGS = $(CFLAGS) $(PG_CFLAGS) @@ -54,11 +57,11 @@ install: all copy $(SHLIB) "$(PKGLIBDIR)" copy $(EXTENSION).control "$(SHAREDIR)\extension" copy sql\$(EXTENSION)--*.sql "$(SHAREDIR)\extension" - mkdir "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)" + if not exist "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)" mkdir "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)" for %f in ($(HEADERS)) do copy %f "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)" installcheck: - "$(BINDIR)\pg_regress" --bindir="$(BINDIR)" $(REGRESS_OPTS) $(REGRESS) + "$(PG_REGRESS)" --bindir="$(BINDIR)" $(REGRESS_OPTS) $(REGRESS) uninstall: del /f "$(PKGLIBDIR)\$(SHLIB)" diff --git a/README.md b/README.md index 04e795343..5f24eb92f 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Compile and install the extension (supports Postgres 13+) ```sh cd /tmp -git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git +git clone --branch v0.8.1 https://github.com/pgvector/pgvector.git cd pgvector make make install # may need sudo @@ -29,31 +29,21 @@ make install # may need sudo See the [installation notes](#installation-notes---linux-and-mac) if you run into issues -You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [APT](#apt), [Yum](#yum), [pkg](#pkg), or [conda-forge](#conda-forge), and it comes preinstalled with [Postgres.app](#postgresapp) and many [hosted providers](#hosted-postgres). There are also instructions for [GitHub Actions](https://github.com/pgvector/setup-pgvector). +You can also install it with [Docker](#docker), [Homebrew](#homebrew), [PGXN](#pgxn), [APT](#apt), [Yum](#yum), [pkg](#pkg), [APK](#apk), or [conda-forge](#conda-forge), and it comes preinstalled with [Postgres.app](#postgresapp) and many [hosted providers](#hosted-postgres). There are also instructions for [GitHub Actions](https://github.com/pgvector/setup-pgvector). ### Windows -Ensure [C++ support in Visual Studio](https://learn.microsoft.com/en-us/cpp/build/building-on-the-command-line?view=msvc-170#download-and-install-the-tools) is installed, and run: +Ensure [C++ support in Visual Studio](https://learn.microsoft.com/en-us/cpp/build/building-on-the-command-line?view=msvc-170#download-and-install-the-tools) is installed and run `x64 Native Tools Command Prompt for VS [version]` as administrator. Then use `nmake` to build: ```cmd -call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat" -``` - -Note: The exact path will vary depending on your Visual Studio version and edition - -Then use `nmake` to build: - -```cmd -set "PGROOT=C:\Program Files\PostgreSQL\16" +set "PGROOT=C:\Program Files\PostgreSQL\18" cd %TEMP% -git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git +git clone --branch v0.8.1 https://github.com/pgvector/pgvector.git cd pgvector nmake /F Makefile.win nmake /F Makefile.win install ``` -Note: Postgres 17 is not supported with MSVC yet due to an [upstream issue](https://www.postgresql.org/message-id/flat/CAOdR5yF0krWrxycA04rgUKCgKugRvGWzzGLAhDZ9bzNv8g0Lag%40mail.gmail.com) - See the [installation notes](#installation-notes---windows) if you run into issues You can also install it with [Docker](#docker) or [conda-forge](#conda-forge). @@ -84,7 +74,7 @@ Get the nearest neighbors by L2 distance SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5; ``` -Also supports inner product (`<#>`), cosine distance (`<=>`), and L1 distance (`<+>`, added in 0.7.0) +Also supports inner product (`<#>`), cosine distance (`<=>`), and L1 distance (`<+>`) Note: `<#>` returns the negative inner product since Postgres only supports `ASC` order index scans on operators @@ -148,9 +138,9 @@ Supported distance functions are: - `<->` - L2 distance - `<#>` - (negative) inner product - `<=>` - cosine distance -- `<+>` - L1 distance (added in 0.7.0) -- `<~>` - Hamming distance (binary vectors, added in 0.7.0) -- `<%>` - Jaccard distance (binary vectors, added in 0.7.0) +- `<+>` - L1 distance +- `<~>` - Hamming distance (binary vectors) +- `<%>` - Jaccard distance (binary vectors) Get the nearest neighbors to a row @@ -237,19 +227,19 @@ Cosine distance CREATE INDEX ON items USING hnsw (embedding vector_cosine_ops); ``` -L1 distance - added in 0.7.0 +L1 distance ```sql CREATE INDEX ON items USING hnsw (embedding vector_l1_ops); ``` -Hamming distance - added in 0.7.0 +Hamming distance ```sql CREATE INDEX ON items USING hnsw (embedding bit_hamming_ops); ``` -Jaccard distance - added in 0.7.0 +Jaccard distance ```sql CREATE INDEX ON items USING hnsw (embedding bit_jaccard_ops); @@ -258,9 +248,9 @@ CREATE INDEX ON items USING hnsw (embedding bit_jaccard_ops); Supported types are: - `vector` - up to 2,000 dimensions -- `halfvec` - up to 4,000 dimensions (added in 0.7.0) -- `bit` - up to 64,000 dimensions (added in 0.7.0) -- `sparsevec` - up to 1,000 non-zero elements (added in 0.7.0) +- `halfvec` - up to 4,000 dimensions +- `bit` - up to 64,000 dimensions +- `sparsevec` - up to 1,000 non-zero elements ### Index Options @@ -314,13 +304,15 @@ Note: Do not set `maintenance_work_mem` so high that it exhausts the memory on t Like other index types, it’s faster to create an index after loading your initial data -Starting with 0.6.0, you can also speed up index creation by increasing the number of parallel workers (2 by default) +You can also speed up index creation by increasing the number of parallel workers (2 by default) ```sql SET max_parallel_maintenance_workers = 7; -- plus leader ``` -For a large number of workers, you may also need to increase `max_parallel_workers` (8 by default) +For a large number of workers, you may need to increase `max_parallel_workers` (8 by default) + +The [index options](#index-options) also have a significant impact on build time (use the defaults unless seeing low recall) ### Indexing Progress @@ -367,7 +359,7 @@ Cosine distance CREATE INDEX ON items USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100); ``` -Hamming distance - added in 0.7.0 +Hamming distance ```sql CREATE INDEX ON items USING ivfflat (embedding bit_hamming_ops) WITH (lists = 100); @@ -376,8 +368,8 @@ CREATE INDEX ON items USING ivfflat (embedding bit_hamming_ops) WITH (lists = 10 Supported types are: - `vector` - up to 2,000 dimensions -- `halfvec` - up to 4,000 dimensions (added in 0.7.0) -- `bit` - up to 64,000 dimensions (added in 0.7.0) +- `halfvec` - up to 4,000 dimensions +- `bit` - up to 64,000 dimensions ### Query Options @@ -477,8 +469,6 @@ CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(cate ## Iterative Index Scans -*Added in 0.8.0* - With approximate indexes, queries with filtering can return less results since filtering is applied *after* the index is scanned. Starting with 0.8.0, you can enable iterative index scans, which will automatically scan more of the index until enough results are found (or it reaches `hnsw.max_scan_tuples` or `ivfflat.max_probes`). Iterative scans can use strict or relaxed ordering. @@ -502,9 +492,11 @@ With relaxed ordering, you can use a [materialized CTE](https://www.postgresql.o ```sql WITH relaxed_results AS MATERIALIZED ( SELECT id, embedding <-> '[1,2,3]' AS distance FROM items WHERE category_id = 123 ORDER BY distance LIMIT 5 -) SELECT * FROM relaxed_results ORDER BY distance; +) SELECT * FROM relaxed_results ORDER BY distance + 0; ``` +Note: `+ 0` is needed for Postgres 17+ + For queries that filter by distance, use a materialized CTE and place the distance filter outside of it for best performance (due to the [current behavior](https://www.postgresql.org/message-id/flat/CAOdR5yGUoMQ6j7M5hNUXrySzaqZVGf_Ne%2B8fwZMRKTFxU1nbJg%40mail.gmail.com) of the Postgres executor) ```sql @@ -549,8 +541,6 @@ Note: If this is lower than `ivfflat.probes`, `ivfflat.probes` will be used ## Half-Precision Vectors -*Added in 0.7.0* - Use the `halfvec` type to store half-precision vectors ```sql @@ -559,8 +549,6 @@ CREATE TABLE items (id bigserial PRIMARY KEY, embedding halfvec(3)); ## Half-Precision Indexing -*Added in 0.7.0* - Index vectors at half precision for smaller indexes ```sql @@ -582,24 +570,16 @@ CREATE TABLE items (id bigserial PRIMARY KEY, embedding bit(3)); INSERT INTO items (embedding) VALUES ('000'), ('111'); ``` -Get the nearest neighbors by Hamming distance (added in 0.7.0) +Get the nearest neighbors by Hamming distance ```sql SELECT * FROM items ORDER BY embedding <~> '101' LIMIT 5; ``` -Or (before 0.7.0) - -```sql -SELECT * FROM items ORDER BY bit_count(embedding # '101') LIMIT 5; -``` - Also supports Jaccard distance (`<%>`) ## Binary Quantization -*Added in 0.7.0* - Use expression indexing for binary quantization ```sql @@ -622,8 +602,6 @@ SELECT * FROM ( ## Sparse Vectors -*Added in 0.7.0* - Use the `sparsevec` type to store sparse vectors ```sql @@ -657,8 +635,6 @@ You can use [Reciprocal Rank Fusion](https://github.com/pgvector/pgvector-python ## Indexing Subvectors -*Added in 0.7.0* - Use expression indexing to index subvectors ```sql @@ -719,10 +695,10 @@ CREATE INDEX CONCURRENTLY ... ### Querying -Use `EXPLAIN ANALYZE` to debug performance. +Use `EXPLAIN (ANALYZE, BUFFERS)` to debug performance. ```sql -EXPLAIN ANALYZE SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5; +EXPLAIN (ANALYZE, BUFFERS) SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5; ``` #### Exact Search @@ -772,8 +748,6 @@ SELECT query, calls, ROUND((total_plan_time + total_exec_time) / calls) AS avg_t FROM pg_stat_statements ORDER BY total_plan_time + total_exec_time DESC LIMIT 20; ``` -Note: Replace `total_plan_time + total_exec_time` with `total_time` for Postgres < 13 - Monitor recall by comparing results from approximate search with exact search. ```sql @@ -797,9 +771,12 @@ Use pgvector from any language with a Postgres client. You can even generate and Language | Libraries / Examples --- | --- +Ada | [pgvector-ada](https://github.com/pgvector/pgvector-ada) +Algol | [pgvector-algol](https://github.com/pgvector/pgvector-algol) C | [pgvector-c](https://github.com/pgvector/pgvector-c) C++ | [pgvector-cpp](https://github.com/pgvector/pgvector-cpp) C#, F#, Visual Basic | [pgvector-dotnet](https://github.com/pgvector/pgvector-dotnet) +COBOL | [pgvector-cobol](https://github.com/pgvector/pgvector-cobol) Crystal | [pgvector-crystal](https://github.com/pgvector/pgvector-crystal) D | [pgvector-d](https://github.com/pgvector/pgvector-d) Dart | [pgvector-dart](https://github.com/pgvector/pgvector-dart) @@ -811,19 +788,23 @@ Go | [pgvector-go](https://github.com/pgvector/pgvector-go) Haskell | [pgvector-haskell](https://github.com/pgvector/pgvector-haskell) Java, Kotlin, Groovy, Scala | [pgvector-java](https://github.com/pgvector/pgvector-java) JavaScript, TypeScript | [pgvector-node](https://github.com/pgvector/pgvector-node) -Julia | [pgvector-julia](https://github.com/pgvector/pgvector-julia) +Julia | [Pgvector.jl](https://github.com/pgvector/Pgvector.jl) Lisp | [pgvector-lisp](https://github.com/pgvector/pgvector-lisp) Lua | [pgvector-lua](https://github.com/pgvector/pgvector-lua) Nim | [pgvector-nim](https://github.com/pgvector/pgvector-nim) OCaml | [pgvector-ocaml](https://github.com/pgvector/pgvector-ocaml) +Pascal | [pgvector-pascal](https://github.com/pgvector/pgvector-pascal) Perl | [pgvector-perl](https://github.com/pgvector/pgvector-perl) PHP | [pgvector-php](https://github.com/pgvector/pgvector-php) +Prolog | [pgvector-prolog](https://github.com/pgvector/pgvector-prolog) Python | [pgvector-python](https://github.com/pgvector/pgvector-python) R | [pgvector-r](https://github.com/pgvector/pgvector-r) +Racket | [pgvector-racket](https://github.com/pgvector/pgvector-racket) Raku | [pgvector-raku](https://github.com/pgvector/pgvector-raku) Ruby | [pgvector-ruby](https://github.com/pgvector/pgvector-ruby), [Neighbor](https://github.com/ankane/neighbor) Rust | [pgvector-rust](https://github.com/pgvector/pgvector-rust) Swift | [pgvector-swift](https://github.com/pgvector/pgvector-swift) +Tcl | [pgvector-tcl](https://github.com/pgvector/pgvector-tcl) Zig | [pgvector-zig](https://github.com/pgvector/pgvector-zig) ## Frequently Asked Questions @@ -838,11 +819,11 @@ Yes, pgvector uses the write-ahead log (WAL), which allows for replication and p #### What if I want to index vectors with more than 2,000 dimensions? -You can use [half-precision indexing](#half-precision-indexing) to index up to 4,000 dimensions or [binary quantization](#binary-quantization) to index up to 64,000 dimensions. Another option is [dimensionality reduction](https://en.wikipedia.org/wiki/Dimensionality_reduction). +You can use [half-precision vectors](#half-precision-vectors) or [half-precision indexing](#half-precision-indexing) to index up to 4,000 dimensions or [binary quantization](#binary-quantization) to index up to 64,000 dimensions. Other options are [indexing subvectors](#indexing-subvectors) (for models that support it) or [dimensionality reduction](https://en.wikipedia.org/wiki/Dimensionality_reduction). #### Can I store vectors with different dimensions in the same column? -You can use `vector` as the type (instead of `vector(3)`). +You can use `vector` as the type (instead of `vector(n)`). ```sql CREATE TABLE embeddings (model_id bigint, item_id bigint, embedding vector, PRIMARY KEY (model_id, item_id)); @@ -1090,7 +1071,7 @@ l2_normalize(sparsevec) → sparsevec | Normalize with Euclidean norm | 0.7.0 If your machine has multiple Postgres installations, specify the path to [pg_config](https://www.postgresql.org/docs/current/app-pgconfig.html) with: ```sh -export PG_CONFIG=/Library/PostgreSQL/17/bin/pg_config +export PG_CONFIG=/Library/PostgreSQL/18/bin/pg_config ``` Then re-run the installation instructions (run `make clean` before `make` if needed). If `sudo` is needed for `make install`, use: @@ -1101,11 +1082,11 @@ sudo --preserve-env=PG_CONFIG make install A few common paths on Mac are: -- EDB installer - `/Library/PostgreSQL/17/bin/pg_config` -- Homebrew (arm64) - `/opt/homebrew/opt/postgresql@17/bin/pg_config` -- Homebrew (x86-64) - `/usr/local/opt/postgresql@17/bin/pg_config` +- EDB installer - `/Library/PostgreSQL/18/bin/pg_config` +- Homebrew (arm64) - `/opt/homebrew/opt/postgresql@18/bin/pg_config` +- Homebrew (x86-64) - `/usr/local/opt/postgresql@18/bin/pg_config` -Note: Replace `17` with your Postgres server version +Note: Replace `18` with your Postgres server version ### Missing Header @@ -1114,14 +1095,20 @@ If compilation fails with `fatal error: postgres.h: No such file or directory`, For Ubuntu and Debian, use: ```sh -sudo apt install postgresql-server-dev-17 +sudo apt install postgresql-server-dev-18 ``` -Note: Replace `17` with your Postgres server version +Note: Replace `18` with your Postgres server version ### Missing SDK -If compilation fails and the output includes `warning: no such sysroot directory` on Mac, reinstall Xcode Command Line Tools. +If compilation fails and the output includes `warning: no such sysroot directory` on Mac, your Postgres installation points to a path that no longer exists. + +```sh +pg_config --cppflags +``` + +Reinstall Postgres to fix this. ### Portability @@ -1139,6 +1126,14 @@ make OPTFLAGS="" If compilation fails with `Cannot open include file: 'postgres.h': No such file or directory`, make sure `PGROOT` is correct. +### Mismatched Architecture + +If compilation fails with `error C2196: case value '4' already used`, make sure you’re using the `x64 Native Tools Command Prompt`. Then run `nmake /F Makefile.win clean` and re-run the installation instructions. + +### Missing Symbol + +If linking fails with `unresolved external symbol float_to_shortest_decimal_bufn` with Postgres 17.0-17.2, upgrade to Postgres 17.3+. + ### Permissions If installation fails with `Access is denied`, re-run the installation instructions as an administrator. @@ -1150,17 +1145,38 @@ If installation fails with `Access is denied`, re-run the installation instructi Get the [Docker image](https://hub.docker.com/r/pgvector/pgvector) with: ```sh -docker pull pgvector/pgvector:pg17 +docker pull pgvector/pgvector:pg18-trixie ``` -This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (replace `17` with your Postgres server version, and run it the same way). +This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (replace `18` with your Postgres server version, and run it the same way). + +Supported tags are: + +- `pg18-trixie`, `0.8.1-pg18-trixie` +- `pg18-bookworm`, `0.8.1-pg18-bookworm`, `pg18`, `0.8.1-pg18` +- `pg17-trixie`, `0.8.1-pg17-trixie` +- `pg17-bookworm`, `0.8.1-pg17-bookworm`, `pg17`, `0.8.1-pg17` +- `pg16-trixie`, `0.8.1-pg16-trixie` +- `pg16-bookworm`, `0.8.1-pg16-bookworm`, `pg16`, `0.8.1-pg16` +- `pg15-trixie`, `0.8.1-pg15-trixie` +- `pg15-bookworm`, `0.8.1-pg15-bookworm`, `pg15`, `0.8.1-pg15` +- `pg14-trixie`, `0.8.1-pg14-trixie` +- `pg14-bookworm`, `0.8.1-pg14-bookworm`, `pg14`, `0.8.1-pg14` +- `pg13-trixie`, `0.8.1-pg13-trixie` +- `pg13-bookworm`, `0.8.1-pg13-bookworm`, `pg13`, `0.8.1-pg13` You can also build the image manually: ```sh -git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git +git clone --branch v0.8.1 https://github.com/pgvector/pgvector.git cd pgvector -docker build --pull --build-arg PG_MAJOR=17 -t myuser/pgvector . +docker build --pull --build-arg PG_MAJOR=18 -t myuser/pgvector . +``` + +If you increase `maintenance_work_mem`, make sure `--shm-size` is at least that size to avoid an error with parallel HNSW index builds. + +```sh +docker run --shm-size=1g ... ``` ### Homebrew @@ -1171,7 +1187,7 @@ With Homebrew Postgres, you can use: brew install pgvector ``` -Note: This only adds it to the `postgresql@17` and `postgresql@14` formulas +Note: This only adds it to the `postgresql@18` and `postgresql@17` formulas ### PGXN @@ -1186,29 +1202,29 @@ pgxn install vector Debian and Ubuntu packages are available from the [PostgreSQL APT Repository](https://wiki.postgresql.org/wiki/Apt). Follow the [setup instructions](https://wiki.postgresql.org/wiki/Apt#Quickstart) and run: ```sh -sudo apt install postgresql-17-pgvector +sudo apt install postgresql-18-pgvector ``` -Note: Replace `17` with your Postgres server version +Note: Replace `18` with your Postgres server version ### Yum RPM packages are available from the [PostgreSQL Yum Repository](https://yum.postgresql.org/). Follow the [setup instructions](https://www.postgresql.org/download/linux/redhat/) for your distribution and run: ```sh -sudo yum install pgvector_17 +sudo yum install pgvector_18 # or -sudo dnf install pgvector_17 +sudo dnf install pgvector_18 ``` -Note: Replace `17` with your Postgres server version +Note: Replace `18` with your Postgres server version ### pkg Install the FreeBSD package with: ```sh -pkg install postgresql16-pgvector +pkg install postgresql17-pgvector ``` or the port with: @@ -1218,6 +1234,14 @@ cd /usr/ports/databases/pgvector make install ``` +### APK + +Install the Alpine package with: + +```sh +apk add postgresql-pgvector +``` + ### conda-forge With Conda Postgres, install from [conda-forge](https://anaconda.org/conda-forge/pgvector) with: @@ -1250,36 +1274,6 @@ You can check the version in the current database with: SELECT extversion FROM pg_extension WHERE extname = 'vector'; ``` -## Upgrade Notes - -### 0.6.0 - -#### Postgres 12 - -If upgrading with Postgres 12, remove this line from `sql/vector--0.5.1--0.6.0.sql`: - -```sql -ALTER TYPE vector SET (STORAGE = external); -``` - -Then run `make install` and `ALTER EXTENSION vector UPDATE;`. - -#### Docker - -The Docker image is now published in the `pgvector` org, and there are tags for each supported version of Postgres (rather than a `latest` tag). - -```sh -docker pull pgvector/pgvector:pg16 -# or -docker pull pgvector/pgvector:0.6.0-pg16 -``` - -Also, if you’ve increased `maintenance_work_mem`, make sure `--shm-size` is at least that size to avoid an error with parallel HNSW index builds. - -```sh -docker run --shm-size=1g ... -``` - ## Thanks Thanks to: diff --git a/sql/vector--0.8.0--0.8.1.sql b/sql/vector--0.8.0--0.8.1.sql new file mode 100644 index 000000000..547bd44a3 --- /dev/null +++ b/sql/vector--0.8.0--0.8.1.sql @@ -0,0 +1,2 @@ +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.1'" to load this file. \quit diff --git a/src/bitvec.c b/src/bitvec.c index 094ddd282..8bdff956b 100644 --- a/src/bitvec.c +++ b/src/bitvec.c @@ -2,6 +2,7 @@ #include "bitutils.h" #include "bitvec.h" +#include "fmgr.h" #include "utils/varbit.h" #include "vector.h" diff --git a/src/halfutils.c b/src/halfutils.c index d16909409..2df4995c0 100644 --- a/src/halfutils.c +++ b/src/halfutils.c @@ -1,5 +1,7 @@ #include "postgres.h" +#include + #include "halfutils.h" #include "halfvec.h" diff --git a/src/halfvec.c b/src/halfvec.c index aad320b1c..6854c3d71 100644 --- a/src/halfvec.c +++ b/src/halfvec.c @@ -13,12 +13,16 @@ #include "port.h" /* for strtof() */ #include "sparsevec.h" #include "utils/array.h" -#include "utils/builtins.h" #include "utils/float.h" +#include "utils/fmgrprotos.h" #include "utils/lsyscache.h" -#include "utils/numeric.h" +#include "utils/varbit.h" #include "vector.h" +#if PG_VERSION_NUM >= 160000 +#include "varatt.h" +#endif + #define STATE_DIMS(x) (ARR_DIMS(x)[0] - 1) #define CreateStateDatums(dim) palloc(sizeof(Datum) * (dim + 1)) @@ -898,8 +902,21 @@ halfvec_binary_quantize(PG_FUNCTION_ARGS) half *ax = a->x; VarBit *result = InitBitVector(a->dim); unsigned char *rx = VARBITS(result); + int i = 0; + int count = (a->dim / 8) * 8; - for (int i = 0; i < a->dim; i++) + /* Auto-vectorized on aarch64 */ + for (; i < count; i += 8) + { + unsigned char result_byte = 0; + + for (int j = 0; j < 8; j++) + result_byte |= (HalfToFloat4(ax[i + j]) > 0) << (7 - j); + + rx[i / 8] = result_byte; + } + + for (; i < a->dim; i++) rx[i / 8] |= (HalfToFloat4(ax[i]) > 0) << (7 - (i % 8)); PG_RETURN_VARBIT_P(result); diff --git a/src/hnsw.c b/src/hnsw.c index 5bfc6193e..891d0701e 100644 --- a/src/hnsw.c +++ b/src/hnsw.c @@ -1,18 +1,24 @@ #include "postgres.h" #include +#include #include #include "access/amapi.h" +#include "access/genam.h" #include "access/reloptions.h" #include "commands/progress.h" #include "commands/vacuum.h" +#include "fmgr.h" #include "hnsw.h" #include "miscadmin.h" +#include "nodes/pg_list.h" #include "utils/float.h" #include "utils/guc.h" +#include "utils/relcache.h" #include "utils/selfuncs.h" #include "utils/spccache.h" +#include "vector.h" #if PG_VERSION_NUM < 150000 #define MarkGUCPrefixReserved(x) EmitWarningsOnPlaceholders(x) @@ -52,12 +58,20 @@ HnswInitLockTranche(void) sizeof(int) * 1, &found); if (!found) + { +#if PG_VERSION_NUM >= 190000 + tranche_ids[0] = LWLockNewTrancheId("HnswBuild"); +#else tranche_ids[0] = LWLockNewTrancheId(); +#endif + } hnsw_lock_tranche_id = tranche_ids[0]; LWLockRelease(AddinShmemInitLock); +#if PG_VERSION_NUM < 190000 /* Per-backend registration of the tranche ID */ LWLockRegisterTranche(hnsw_lock_tranche_id, "HnswBuild"); +#endif } /* @@ -130,7 +144,7 @@ hnswcostestimate(PlannerInfo *root, IndexPath *path, double loop_count, Relation index; /* Never use index without order */ - if (path->indexorderbys == NULL) + if (path->indexorderbys == NIL) { *indexStartupCost = get_float8_infinity(); *indexTotalCost = get_float8_infinity(); @@ -259,6 +273,11 @@ hnswhandler(PG_FUNCTION_ARGS) amroutine->amoptsprocnum = 0; amroutine->amcanorder = false; amroutine->amcanorderbyop = true; +#if PG_VERSION_NUM >= 180000 + amroutine->amcanhash = false; + amroutine->amconsistentequality = false; + amroutine->amconsistentordering = false; +#endif amroutine->amcanbackward = false; /* can change direction mid-scan */ amroutine->amcanunique = false; amroutine->amcanmulticol = false; @@ -291,6 +310,9 @@ hnswhandler(PG_FUNCTION_ARGS) amroutine->amvacuumcleanup = hnswvacuumcleanup; amroutine->amcanreturn = NULL; amroutine->amcostestimate = hnswcostestimate; +#if PG_VERSION_NUM >= 180000 + amroutine->amgettreeheight = NULL; +#endif amroutine->amoptions = hnswoptions; amroutine->amproperty = NULL; /* TODO AMPROP_DISTANCE_ORDERABLE */ amroutine->ambuildphasename = hnswbuildphasename; @@ -311,5 +333,10 @@ hnswhandler(PG_FUNCTION_ARGS) amroutine->aminitparallelscan = NULL; amroutine->amparallelrescan = NULL; +#if PG_VERSION_NUM >= 180000 + amroutine->amtranslatestrategy = NULL; + amroutine->amtranslatecmptype = NULL; +#endif + PG_RETURN_POINTER(amroutine); } diff --git a/src/hnsw.h b/src/hnsw.h index 5102bfb5a..841af0fa7 100644 --- a/src/hnsw.h +++ b/src/hnsw.h @@ -3,6 +3,8 @@ #include "postgres.h" +#include + #include "access/genam.h" #include "access/parallel.h" #include "lib/pairingheap.h" @@ -12,6 +14,10 @@ #include "utils/sampling.h" #include "vector.h" +#if PG_VERSION_NUM >= 190000 +typedef Pointer Item; +#endif + #define HNSW_MAX_DIM 2000 #define HNSW_MAX_NNZ 1000 @@ -424,7 +430,7 @@ void *HnswAlloc(HnswAllocator * allocator, Size size); HnswElement HnswInitElement(char *base, ItemPointer tid, int m, double ml, int maxLevel, HnswAllocator * alloc); HnswElement HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno); void HnswFindElementNeighbors(char *base, HnswElement element, HnswElement entryPoint, Relation index, HnswSupport * support, int m, int efConstruction, bool existing); -HnswSearchCandidate *HnswEntryCandidate(char *base, HnswElement em, HnswQuery * q, Relation rel, HnswSupport * support, bool loadVec); +HnswSearchCandidate *HnswEntryCandidate(char *base, HnswElement entryPoint, HnswQuery * q, Relation index, HnswSupport * support, bool loadVec); void HnswUpdateMetaPage(Relation index, int updateEntry, HnswElement entryPoint, BlockNumber insertPage, ForkNumber forkNum, bool building); void HnswSetNeighborTuple(char *base, HnswNeighborTuple ntup, HnswElement e, int m); void HnswAddHeapTid(HnswElement element, ItemPointer heaptid); diff --git a/src/hnswbuild.c b/src/hnswbuild.c index b667478b6..a32bf8f24 100644 --- a/src/hnswbuild.c +++ b/src/hnswbuild.c @@ -36,11 +36,12 @@ */ #include "postgres.h" -#include - +#include "access/genam.h" #include "access/parallel.h" +#include "access/relscan.h" #include "access/table.h" #include "access/tableam.h" +#include "access/tupdesc.h" #include "access/xact.h" #include "access/xloginsert.h" #include "catalog/index.h" @@ -48,11 +49,18 @@ #include "commands/progress.h" #include "hnsw.h" #include "miscadmin.h" +#include "nodes/execnodes.h" #include "optimizer/optimizer.h" #include "storage/bufmgr.h" #include "tcop/tcopprot.h" #include "utils/datum.h" #include "utils/memutils.h" +#include "utils/rel.h" +#include "utils/snapmgr.h" + +#if PG_VERSION_NUM >= 160000 +#include "varatt.h" +#endif #if PG_VERSION_NUM >= 140000 #include "utils/backend_progress.h" @@ -398,7 +406,7 @@ UpdateNeighborsInMemory(char *base, HnswSupport * support, HnswElement e, int m) * Update graph in memory */ static void -UpdateGraphInMemory(HnswSupport * support, HnswElement element, int m, int efConstruction, HnswElement entryPoint, HnswBuildState * buildstate) +UpdateGraphInMemory(HnswSupport * support, HnswElement element, int m, HnswElement entryPoint, HnswBuildState * buildstate) { HnswGraph *graph = buildstate->graph; char *base = buildstate->hnswarea; @@ -460,7 +468,7 @@ InsertTupleInMemory(HnswBuildState * buildstate, HnswElement element) HnswFindElementNeighbors(base, element, entryPoint, NULL, support, m, efConstruction, false); /* Update graph in memory */ - UpdateGraphInMemory(support, element, m, efConstruction, entryPoint, buildstate); + UpdateGraphInMemory(support, element, m, entryPoint, buildstate); /* Release entry lock */ LWLockRelease(entryLock); @@ -545,7 +553,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn /* Copy the datum */ memcpy(valuePtr, DatumGetPointer(value), valueSize); - HnswPtrStore(base, element->value, valuePtr); + HnswPtrStore(base, element->value, (char *) valuePtr); /* Create a lock for the element */ LWLockInitialize(&element->lock, hnsw_lock_tranche_id); @@ -1054,7 +1062,7 @@ ComputeParallelWorkers(Relation heap, Relation index) * Build graph */ static void -BuildGraph(HnswBuildState * buildstate, ForkNumber forkNum) +BuildGraph(HnswBuildState * buildstate) { int parallel_workers = 0; @@ -1102,7 +1110,7 @@ BuildIndex(Relation heap, Relation index, IndexInfo *indexInfo, InitBuildState(buildstate, heap, index, indexInfo, forkNum); - BuildGraph(buildstate, forkNum); + BuildGraph(buildstate); if (RelationNeedsWAL(index) || forkNum == INIT_FORKNUM) log_newpage_range(index, forkNum, 0, RelationGetNumberOfBlocksInFork(index, forkNum), true); diff --git a/src/hnswinsert.c b/src/hnswinsert.c index a5fac4eda..8bd4d24ef 100644 --- a/src/hnswinsert.c +++ b/src/hnswinsert.c @@ -1,13 +1,18 @@ #include "postgres.h" -#include - +#include "access/genam.h" #include "access/generic_xlog.h" #include "hnsw.h" +#include "nodes/execnodes.h" #include "storage/bufmgr.h" #include "storage/lmgr.h" #include "utils/datum.h" #include "utils/memutils.h" +#include "utils/rel.h" + +#if PG_VERSION_NUM >= 160000 +#include "varatt.h" +#endif /* * Get the insert page @@ -660,7 +665,7 @@ FindDuplicateOnDisk(Relation index, HnswElement element, bool building) * Update graph on disk */ static void -UpdateGraphOnDisk(Relation index, HnswSupport * support, HnswElement element, int m, int efConstruction, HnswElement entryPoint, bool building) +UpdateGraphOnDisk(Relation index, HnswSupport * support, HnswElement element, int m, HnswElement entryPoint, bool building) { BlockNumber newInsertPage = InvalidBlockNumber; @@ -708,7 +713,7 @@ HnswInsertTupleOnDisk(Relation index, HnswSupport * support, Datum value, ItemPo /* Create an element */ element = HnswInitElement(base, heaptid, m, HnswGetMl(m), HnswGetMaxLevel(m), NULL); - HnswPtrStore(base, element->value, DatumGetPointer(value)); + HnswPtrStore(base, element->value, (char *) DatumGetPointer(value)); /* Prevent concurrent inserts when likely updating entry point */ if (entryPoint == NULL || element->level > entryPoint->level) @@ -728,7 +733,7 @@ HnswInsertTupleOnDisk(Relation index, HnswSupport * support, Datum value, ItemPo HnswFindElementNeighbors(base, element, entryPoint, index, support, m, efConstruction, false); /* Update graph on disk */ - UpdateGraphOnDisk(index, support, element, m, efConstruction, entryPoint, building); + UpdateGraphOnDisk(index, support, element, m, entryPoint, building); /* Release lock */ UnlockPage(index, HNSW_UPDATE_LOCK, lockmode); diff --git a/src/hnswscan.c b/src/hnswscan.c index 955998a52..61a8d63e8 100644 --- a/src/hnswscan.c +++ b/src/hnswscan.c @@ -1,12 +1,21 @@ #include "postgres.h" +#include "access/genam.h" #include "access/relscan.h" #include "hnsw.h" +#include "lib/pairingheap.h" +#include "miscadmin.h" +#include "nodes/pg_list.h" #include "pgstat.h" -#include "storage/bufmgr.h" #include "storage/lmgr.h" #include "utils/float.h" #include "utils/memutils.h" +#include "utils/relcache.h" +#include "utils/snapmgr.h" + +#if PG_VERSION_NUM >= 160000 +#include "varatt.h" +#endif /* * Algorithm 5 from paper @@ -193,6 +202,10 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir) /* Count index scan for stats */ pgstat_count_index_scan(scan->indexRelation); +#if PG_VERSION_NUM >= 180000 + if (scan->instrument) + scan->instrument->nsearches++; +#endif /* Safety check */ if (scan->orderByData == NULL) diff --git a/src/hnswutils.c b/src/hnswutils.c index c52d2c78a..f4e4d824b 100644 --- a/src/hnswutils.c +++ b/src/hnswutils.c @@ -2,18 +2,24 @@ #include +#include "access/genam.h" #include "access/generic_xlog.h" -#include "catalog/pg_type.h" -#include "catalog/pg_type_d.h" #include "common/hashfn.h" #include "fmgr.h" #include "hnsw.h" #include "lib/pairingheap.h" +#include "nodes/pg_list.h" +#include "port/atomics.h" #include "sparsevec.h" #include "storage/bufmgr.h" #include "utils/datum.h" #include "utils/memdebug.h" #include "utils/rel.h" +#include "vector.h" + +#if PG_VERSION_NUM >= 160000 +#include "varatt.h" +#endif #if PG_VERSION_NUM < 170000 static inline uint64 @@ -256,7 +262,7 @@ HnswInitElement(char *base, ItemPointer heaptid, int m, double ml, int maxLevel, HnswInitNeighbors(base, element, m, allocator); - HnswPtrStore(base, element->value, (Pointer) NULL); + HnswPtrStore(base, element->value, (char *) NULL); return element; } @@ -282,7 +288,7 @@ HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno) element->blkno = blkno; element->offno = offno; HnswPtrStore(base, element->neighbors, (HnswNeighborArrayPtr *) NULL); - HnswPtrStore(base, element->value, (Pointer) NULL); + HnswPtrStore(base, element->value, (char *) NULL); return element; } @@ -508,7 +514,7 @@ HnswLoadElementFromTuple(HnswElement element, HnswElementTuple etup, bool loadHe char *base = NULL; Datum value = datumCopy(PointerGetDatum(&etup->data), false, -1); - HnswPtrStore(base, element->value, DatumGetPointer(value)); + HnswPtrStore(base, element->value, (char *) DatumGetPointer(value)); } } @@ -922,7 +928,7 @@ HnswSearchLayer(char *base, HnswQuery * q, List *ep, int ef, int lc, Relation in continue; } - if (eElement == NULL || !(eDistance < f->distance || alwaysAdd)) + if (!(eDistance < f->distance || alwaysAdd)) { if (discarded != NULL) { diff --git a/src/hnswvacuum.c b/src/hnswvacuum.c index 251d9d9ad..5de9b43b5 100644 --- a/src/hnswvacuum.c +++ b/src/hnswvacuum.c @@ -1,13 +1,22 @@ #include "postgres.h" -#include - +#include "access/genam.h" #include "access/generic_xlog.h" #include "commands/vacuum.h" #include "hnsw.h" +#include "nodes/pg_list.h" #include "storage/bufmgr.h" #include "storage/lmgr.h" #include "utils/memutils.h" +#include "utils/rel.h" + +#if PG_VERSION_NUM >= 160000 +#include "varatt.h" +#endif + +#if PG_VERSION_NUM >= 180000 +#define vacuum_delay_point() vacuum_delay_point(false) +#endif /* * Check if deleted list contains an index TID @@ -170,7 +179,12 @@ NeedsUpdated(HnswVacuumState * vacuumstate, HnswElement element) /* Also update if layer 0 is not full */ /* This could indicate too many candidates being deleted during insert */ if (!needsUpdated) + { + /* Keep clang-tidy happy */ + Assert(ntup->count > 0); + needsUpdated = !ItemPointerIsValid(&ntup->indextids[ntup->count - 1]); + } UnlockReleaseBuffer(buf); @@ -520,8 +534,9 @@ MarkDeleted(HnswVacuumState * vacuumstate) ntup = (HnswNeighborTuple) PageGetItem(npage, PageGetItemId(npage, neighborOffno)); /* Overwrite element */ + /* Use memset instead of MemSet to keep clang-tidy happy */ etup->deleted = 1; - MemSet(&etup->data, 0, VARSIZE_ANY(&etup->data)); + memset(&etup->data, 0, VARSIZE_ANY(&etup->data)); /* Overwrite neighbors */ for (int i = 0; i < ntup->count; i++) diff --git a/src/ivfbuild.c b/src/ivfbuild.c index 944f07b55..d3f3c2415 100644 --- a/src/ivfbuild.c +++ b/src/ivfbuild.c @@ -2,23 +2,36 @@ #include +#include "access/genam.h" +#include "access/generic_xlog.h" +#include "access/itup.h" +#include "access/relscan.h" #include "access/table.h" #include "access/tableam.h" +#include "access/tupdesc.h" #include "access/parallel.h" #include "access/xact.h" -#include "bitvec.h" +#include "access/xloginsert.h" #include "catalog/index.h" #include "catalog/pg_operator_d.h" #include "catalog/pg_type_d.h" #include "commands/progress.h" -#include "halfvec.h" +#include "fmgr.h" #include "ivfflat.h" #include "miscadmin.h" +#include "nodes/execnodes.h" #include "optimizer/optimizer.h" #include "storage/bufmgr.h" #include "tcop/tcopprot.h" #include "utils/memutils.h" -#include "vector.h" +#include "utils/rel.h" +#include "utils/sampling.h" +#include "utils/snapmgr.h" +#include "utils/tuplesort.h" + +#if PG_VERSION_NUM >= 160000 +#include "varatt.h" +#endif #if PG_VERSION_NUM >= 140000 #include "utils/backend_progress.h" @@ -138,7 +151,7 @@ SampleRows(IvfflatBuildState * buildstate) * Add tuple to sort */ static void -AddTupleToSort(Relation index, ItemPointer tid, Datum *values, IvfflatBuildState * buildstate) +AddTupleToSort(ItemPointer tid, Datum *values, IvfflatBuildState * buildstate) { double distance; double minDistance = DBL_MAX; @@ -215,7 +228,7 @@ BuildCallback(Relation index, ItemPointer tid, Datum *values, oldCtx = MemoryContextSwitchTo(buildstate->tmpCtx); /* Add tuple to sort */ - AddTupleToSort(index, tid, values, buildstate); + AddTupleToSort(tid, values, buildstate); /* Reset memory context */ MemoryContextSwitchTo(oldCtx); @@ -470,8 +483,8 @@ CreateMetaPage(Relation index, int dimensions, int lists, ForkNumber forkNum) * Create list pages */ static void -CreateListPages(Relation index, VectorArray centers, int dimensions, - int lists, ForkNumber forkNum, ListInfo * *listInfo) +CreateListPages(Relation index, VectorArray centers, int lists, + ForkNumber forkNum, ListInfo * *listInfo) { Buffer buf; Page page; @@ -1004,7 +1017,7 @@ BuildIndex(Relation heap, Relation index, IndexInfo *indexInfo, /* Create pages */ CreateMetaPage(index, buildstate->dimensions, buildstate->lists, forkNum); - CreateListPages(index, buildstate->centers, buildstate->dimensions, buildstate->lists, forkNum, &buildstate->listInfo); + CreateListPages(index, buildstate->centers, buildstate->lists, forkNum, &buildstate->listInfo); CreateEntryPages(buildstate, forkNum); /* Write WAL for initialization fork since GenericXLog functions do not */ @@ -1023,6 +1036,10 @@ ivfflatbuild(Relation heap, Relation index, IndexInfo *indexInfo) IndexBuildResult *result; IvfflatBuildState buildstate; +#ifdef IVFFLAT_BENCH + SeedRandom(42); +#endif + BuildIndex(heap, index, indexInfo, &buildstate, MAIN_FORKNUM); result = (IndexBuildResult *) palloc(sizeof(IndexBuildResult)); diff --git a/src/ivfflat.c b/src/ivfflat.c index 9e8370ffe..49c14783f 100644 --- a/src/ivfflat.c +++ b/src/ivfflat.c @@ -3,14 +3,19 @@ #include #include "access/amapi.h" +#include "access/genam.h" #include "access/reloptions.h" #include "commands/progress.h" #include "commands/vacuum.h" +#include "fmgr.h" #include "ivfflat.h" +#include "nodes/pg_list.h" #include "utils/float.h" #include "utils/guc.h" +#include "utils/relcache.h" #include "utils/selfuncs.h" #include "utils/spccache.h" +#include "vector.h" #if PG_VERSION_NUM < 150000 #define MarkGUCPrefixReserved(x) EmitWarningsOnPlaceholders(x) @@ -92,7 +97,7 @@ ivfflatcostestimate(PlannerInfo *root, IndexPath *path, double loop_count, Relation index; /* Never use index without order */ - if (path->indexorderbys == NULL) + if (path->indexorderbys == NIL) { *indexStartupCost = get_float8_infinity(); *indexTotalCost = get_float8_infinity(); @@ -186,6 +191,11 @@ ivfflathandler(PG_FUNCTION_ARGS) amroutine->amoptsprocnum = 0; amroutine->amcanorder = false; amroutine->amcanorderbyop = true; +#if PG_VERSION_NUM >= 180000 + amroutine->amcanhash = false; + amroutine->amconsistentequality = false; + amroutine->amconsistentordering = false; +#endif amroutine->amcanbackward = false; /* can change direction mid-scan */ amroutine->amcanunique = false; amroutine->amcanmulticol = false; @@ -218,6 +228,9 @@ ivfflathandler(PG_FUNCTION_ARGS) amroutine->amvacuumcleanup = ivfflatvacuumcleanup; amroutine->amcanreturn = NULL; /* tuple not included in heapsort */ amroutine->amcostestimate = ivfflatcostestimate; +#if PG_VERSION_NUM >= 180000 + amroutine->amgettreeheight = NULL; +#endif amroutine->amoptions = ivfflatoptions; amroutine->amproperty = NULL; /* TODO AMPROP_DISTANCE_ORDERABLE */ amroutine->ambuildphasename = ivfflatbuildphasename; @@ -238,5 +251,10 @@ ivfflathandler(PG_FUNCTION_ARGS) amroutine->aminitparallelscan = NULL; amroutine->amparallelrescan = NULL; +#if PG_VERSION_NUM >= 180000 + amroutine->amtranslatestrategy = NULL; + amroutine->amtranslatecmptype = NULL; +#endif + PG_RETURN_POINTER(amroutine); } diff --git a/src/ivfflat.h b/src/ivfflat.h index c296b6677..0d9601e03 100644 --- a/src/ivfflat.h +++ b/src/ivfflat.h @@ -13,6 +13,10 @@ #include "utils/tuplesort.h" #include "vector.h" +#if PG_VERSION_NUM >= 160000 +#include "varatt.h" +#endif + #if PG_VERSION_NUM >= 150000 #include "common/pg_prng.h" #endif @@ -21,6 +25,10 @@ #include "portability/instr_time.h" #endif +#if PG_VERSION_NUM >= 190000 +typedef Pointer Item; +#endif + #define IVFFLAT_MAX_DIM 2000 /* Support functions */ @@ -73,9 +81,11 @@ #if PG_VERSION_NUM >= 150000 #define RandomDouble() pg_prng_double(&pg_global_prng_state) #define RandomInt() pg_prng_uint32(&pg_global_prng_state) +#define SeedRandom(seed) pg_prng_seed(&pg_global_prng_state, seed) #else #define RandomDouble() (((double) random()) / MAX_RANDOM_VALUE) #define RandomInt() random() +#define SeedRandom(seed) srandom(seed) #endif /* Variables */ diff --git a/src/ivfinsert.c b/src/ivfinsert.c index 014c9be82..f9f08d8c1 100644 --- a/src/ivfinsert.c +++ b/src/ivfinsert.c @@ -2,11 +2,16 @@ #include +#include "access/genam.h" #include "access/generic_xlog.h" +#include "access/itup.h" +#include "fmgr.h" #include "ivfflat.h" +#include "nodes/execnodes.h" #include "storage/bufmgr.h" #include "storage/lmgr.h" #include "utils/memutils.h" +#include "utils/rel.h" /* * Find the list that minimizes the distance function @@ -65,7 +70,7 @@ FindInsertPage(Relation index, Datum *values, BlockNumber *insertPage, ListInfo * Insert a tuple into the index */ static void -InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, Relation heapRel) +InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid) { const IvfflatTypeInfo *typeInfo = IvfflatGetTypeInfo(index); IndexTuple itup; @@ -204,7 +209,7 @@ ivfflatinsert(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, oldCtx = MemoryContextSwitchTo(insertCtx); /* Insert tuple */ - InsertTuple(index, values, isnull, heap_tid, heap); + InsertTuple(index, values, isnull, heap_tid); /* Delete memory context */ MemoryContextSwitchTo(oldCtx); diff --git a/src/ivfkmeans.c b/src/ivfkmeans.c index 4b6d14f1a..293922807 100644 --- a/src/ivfkmeans.c +++ b/src/ivfkmeans.c @@ -1,17 +1,19 @@ #include "postgres.h" #include +#include #include -#include "bitvec.h" -#include "halfutils.h" -#include "halfvec.h" +#include "access/genam.h" +#include "fmgr.h" #include "ivfflat.h" #include "miscadmin.h" -#include "utils/builtins.h" -#include "utils/datum.h" #include "utils/memutils.h" -#include "vector.h" +#include "utils/relcache.h" + +#if PG_VERSION_NUM >= 160000 +#include "varatt.h" +#endif /* * Initialize with kmeans++ diff --git a/src/ivfscan.c b/src/ivfscan.c index 6cc5d2efd..dda1da2be 100644 --- a/src/ivfscan.c +++ b/src/ivfscan.c @@ -2,15 +2,26 @@ #include +#include "access/genam.h" +#include "access/itup.h" #include "access/relscan.h" +#include "access/tupdesc.h" #include "catalog/pg_operator_d.h" #include "catalog/pg_type_d.h" +#include "fmgr.h" #include "lib/pairingheap.h" #include "ivfflat.h" #include "miscadmin.h" #include "pgstat.h" #include "storage/bufmgr.h" #include "utils/memutils.h" +#include "utils/rel.h" +#include "utils/snapmgr.h" +#include "utils/tuplesort.h" + +#if PG_VERSION_NUM >= 160000 +#include "varatt.h" +#endif #define GetScanList(ptr) pairingheap_container(IvfflatScanList, ph_node, ptr) #define GetScanListConst(ptr) pairingheap_const_container(IvfflatScanList, ph_node, ptr) @@ -355,6 +366,10 @@ ivfflatgettuple(IndexScanDesc scan, ScanDirection dir) /* Count index scan for stats */ pgstat_count_index_scan(scan->indexRelation); +#if PG_VERSION_NUM >= 180000 + if (scan->instrument) + scan->instrument->nsearches++; +#endif /* Safety check */ if (scan->orderByData == NULL) diff --git a/src/ivfutils.c b/src/ivfutils.c index da241ee0a..f1586dd97 100644 --- a/src/ivfutils.c +++ b/src/ivfutils.c @@ -1,13 +1,19 @@ #include "postgres.h" +#include "access/genam.h" #include "access/generic_xlog.h" -#include "bitvec.h" -#include "catalog/pg_type.h" #include "fmgr.h" #include "halfutils.h" #include "halfvec.h" #include "ivfflat.h" #include "storage/bufmgr.h" +#include "utils/relcache.h" +#include "utils/varbit.h" +#include "vector.h" + +#if PG_VERSION_NUM >= 160000 +#include "varatt.h" +#endif /* * Allocate a vector array @@ -259,8 +265,8 @@ VectorUpdateCenter(Pointer v, int dimensions, float *x) SET_VARSIZE(vec, VECTOR_SIZE(dimensions)); vec->dim = dimensions; - for (int k = 0; k < dimensions; k++) - vec->x[k] = x[k]; + for (int i = 0; i < dimensions; i++) + vec->x[i] = x[i]; } static void @@ -271,8 +277,8 @@ HalfvecUpdateCenter(Pointer v, int dimensions, float *x) SET_VARSIZE(vec, HALFVEC_SIZE(dimensions)); vec->dim = dimensions; - for (int k = 0; k < dimensions; k++) - vec->x[k] = Float4ToHalfUnchecked(x[k]); + for (int i = 0; i < dimensions; i++) + vec->x[i] = Float4ToHalfUnchecked(x[i]); } static void @@ -284,29 +290,33 @@ BitUpdateCenter(Pointer v, int dimensions, float *x) SET_VARSIZE(vec, VARBITTOTALLEN(dimensions)); VARBITLEN(vec) = dimensions; - for (uint32 k = 0; k < VARBITBYTES(vec); k++) - nx[k] = 0; + for (uint32 i = 0; i < VARBITBYTES(vec); i++) + nx[i] = 0; - for (int k = 0; k < dimensions; k++) - nx[k / 8] |= (x[k] > 0.5 ? 1 : 0) << (7 - (k % 8)); + for (int i = 0; i < dimensions; i++) + nx[i / 8] |= (x[i] > 0.5 ? 1 : 0) << (7 - (i % 8)); } static void VectorSumCenter(Pointer v, float *x) { Vector *vec = (Vector *) v; + int dim = vec->dim; - for (int k = 0; k < vec->dim; k++) - x[k] += vec->x[k]; + /* Auto-vectorized */ + for (int i = 0; i < dim; i++) + x[i] += vec->x[i]; } static void HalfvecSumCenter(Pointer v, float *x) { HalfVector *vec = (HalfVector *) v; + int dim = vec->dim; - for (int k = 0; k < vec->dim; k++) - x[k] += HalfToFloat4(vec->x[k]); + /* Auto-vectorized on aarch64 */ + for (int i = 0; i < dim; i++) + x[i] += HalfToFloat4(vec->x[i]); } static void @@ -314,8 +324,8 @@ BitSumCenter(Pointer v, float *x) { VarBit *vec = (VarBit *) v; - for (int k = 0; k < VARBITLEN(vec); k++) - x[k] += (float) (((VARBITS(vec)[k / 8]) >> (7 - (k % 8))) & 0x01); + for (int i = 0; i < VARBITLEN(vec); i++) + x[i] += (float) (((VARBITS(vec)[i / 8]) >> (7 - (i % 8))) & 0x01); } /* diff --git a/src/ivfvacuum.c b/src/ivfvacuum.c index 1272da831..5408609d3 100644 --- a/src/ivfvacuum.c +++ b/src/ivfvacuum.c @@ -1,9 +1,16 @@ #include "postgres.h" +#include "access/genam.h" #include "access/generic_xlog.h" +#include "access/itup.h" #include "commands/vacuum.h" #include "ivfflat.h" #include "storage/bufmgr.h" +#include "utils/relcache.h" + +#if PG_VERSION_NUM >= 180000 +#define vacuum_delay_point() vacuum_delay_point(false) +#endif /* * Bulk delete tuples from the index diff --git a/src/sparsevec.c b/src/sparsevec.c index 1893752d8..ca1648f2a 100644 --- a/src/sparsevec.c +++ b/src/sparsevec.c @@ -5,18 +5,23 @@ #include "catalog/pg_type.h" #include "common/shortest_dec.h" -#include "common/string.h" #include "fmgr.h" #include "halfutils.h" #include "halfvec.h" +#include "lib/stringinfo.h" #include "libpq/pqformat.h" #include "sparsevec.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/float.h" +#include "utils/fmgrprotos.h" #include "utils/lsyscache.h" #include "vector.h" +#if PG_VERSION_NUM >= 160000 +#include "varatt.h" +#endif + typedef struct SparseInputElement { int32 index; diff --git a/src/vector.c b/src/vector.c index a5b2aac36..473683801 100644 --- a/src/vector.c +++ b/src/vector.c @@ -16,10 +16,10 @@ #include "port.h" /* for strtof() */ #include "sparsevec.h" #include "utils/array.h" -#include "utils/builtins.h" #include "utils/float.h" +#include "utils/fmgrprotos.h" #include "utils/lsyscache.h" -#include "utils/numeric.h" +#include "utils/varbit.h" #include "vector.h" #if PG_VERSION_NUM >= 160000 @@ -35,7 +35,11 @@ #define VECTOR_TARGET_CLONES #endif +#if PG_VERSION_NUM >= 180000 +PG_MODULE_MAGIC_EXT(.name = "vector",.version = "0.8.1"); +#else PG_MODULE_MAGIC; +#endif /* * Initialize index options and variables @@ -920,11 +924,13 @@ vector_concat(PG_FUNCTION_ARGS) CheckDim(dim); result = InitVector(dim); - for (int i = 0; i < a->dim; i++) + /* Auto-vectorized */ + for (int i = 0, imax = a->dim; i < imax; i++) result->x[i] = a->x[i]; - for (int i = 0; i < b->dim; i++) - result->x[i + a->dim] = b->x[i]; + /* Auto-vectorized */ + for (int i = 0, imax = b->dim, start = a->dim; i < imax; i++) + result->x[i + start] = b->x[i]; PG_RETURN_POINTER(result); } @@ -940,8 +946,21 @@ binary_quantize(PG_FUNCTION_ARGS) float *ax = a->x; VarBit *result = InitBitVector(a->dim); unsigned char *rx = VARBITS(result); + int i = 0; + int count = (a->dim / 8) * 8; - for (int i = 0; i < a->dim; i++) + /* Auto-vectorized */ + for (; i < count; i += 8) + { + unsigned char result_byte = 0; + + for (int j = 0; j < 8; j++) + result_byte |= (ax[i + j] > 0) << (7 - j); + + rx[i / 8] = result_byte; + } + + for (; i < a->dim; i++) rx[i / 8] |= (ax[i] > 0) << (7 - (i % 8)); PG_RETURN_VARBIT_P(result); diff --git a/test/expected/halfvec.out b/test/expected/halfvec.out index a3ce8931f..3ef913d6c 100644 --- a/test/expected/halfvec.out +++ b/test/expected/halfvec.out @@ -540,6 +540,12 @@ SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::halfvec) 01001110101 (1 row) +SELECT binary_quantize('[1,2,3,-4,5,6,-7,8,1,-2,-3,4,5,-6,7,8,-1,2,3]'::halfvec); + binary_quantize +--------------------- + 1110110110011011011 +(1 row) + SELECT subvector('[1,2,3,4,5]'::halfvec, 1, 3); subvector ----------- diff --git a/test/expected/hnsw_vector.out b/test/expected/hnsw_vector.out index 29ea724e9..cb4d8b97b 100644 --- a/test/expected/hnsw_vector.out +++ b/test/expected/hnsw_vector.out @@ -123,6 +123,12 @@ SELECT * FROM t ORDER BY val <-> '[3,3,3]'; [0,0,0] (3 rows) +TRUNCATE t; +SELECT * FROM t ORDER BY val <-> '[3,3,3]'; + val +----- +(0 rows) + RESET hnsw.iterative_scan; RESET hnsw.ef_search; DROP TABLE t; diff --git a/test/expected/ivfflat_vector.out b/test/expected/ivfflat_vector.out index 5a6bc757c..aad8226c1 100644 --- a/test/expected/ivfflat_vector.out +++ b/test/expected/ivfflat_vector.out @@ -110,6 +110,15 @@ SELECT * FROM t ORDER BY val <-> '[3,3,3]'; [1,1,1] (2 rows) +TRUNCATE t; +NOTICE: ivfflat index created with little data +DETAIL: This will cause low recall. +HINT: Drop the index until the table has more data. +SELECT * FROM t ORDER BY val <-> '[3,3,3]'; + val +----- +(0 rows) + RESET ivfflat.iterative_scan; RESET ivfflat.max_probes; DROP TABLE t; diff --git a/test/expected/vector_type.out b/test/expected/vector_type.out index 674865822..f4c85d035 100644 --- a/test/expected/vector_type.out +++ b/test/expected/vector_type.out @@ -576,6 +576,12 @@ SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::vector); 01001110101 (1 row) +SELECT binary_quantize('[1,2,3,-4,5,6,-7,8,1,-2,-3,4,5,-6,7,8,-1,2,3]'::vector); + binary_quantize +--------------------- + 1110110110011011011 +(1 row) + SELECT subvector('[1,2,3,4,5]'::vector, 1, 3); subvector ----------- diff --git a/test/sql/halfvec.sql b/test/sql/halfvec.sql index 1a3fd1b82..592a6e38a 100644 --- a/test/sql/halfvec.sql +++ b/test/sql/halfvec.sql @@ -121,6 +121,7 @@ SELECT l2_normalize('[65504]'::halfvec); SELECT binary_quantize('[1,0,-1]'::halfvec); SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::halfvec); +SELECT binary_quantize('[1,2,3,-4,5,6,-7,8,1,-2,-3,4,5,-6,7,8,-1,2,3]'::halfvec); SELECT subvector('[1,2,3,4,5]'::halfvec, 1, 3); SELECT subvector('[1,2,3,4,5]'::halfvec, 3, 2); diff --git a/test/sql/hnsw_vector.sql b/test/sql/hnsw_vector.sql index 184d65faf..a928ba44e 100644 --- a/test/sql/hnsw_vector.sql +++ b/test/sql/hnsw_vector.sql @@ -70,6 +70,9 @@ SELECT * FROM t ORDER BY val <-> '[3,3,3]'; SET hnsw.iterative_scan = relaxed_order; SELECT * FROM t ORDER BY val <-> '[3,3,3]'; +TRUNCATE t; +SELECT * FROM t ORDER BY val <-> '[3,3,3]'; + RESET hnsw.iterative_scan; RESET hnsw.ef_search; DROP TABLE t; diff --git a/test/sql/ivfflat_vector.sql b/test/sql/ivfflat_vector.sql index 785fc6c0f..4c20d2faf 100644 --- a/test/sql/ivfflat_vector.sql +++ b/test/sql/ivfflat_vector.sql @@ -59,6 +59,9 @@ SELECT * FROM t ORDER BY val <-> '[3,3,3]'; SET ivfflat.max_probes = 2; SELECT * FROM t ORDER BY val <-> '[3,3,3]'; +TRUNCATE t; +SELECT * FROM t ORDER BY val <-> '[3,3,3]'; + RESET ivfflat.iterative_scan; RESET ivfflat.max_probes; DROP TABLE t; diff --git a/test/sql/vector_type.sql b/test/sql/vector_type.sql index 088b040aa..086a39bcf 100644 --- a/test/sql/vector_type.sql +++ b/test/sql/vector_type.sql @@ -128,6 +128,7 @@ SELECT l2_normalize('[3e38]'::vector); SELECT binary_quantize('[1,0,-1]'::vector); SELECT binary_quantize('[0,0.1,-0.2,-0.3,0.4,0.5,0.6,-0.7,0.8,-0.9,1]'::vector); +SELECT binary_quantize('[1,2,3,-4,5,6,-7,8,1,-2,-3,4,5,-6,7,8,-1,2,3]'::vector); SELECT subvector('[1,2,3,4,5]'::vector, 1, 3); SELECT subvector('[1,2,3,4,5]'::vector, 3, 2); diff --git a/vector.control b/vector.control index 7bfc0f1fa..2ad02286a 100644 --- a/vector.control +++ b/vector.control @@ -1,4 +1,4 @@ comment = 'vector data type and ivfflat and hnsw access methods' -default_version = '0.8.0' +default_version = '0.8.1' module_pathname = '$libdir/vector' relocatable = true