diff --git a/packages/pgml-rds-proxy/ec2/.gitignore b/packages/pgml-rds-proxy/ec2/.gitignore new file mode 100644 index 000000000..b3860e0bf --- /dev/null +++ b/packages/pgml-rds-proxy/ec2/.gitignore @@ -0,0 +1,4 @@ +.terraform +*.lock.hcl +*.tfstate +*.tfstate.backup diff --git a/packages/pgml-rds-proxy/ec2/README.md b/packages/pgml-rds-proxy/ec2/README.md new file mode 100644 index 000000000..a82c64e03 --- /dev/null +++ b/packages/pgml-rds-proxy/ec2/README.md @@ -0,0 +1,7 @@ +# Terraform configuration for pgml-rds-proxy on EC2 + +This is a sample Terraform deployment for running pgml-rds-proxy on EC2. This will spin up an EC2 instance +with a public IP and a working security group & install the community Docker runtime. + +Once the instance is running, you can connect to it using the root key and run the pgml-rds-proxy Docker container +with the correct PostgresML `DATABASE_URL`. diff --git a/packages/pgml-rds-proxy/ec2/ec2-deployment.tf b/packages/pgml-rds-proxy/ec2/ec2-deployment.tf new file mode 100644 index 000000000..f724e3666 --- /dev/null +++ b/packages/pgml-rds-proxy/ec2/ec2-deployment.tf @@ -0,0 +1,84 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.46" + } + } + + required_version = ">= 1.2.0" +} + +provider "aws" { + region = "us-west-2" +} + +data "aws_ami" "ubuntu" { + most_recent = true + + filter { + name = "name" + values = ["ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } + + owners = ["099720109477"] # Canonical +} + +resource "aws_security_group" "pgml-rds-proxy" { + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + + ingress { + from_port = 6432 + to_port = 6432 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } +} + +resource "aws_instance" "pgml-rds-proxy" { + ami = data.aws_ami.ubuntu.id + instance_type = "t3.micro" + key_name = var.root_key + + root_block_device { + volume_size = 30 + delete_on_termination = true + } + + vpc_security_group_ids = [ + "${aws_security_group.pgml-rds-proxy.id}", + ] + + associate_public_ip_address = true + user_data = file("${path.module}/user_data.sh") + user_data_replace_on_change = false + + tags = { + Name = "pgml-rds-proxy" + } +} + +variable "root_key" { + type = string + description = "The name of the SSH Root Key you'd like to assign to this EC2 instance. Make sure it's a key you have access to." +} diff --git a/packages/pgml-rds-proxy/ec2/user_data.sh b/packages/pgml-rds-proxy/ec2/user_data.sh new file mode 100644 index 000000000..afa0609c0 --- /dev/null +++ b/packages/pgml-rds-proxy/ec2/user_data.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Cloud init script to install Docker on an EC2 instance running Ubuntu 22.04. +# + +sudo apt-get update +sudo apt-get install ca-certificates curl +sudo install -m 0755 -d /etc/apt/keyrings +sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc +sudo chmod a+r /etc/apt/keyrings/docker.asc + +# Add the repository to Apt sources: +echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt-get update + +sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +sudo groupadd docker +sudo usermod -aG docker ubuntu diff --git a/pgml-cms/.gitignore b/pgml-cms/.gitignore new file mode 100644 index 000000000..92ea6b2b7 --- /dev/null +++ b/pgml-cms/.gitignore @@ -0,0 +1 @@ +*.md.bak diff --git a/pgml-cms/blog/.gitbook/assets/pgml_rds_proxy_arch.png b/pgml-cms/blog/.gitbook/assets/pgml_rds_proxy_arch.png new file mode 100644 index 000000000..5552633d9 Binary files /dev/null and b/pgml-cms/blog/.gitbook/assets/pgml_rds_proxy_arch.png differ diff --git a/pgml-cms/docs/.gitbook/assets/architecture.png b/pgml-cms/docs/.gitbook/assets/architecture.png index d69edebdb..de7da35c2 100644 Binary files a/pgml-cms/docs/.gitbook/assets/architecture.png and b/pgml-cms/docs/.gitbook/assets/architecture.png differ diff --git a/pgml-cms/docs/.gitbook/assets/fdw_1.png b/pgml-cms/docs/.gitbook/assets/fdw_1.png new file mode 100644 index 000000000..0dbce2380 Binary files /dev/null and b/pgml-cms/docs/.gitbook/assets/fdw_1.png differ diff --git a/pgml-cms/docs/.gitbook/assets/logical_replication_1.png b/pgml-cms/docs/.gitbook/assets/logical_replication_1.png new file mode 100644 index 000000000..ac15be858 Binary files /dev/null and b/pgml-cms/docs/.gitbook/assets/logical_replication_1.png differ diff --git a/pgml-cms/docs/.gitbook/assets/vpc_1.png b/pgml-cms/docs/.gitbook/assets/vpc_1.png new file mode 100644 index 000000000..5137d49b5 Binary files /dev/null and b/pgml-cms/docs/.gitbook/assets/vpc_1.png differ diff --git a/pgml-cms/docs/README.md b/pgml-cms/docs/README.md index 492186ec3..6e14c3258 100644 --- a/pgml-cms/docs/README.md +++ b/pgml-cms/docs/README.md @@ -4,38 +4,50 @@ description: The key concepts that make up PostgresML. # Overview -PostgresML is a complete MLOps platform built on PostgreSQL. +PostgresML is a complete MLOps platform built on PostgreSQL. Our operating principle is: -> _Move the models to the database, rather than continuously moving the data to the models._ +> _Move the models to the database, rather than constantly moving the data to the models._ -The data for ML & AI systems is inherently larger and more dynamic than the models. It's more efficient, manageable and reliable to move the models to the database, rather than continuously moving the data to the models. PostgresML allows you to take advantage of the fundamental relationship between data and models, by extending the database with the following capabilities and goals: +The data for ML & AI systems is inherently larger and more dynamic than the models. It's more efficient, manageable and reliable to move the models to the database, rather than continuously moving data to the models. -* **Model Serving** - _**GPU accelerated**_ inference engine for interactive applications, with no additional networking latency or reliability costs. -* **Model Store** - Download _**open-source**_ models including state of the art LLMs from HuggingFace, and track changes in performance between versions. -* **Model Training** - Train models with _**your application data**_ using more than 50 algorithms for regression, classification or clustering tasks. Fine tune pre-trained models like LLaMA and BERT to improve performance. -* **Feature Store** - _**Scalable**_ access to model inputs, including vector, text, categorical, and numeric data. Vector database, text search, knowledge graph and application data all in one _**low-latency**_ system. +## AI engine -
Machine Learning Infrastructure (2.0) by a16z

PostgresML handles all of the functions typically performed by a cacophony of services, described by a16z

+PostgresML allows you to take advantage of the fundamental relationship between data and models, by extending the database with the following capabilities: -These capabilities are primarily provided by two open-source software projects, that may be used independently, but are designed to be used with the rest of the Postgres ecosystem, including trusted extensions like pgvector and pg\_partman. +* **Model Serving** - GPU accelerated inference engine for interactive applications, with no additional networking latency or reliability costs +* **Model Store** - Access to open-source models including state of the art LLMs from HuggingFace, and track changes in performance between versions +* **Model Training** - Train models with your application data using more than 50 algorithms for regression, classification or clustering tasks; fine tune pre-trained models like LLaMA and BERT to improve performance +* **Feature Store** - Scalable access to model inputs, including vector, text, categorical, and numeric data: vector database, text search, knowledge graph and application data all in one low-latency system -* **pgml** is an open source extension for PostgreSQL. It adds support for GPUs and the latest ML & AI algorithms _**inside**_ the database with a SQL API and no additional infrastructure, networking latency, or reliability costs. -* **PgCat** is an open source proxy pooler for PostgreSQL. It abstracts the scalability and reliability concerns of managing a distributed cluster of Postgres databases. Client applications connect only to the proxy, which handles load balancing and failover, _**outside**_ of any single database. +
Machine Learning Infrastructure (2.0) by a16z

PostgresML handles all of the functions described by a16z

-
PostgresML architectural diagram

A PostgresML deployment at scale

+These capabilities are primarily provided by two open-source software projects, that may be used independently, but are designed to be used with the rest of the Postgres ecosystem: -In addition, PostgresML provides [native language SDKs](https://github.com/postgresml/postgresml/tree/master/pgml-sdks/pgml) to implement best practices for common ML & AI applications. The JavaScript and Python SDKs are generated from the core Rust SDK, to provide the same API, correctness and efficiency across all application runtimes. +* **pgml** - an open source extension for PostgreSQL. It adds support for GPUs and the latest ML & AI algorithms _inside_ the database with a SQL API and no additional infrastructure, networking latency, or reliability costs +* **PgCat** - an open source pooler for PostgreSQL. It abstracts the scalability and reliability concerns of managing a distributed cluster of Postgres databases. Client applications connect only to the pooler, which handles load balancing, sharding, and failover, outside of any single database server. -SDK clients can perform advanced machine learning tasks in a single SQL request, without having to transfer additional data, models, hardware or dependencies to the client application. For example: +
PostgresML architectural diagram
-* Chat with streaming response support from the latest LLMs -* Search with both keywords and embedding vectors -* Text Generation with RAG in a single request -* Translate text between hundreds of language pairs -* Summarization to distil complex documents -* Forecasting timeseries data for key metrics with complex metadata -* Fraud and anomaly detection with application data +## Client SDK -Our goal is to provide access to Open Source AI for everyone. PostgresML is under continuous development to keep up with the rapidly evolving use cases for ML & AI, and we release non breaking changes with minor version updates in accordance with SemVer. We welcome contributions to our [open source code and documentation](https://github.com/postgresml). +The PostgresML team also provides [native language SDKs](https://github.com/postgresml/postgresml/tree/master/pgml-sdks/pgml) which implement best practices for common ML & AI applications. The JavaScript and Python SDKs are generated from the a core Rust library, which provides a uniform API, correctness and efficiency across all environments. -We can host your AI database in our cloud, or you can run our Docker image locally with PostgreSQL, pgml, pgvector and NVIDIA drivers included. +While using the SDK is completely optional, SDK clients can perform advanced machine learning tasks in a single SQL request, without having to transfer additional data, models, hardware or dependencies to the client application. + +Use cases include: + +* Chat with streaming responses from state-of-the-art open source LLMs +* Semantic search with keywords and embeddings +* RAG in a single request without using any third-party services +* Text translation between hundreds of languages +* Text summarization to distill complex documents +* Forecasting timeseries data for key metrics with and metadata +* Anomaly detection using application data + +## Our mission + +PostgresML strives to provide access to open source AI for everyone. We are continuously developping PostgresML to keep up with the rapidly evolving use cases for ML & AI, but we remain committed to never breaking user facing APIs. We welcome contributions to our [open source code and documentation](https://github.com/postgresml) from the community. + +## Managed cloud + +While our extension and pooler are open source, we also offer a managed cloud database service for production deployments of PostgresML. You can [sign up](https://postgresml.org/signup) for an account and get a free Serverless database in seconds. diff --git a/pgml-cms/docs/SUMMARY.md b/pgml-cms/docs/SUMMARY.md index 174e2f7e1..c278fadb1 100644 --- a/pgml-cms/docs/SUMMARY.md +++ b/pgml-cms/docs/SUMMARY.md @@ -6,9 +6,11 @@ * [Getting Started](introduction/getting-started/README.md) * [Create your database](introduction/getting-started/create-your-database.md) * [Connect your app](introduction/getting-started/connect-your-app.md) - * [Import your data](introduction/getting-started/import-your-data/README.md) - * [CSV](introduction/getting-started/import-your-data/csv.md) - * [Foreign Data Wrapper](introduction/getting-started/import-your-data/foreign-data-wrapper.md) +* [Import your data](introduction/getting-started/import-your-data/README.md) + * [Logical replication](introduction/getting-started/import-your-data/logical-replication/README.md) + * [Foreign Data Wrappers](introduction/getting-started/import-your-data/foreign-data-wrappers.md) + * [Move data with COPY](introduction/getting-started/import-your-data/copy.md) + * [Migrate with pg_dump](introduction/getting-started/import-your-data/pg-dump.md) ## API @@ -51,7 +53,7 @@ ## Product * [Cloud Database](product/cloud-database/README.md) - * [Serverless databases](product/cloud-database/serverless-databases.md) + * [Serverless](product/cloud-database/serverless.md) * [Dedicated](product/cloud-database/dedicated.md) * [Enterprise](product/cloud-database/plans.md) * [Vector Database](product/vector-database.md) @@ -79,7 +81,7 @@ ## Resources * [FAQs](resources/faqs.md) -* [Data Storage & Retrieval](resources/data-storage-and-retrieval/README.md) +* [Data Storage & Retrieval](resources/data-storage-and-retrieval/tabular-data.md) * [Tabular data](resources/data-storage-and-retrieval/tabular-data.md) * [Documents](resources/data-storage-and-retrieval/documents.md) * [Partitioning](resources/data-storage-and-retrieval/partitioning.md) diff --git a/pgml-cms/docs/api/client-sdk/README.md b/pgml-cms/docs/api/client-sdk/README.md index fc2ee4134..01286e9cb 100644 --- a/pgml-cms/docs/api/client-sdk/README.md +++ b/pgml-cms/docs/api/client-sdk/README.md @@ -1,4 +1,4 @@ -# Client SDKs +# Client SDK ### Key Features diff --git a/pgml-cms/docs/api/client-sdk/getting-started.md b/pgml-cms/docs/api/client-sdk/getting-started.md index 326a76ac3..fd2f590ae 100644 --- a/pgml-cms/docs/api/client-sdk/getting-started.md +++ b/pgml-cms/docs/api/client-sdk/getting-started.md @@ -18,7 +18,7 @@ pip install pgml ## Example -Once the SDK is installed, you an use the following example to get started. +Once the SDK is installed, you can use the following example to get started. ### Create a collection @@ -85,7 +85,7 @@ await collection.add_pipeline(pipeline) {% endtab %} {% endtabs %} -#### Explanation: +#### Explanation * The code constructs a pipeline called `"sample_pipeline"` and adds it to the collection we Initialized above. This pipeline automatically generates chunks and embeddings for the `text` key for every upserted document. diff --git a/pgml-cms/docs/introduction/getting-started/README.md b/pgml-cms/docs/introduction/getting-started/README.md index b83c2290f..ec0997468 100644 --- a/pgml-cms/docs/introduction/getting-started/README.md +++ b/pgml-cms/docs/introduction/getting-started/README.md @@ -4,14 +4,14 @@ description: Setup a database and connect your application to PostgresML # Getting Started -A PostgresML deployment consists of multiple components working in concert to provide a complete Machine Learning platform. We provide a fully managed solution in our cloud. +A PostgresML deployment consists of multiple components working in concert to provide a complete Machine Learning platform. We provide a fully managed solution in [our cloud](create-your-database), and document a self-hosted installation in [Developer Docs](/docs/resources/developer-docs/quick-start-with-docker). -* A PostgreSQL database, with pgml and pgvector extensions installed, including backups, metrics, logs, replicas and high availability configurations -* A PgCat pooling proxy to provide secure access and model load balancing across tens of thousands of clients -* A web application to manage deployed models and host SQL notebooks +* PostgreSQL database, with `pgml`, `pgvector` and many other extensions installed, including backups, metrics, logs, replicas and high availability +* PgCat pooler to provide secure access and model load balancing across thousands of clients +* A web application to manage deployed models and share experiments and analysis in SQL notebooks -
+
PostgresML architecture
By building PostgresML on top of a mature database, we get reliable backups for model inputs and proven scalability without reinventing the wheel, so that we can focus on providing access to the latest developments in open source machine learning and artificial intelligence. -This guide will help you get started with a generous free account, that includes access to GPU accelerated models and 5GB of storage, or you can skip to our Developer Docs to see how to run PostgresML locally with our Docker image. +This guide will help you get started with a generous free account, that includes access to GPU accelerated models and 5 GB of storage, or you can skip to our [Developer Docs](/docs/resources/developer-docs/quick-start-with-docker) to see how to run PostgresML locally with our Docker image. diff --git a/pgml-cms/docs/introduction/getting-started/connect-your-app.md b/pgml-cms/docs/introduction/getting-started/connect-your-app.md index 1613f5501..c0b003220 100644 --- a/pgml-cms/docs/introduction/getting-started/connect-your-app.md +++ b/pgml-cms/docs/introduction/getting-started/connect-your-app.md @@ -4,16 +4,16 @@ description: PostgresML is compatible with all standard PostgreSQL clients # Connect your app -You can connect to your database from any Postgres compatible client. PostgresML is intended to serve in the traditional role of an application database, along with it's extended role as an MLOps platform to make it easy to build and maintain AI applications. +You can connect to your PostgresML database from any PostgreSQL-compatible client. PostgresML can serve in the traditional role of an application database, along with it's extended role as an MLOps platform, to make it easy to build and maintain AI applications together with your application data. -## Application SDKs +## Client SDK -We provide client SDKs for JavaScript, Python and Rust apps that manage connections to the Postgres database and make it easy to construct efficient queries for AI use cases, like managing a document collection for RAG, or building a chatbot. All of the ML & AI still happens in the database, with centralized operations, hardware and dependency management. - -These SDKs are under rapid development to add new features and use cases, but we release non breaking changes with minor version updates in accordance with SemVer. It's easy to install into your existing application. +We provide a client SDK for JavaScript, Python and Rust. The SDK manages connections to the database, and makes it easy to construct efficient queries for AI use cases, like managing RAG document collections, or building chatbots. All of the ML & AI still happens inside the database, with centralized operations, hardware and dependency management. ### Installation +The SDK is available from npm and PyPI: + {% tabs %} {% tab title="JavaScript" %} ```bash @@ -28,8 +28,12 @@ pip install pgml {% endtab %} {% endtabs %} +Our SDK comes with zero additional dependencies. The core of the SDK is written in Rust, and we provide language bindings and native packaging & distribution. + ### Test the connection +Once you have installed our SDK into your environment, you can test connectivity to our cloud with just a few lines of code: + {% tabs %} {% tab title="JavaScript" %} ```javascript @@ -80,9 +84,9 @@ async def main(): {% endtab %} {% endtabs %} -## Native Language Bindings +## Native PostgreSQL libraries -You can also connect directly to the database with your favorite bindings or ORM: +Using the SDK is completely optional. If you're comfortable with writing SQL, you can connect directly to the database using your favorite PostgreSQL client library or ORM: * C++: [libpqxx](https://www.tutorialspoint.com/postgresql/postgresql\_c\_cpp.htm) * C#: [Npgsql](https://github.com/npgsql/npgsql),[Dapper](https://github.com/DapperLib/Dapper), or [Entity Framework Core](https://github.com/dotnet/efcore) @@ -101,9 +105,9 @@ You can also connect directly to the database with your favorite bindings or ORM * Rust: [postgres](https://crates.io/crates/postgres), [SQLx](https://github.com/launchbadge/sqlx) or [Diesel](https://github.com/diesel-rs/diesel) * Swift: [PostgresNIO](https://github.com/vapor/postgres-nio) or [PostgresClientKit](https://github.com/codewinsdotcom/PostgresClientKit) -## SQL Editors +## SQL editors -Use any of these popular tools to execute SQL queries directly against the database: +If you need to write ad-hoc queries, you can use any of these popular tools to execute SQL queries directly on your database: * [Apache Superset](https://superset.apache.org/) * [DBeaver](https://dbeaver.io/) diff --git a/pgml-cms/docs/introduction/getting-started/create-your-database.md b/pgml-cms/docs/introduction/getting-started/create-your-database.md index 48d5d21a5..01e8c53f4 100644 --- a/pgml-cms/docs/introduction/getting-started/create-your-database.md +++ b/pgml-cms/docs/introduction/getting-started/create-your-database.md @@ -8,27 +8,25 @@ description: >- ## Sign up for an account -Visit [https://postgresml.org/signup](https://postgresml.org/signup)​ to create a new account with your email, Google or Github authentication. +Visit [https://postgresml.org/signup](https://postgresml.org/signup) to create a new account with your email, Google or GitHub.
- -
Sign up
- +
Sign up
## Select a plan -Choose the type of GPU powered database deployment that is right for you. +Choose the type of GPU powered database deployment that is right for you: -* **Serverless** is the easiest way to get started. We offer a generous free tier with GPU access and 5GB of data storage. -* **Dedicated** offers additional configuration options for more advanced use cases with established workloads and more predictable usage patterns. +* **Serverless** is the easiest way to get started. We offer a generous free tier with GPU access and 5 GB of data storage +* **Dedicated** offers additional configuration options for more advanced use cases with established workloads and more predictable usage patterns Click on **Get Started** under the plan of your choice.
-## Your database credentials +## Database access credentials -We'll automatically provision an initial set of database credentials and provide you with the connection string. You can connect to your database if you have `psql` installed on your machine, or any other PostgreSQL client. +PostgresML Cloud automatically provisions database credentials and provides you with the `DATABASE_URL` connection string. You can connect to your database with `psql`, any other PostgreSQL client library, or application.
diff --git a/pgml-cms/docs/introduction/getting-started/import-your-data/README.md b/pgml-cms/docs/introduction/getting-started/import-your-data/README.md index f9d1d3425..49d2cd15e 100644 --- a/pgml-cms/docs/introduction/getting-started/import-your-data/README.md +++ b/pgml-cms/docs/introduction/getting-started/import-your-data/README.md @@ -1,22 +1,34 @@ # Import your data -Machine learning always depends on input data, whether it's generating text with pretrained LLMs, training a retention model on customer data, or predicting session abandonment in real time. Just like any PostgreSQL database, PostgresML can be configured as the authoritative application data store, a streaming replica from some other primary, or use foreign data wrappers to query another data host on demand. Depending on how frequently your data changes and where your authoritative data resides, different methodologies imply different tradeoffs. +AI needs data, whether it's generating text with LLMs, creating embeddings, or training regression or classification models on customer data. -PostgresML can easily ingest data from your existing data stores. +Just like any PostgreSQL database, PostgresML can be configured as the primary application database, a logical replica of your primary database, or with foreign data wrappers to query your primary database on demand. Depending on how frequently your data changes and your latency requirements, one approach is better than the other. -## Static data +## Primary database -Data that changes infrequently can be easily imported into PostgresML using `COPY`. All you have to do is export your data as a CSV file, create a table in Postgres to store it, and import it using the command line. +If your intention is to use PostgresML as your primary database, your job here is done. You can use the connection credentials provided and start building your application on top of in-database AI right away. -{% content-ref url="csv.md" %} -[csv.md](csv.md) -{% endcontent-ref %} +## [Logical replica](logical-replication/) -## Live data +If your primary database is hosted elsewhere, for example AWS RDS, or Azure Postgres, you can get your data replicated to PostgresML in real time using logical replication. -Importing data from online databases can be done with foreign data wrappers. Hosted PostgresML databases come with both `postgres_fdw` and `dblink` extensions pre-installed, so you can import data from any of your existing Postgres databases, and export machine learning artifacts from PostgresML using just a few lines of SQL. +
Logical replication
-{% content-ref url="foreign-data-wrapper.md" %} -[foreign-data-wrapper.md](foreign-data-wrapper.md) -{% endcontent-ref %} +Having access to your data immediately is very useful to +accelerate your machine learning use cases and removes the need for moving data multiple times between microservices. Latency-sensitive applications should consider using this approach. +## [Foreign data wrappers](foreign-data-wrappers) + +Foreign data wrappers are a set of PostgreSQL extensions that allow making direct connections from inside the database directly to other databases, even if they aren't running on Postgres. For example, Postgres has foreign data wrappers for MySQL, S3, Snowflake and many others. + +
Foreign data wrappers
+ +FDWs are useful when data access is infrequent and not latency-sensitive. For many use cases, like offline batch workloads and not very busy websites, this approach is suitable and easy to get started with. + +## [Move data with COPY](copy) + +`COPY` is a powerful PostgreSQL command to import data from a file format like CSV. Most data stores out there support exporting data using the CSV format, so moving data from your data source to PostgresML can almost always be done this way. + +## [Migrate with pg_dump](pg-dump) + +_pg_dump_ is a command-line PostgreSQL utility to migrate databases from one server to another. Databases of almost any size can be migrated with _pg_dump_ quickly and safely. diff --git a/pgml-cms/docs/introduction/getting-started/import-your-data/copy.md b/pgml-cms/docs/introduction/getting-started/import-your-data/copy.md new file mode 100644 index 000000000..131c4a0fd --- /dev/null +++ b/pgml-cms/docs/introduction/getting-started/import-your-data/copy.md @@ -0,0 +1,75 @@ +# Move data with COPY + +Data that changes infrequently can be easily imported into PostgresML (and any other Postgres database) using `COPY`. All you have to do is export your data as a file, create a table in Postgres to store it, and import it using the command line (or your IDE of choice). + +## Getting started + +We'll be using CSV as our data format of choice. CSV is a supported mechanism for data transport in pretty much every database and system in existence, so you won't have any trouble finding the CSV export functionality in your current data store. + +Let's use a simple CSV file with 3 columns as an example: + +| Column | Data type | Example data | +| ---------------- | --------- | ------- | +| name | text | John | +| age | integer | 30 | +| is\_paying\_user | boolean | true | + +### Export data + +If you're using a Postgres database already, you can export any table as CSV with just one command: + +```bash +psql \ + postgres://user:password@your-production-db.amazonaws.com \ + -c "\copy (SELECT * FROM users) TO '~/users.csv' CSV HEADER" +``` + +If you're using another data store, it will almost always provide a CSV export functionality. + +### Create table in PostgresML + +Create a table in PostgresML with the correct schema: + + +{% tabs %} +{% tab title="SQL" %} + +```postgresql +CREATE TABLE users( + name TEXT, + age INTEGER, + is_paying_user BOOLEAN +); +``` + +{% endtab %} +{% tab title="Output" %} + +``` +CREATE TABLE +``` + +{% endtab %} +{% endtabs %} + +Data types should roughly match to what you have in your CSV file. If the data type is not known, you can always use `TEXT` and figure out what it is later with a few queries. Postgres also supports converting data types, as long as they are formatted correctly. + +### Import data + +Once you have a table and your data exported as CSV, importing it can also be done with just one command: + +```bash +psql \ + postgres://user:password@sql.cloud.postgresml.org/your_pgml_database \ + -c "\copy your_table FROM '~/your_table.csv' CSV HEADER" +``` + +We took our export command and changed `TO` to `FROM`, and that's it. Make sure you're connecting to your PostgresML database when importing data. + +## Refresh data + +If your data changed, repeat this process again. To avoid duplicate entries in your table, you can truncate (or delete) all rows beforehand: + +```sql +TRUNCATE your_table; +``` diff --git a/pgml-cms/docs/introduction/getting-started/import-your-data/csv.md b/pgml-cms/docs/introduction/getting-started/import-your-data/csv.md deleted file mode 100644 index 7c77b776b..000000000 --- a/pgml-cms/docs/introduction/getting-started/import-your-data/csv.md +++ /dev/null @@ -1,53 +0,0 @@ -# CSV - -## Static data - -Data that changes infrequently can be easily imported into PostgresML using `COPY`. All you have to do is export your data as a CSV file, create a table in Postgres to store it, and import it using the command line. - -Let's use a simple CSV file with 3 columns as an example: - -| Column | Data type | Example | -| ---------------- | --------- | ------- | -| name | text | John | -| age | integer | 30 | -| is\_paying\_user | boolean | true | - -### Export data as CSV - -If you're using a Postgres database already, you can export any table as CSV with just one command: - -```bash -psql -c "\copy your_table TO '~/Desktop/your_table.csv' CSV HEADER" -``` - -If you're using another data store, it should almost always provide a CSV export functionality, since CSV is the most commonly used data format in machine learning. - -### Create table in Postgres - -Creating a table in Postgres with the correct schema is as easy as: - -```sql -CREATE TABLE your_table ( - name TEXT, - age INTEGER, - is_paying_user BOOLEAN -); -``` - -### Import data using the command line - -Once you have a table and your data exported as CSV, importing it can also be done with just one command: - -```bash -psql -c "\copy your_table FROM '~/Desktop/your_table.csv' CSV HEADER" -``` - -We took our export command and changed `TO` to `FROM`, and that's it. Make sure you're connecting to your PostgresML database when importing data. - -### Refreshing data - -If your data changed, repeat this process again. To avoid duplicate entries in your table, you can truncate (or delete) all rows beforehand: - -```sql -TRUNCATE your_table; -``` diff --git a/pgml-cms/docs/introduction/getting-started/import-your-data/foreign-data-wrapper.md b/pgml-cms/docs/introduction/getting-started/import-your-data/foreign-data-wrapper.md deleted file mode 100644 index 4b6f16365..000000000 --- a/pgml-cms/docs/introduction/getting-started/import-your-data/foreign-data-wrapper.md +++ /dev/null @@ -1,87 +0,0 @@ -# Foreign Data Wrapper - -## Setting up - -Before you get started with foreign data wrappers, log into your current database hosting provider and grab the following connection details: - -* Host -* Port (typically `5432`) -* Database name -* Postgres user -* Postgres password - -Once you have them, we can setup our live foreign data wrapper connection. All following commands should be executed on your PostgesML database. You don't need to perform any additional steps on your production database. - -### Connecting - -To connect to your database from PostgresML, first create a corresponding `SERVER`: - -```sql -CREATE SERVER live_db -FOREIGN DATA WRAPPER postgres_fdw -OPTIONS ( - host 'Host', - port 'Port', - dbname 'Database name' -); -``` - -Replace `Host`, `Port` and `Database name` with details you've collected in the previous step. - -Once you have a `SERVER`, let's authenticate to your database: - -```sql -CREATE USER MAPPING -FOR CURRENT_USER -SERVER live_db -OPTIONS ( - user 'Postgres user', - password 'Postgres password' -); -``` - -Replace `Postgres user` and `Postgres password` with details collected in the previous step. If everything went well, we'll be able to validate that everything is working with just one query: - -```sql -SELECT * FROM dblink( - 'live_db', - 'SELECT 1 AS one' -) AS t1(one INTEGER); -``` - -You can now execute any query you want on your live database from inside your PostgresML database. - -### Working with your tables - -Instead of creating temporary tables for each query, you can import your entire schema into PostgresML using foreign data wrappers: - -```sql -CREATE SCHEMA live_db_tables; - -IMPORT FOREIGN SCHEMA public -FROM SERVER live_db -INTO live_db_tables; -``` - -All your tables from your `public` schema are now available in the `live_db_tables` schema. You can read and write to those tables as if they were hosted in PostgresML. For example, if you have a table called `users`, you could access it with: - -```sql -SELECT * FROM live_db_tables.users LIMIT 1; -``` - -That's it, your PostgresML database is directly connected to your production database and you can start your machine learning journey. - -### Accelerating bulk access - -To speed up access to your data, you can cache it in PostgresML by copying it from a foreign table into a regular table. Taking the example of the `users` table: - -```sql -CREATE TABLE public.users (LIKE live_db_tables.users); -INSERT INTO public.users SELECT * FROM live_db_tables.users; -``` - -This will copy all rows from your `users` table into PostgresML. You'll be able to access them much quicker if you need to perform a batch job like generating embeddings or training a supervised model. - -### Exporting ML artifacts - -If you want to export some artifacts you've created with PostresML to your live database, you can do so with foreign data wrappers as well. Simply copy them using the same mechanism as above, except instead of copying data from the foreign schema, copy data into the foreign schema from the regular table. diff --git a/pgml-cms/docs/introduction/getting-started/import-your-data/foreign-data-wrappers.md b/pgml-cms/docs/introduction/getting-started/import-your-data/foreign-data-wrappers.md new file mode 100644 index 000000000..e6d068e88 --- /dev/null +++ b/pgml-cms/docs/introduction/getting-started/import-your-data/foreign-data-wrappers.md @@ -0,0 +1,192 @@ +# Foreign Data Wrappers + +Foreign data wrappers are a set of Postgres extensions that allow making direct connections to other databases from inside your PostgresML database. Other databases can be your production Postgres database on RDS or Azure, or another database engine like MySQL, Snowflake, or even an S3 bucket. + +
+ +## Getting started + +A foreign data wrapper connection from PostgresML to another Postgres database requires very little configuration. If your database is accessible from the Internet (like Neon, Supabase, and some AWS RDS & Azure Postgres configurations), you can just grab your connection details from your cloud provider dashboard and create a connection in your PostgresML database with a few SQL commands. + +### Create a FDW connection + +An FDW connection consists of two configuration components: the _server_ which will define where your production database is located and a _user mapping_ which will define which user & password the connection should use to authenticate to your Postgres database. + +FDWs don't require any special configuration on your production database, so all commands below need to be executed on your PostgresML database, not your production database. + +#### Create the server + +To create the server configuration, take the command below, replace the values for `host`, `port`, and `dbname` with the hostname, port (typically _5432_), and Postgres database name of your production database, and run it on your PostgresML database: + +{% tabs %} +{% tab title="SQL" %} + +```postgresql +CREATE SERVER production_db +FOREIGN DATA WRAPPER postgres_fdw +OPTIONS ( + host 'your-production-db.amazonaws.com', + port '5432' + dbname 'production_db' +); +``` + +{% endtab %} +{% tab title="Output" %} + +``` +CREATE SERVER +``` + +{% endtab %} +{% endtabs %} + +Once you have a server, you need to configure authentication for your current user (and any other user you may have created in your PostgresML database). + +#### Create a user mapping + +To create a user mapping, take the command below, replace the values for `user` and `password` and replace them with your actual production user & password. This user doesn't have to be a superuser, and can only have `SELECT` & `USAGE` permissions on your tables and schema. + +{% tabs %} +{% tab title="SQL" %} + +```postgresql +CREATE USER MAPPING +FOR CURRENT_USER +SERVER production_db +OPTIONS ( + user 'readonly_user', + password 'secret_password' +); +``` + +{% endtab %} +{% tab title="Output" %} + +``` +CREATE USER MAPPING +``` + +{% endtab %} +{% endtabs %} + +### Check connectivity + +If everything went well, you should be able to connect to your Postgres database from PostgresML: + +{% tabs %} +{% tab title="SQL" %} + +```postgresql +SELECT * +FROM dblink( + 'production_db', + 'SELECT 1 AS one' +) AS t1(one INTEGER); +``` + +{% endtab %} +{% tab title="Output" %} + +``` + one +----- + 1 +(1 row) +``` + +{% endtab %} +{% endtabs %} + +_dblink_ is another extension that can execute arbitrary queries on databases connected with foreign data wrappers. It's great if you want to fetch some data on demand, but it does require you to write your query & table schema every time, which can be a little tedious. + +Thankfully, this problem has been already solved with another feature of FDWs which removes the need to specify your schema with every query: _foreign tables_. + +### Add your tables + +Foreign tables are table schemas that tell your database that the data is actually located in another database. For each query that touches those tables, the FDW extension will take care of fetching the data from your production database in the most efficient way possible, and combine it with data from your PostgresML tables. + +There are two ways to specify foreign tables: create them one by one with `CREATE FOREIGN TABLE` command or by importing all of them using `IMPORT FOREIGN SCHEMA` command. Unless you have some special user permissions that don't allow the user we've configured in the _user mapping_ above to access all your tables, we recommend you use the second option to import all your tables. + +#### Import tables + +Table import requires two steps: create a schema to host the tables, and import the tables from your database using the FDW connection: + +{% tabs %} +{% tab title="SQL" %} + +```postgresql +CREATE SCHEMA production_tables; + +IMPORT FOREIGN SCHEMA public +FROM SERVER production_db +INTO production_tables; +``` + +{% endtab %} +{% tab title="Output" %} + +``` +CREATE SCHEMA +IMPORT FOREIGN SCHEMA +``` + +{% endtab %} +{% endtabs %} + +If everything went well, your tables should appear in the `production_tables` schema. You can now use them in normal queries without worrying about data types or column names. + +### Accelerate bulk access + +Foreign data wrappers make connections to your database as needed to fetch data. This can add latency when fetching a lot of data at once. If you need to run some kind of batch job, for example to generate embeddings using `pgml.embed()`, it's best to first copy your table data over into your PostgresML database. Using an example of a `users` table, FDWs make that as easy as: + +{% tabs %} +{% tab title="SQL" %} + +```postgresql +CREATE TABLE bulk_access_users ( + LIKE production_tables.users +); + +INSERT INTO bulk_access_users +SELECT * FROM production_tables.users; +``` + +{% endtab %} +{% tab title="Output" %} + +``` +CREATE TABLE +INSERT 0 1000 +``` + +{% endtab %} +{% endtabs %} + +You can now add an embedding column to `bulk_access_users` and generate embeddings for your users using just one query: + +{% tabs %} +{% tab title="SQL" %} + +```postgresql +ALTER TABLE bulk_access_users +ADD COLUMN embedding vector(384); + +UPDATE bulk_access_users +SET embedding = pgml.embed('intfloat/e5-small', email); +``` + +{% endtab %} +{% tab title="Output" %} + +``` +ALTER TABLE +UPDATE 1000 +``` + +{% endtab %} +{% endtabs %} + +Once embedding generation is complete, you can copy the vectors back into your production database using similar SQL commands, just in reverse. + +If you want to use embeddings as part of a real time application, e.g. semantic search, you should add the PostgresML database into your application and connect to it directly instead. diff --git a/pgml-cms/docs/introduction/getting-started/import-your-data/logical-replication/README.md b/pgml-cms/docs/introduction/getting-started/import-your-data/logical-replication/README.md new file mode 100644 index 000000000..fceec7f42 --- /dev/null +++ b/pgml-cms/docs/introduction/getting-started/import-your-data/logical-replication/README.md @@ -0,0 +1,126 @@ +# Logical replication + +Logical replication allows your PostgresML database to copy data from your primary database to PostgresML in real time. As soon as your customers make changes to their data on your website, those changes will become available in PostgresML. + +
+ +## Getting started + +Setting up & maintaining logical replication requires a few steps, but once you're done, you'll be able to generate embeddings, train models & generate text using LLMs directly using your production data. + +### Configure your primary database + +First things first, make sure your primary database is configured to support logical replication. To do so, make sure the following settings are set: + + +| Setting | Value | +|-------------------------|----------------| +| `wal_level` | `logical` | +| `wal_senders` | Greater than 0 | +| `max_replication_slots` | Greater than 0 | +| `rds.logical_replicationion` (only on AWS RDS) | `1` | + +Make sure to **restart your database** after changing any of these settings. + +### Check connectivity + +All PostgresML databases are allowed to connect to any other database through the Internet by default. You can test connectivity to your database from PostgresML by using the `dblink` extension: + +```postgresql +SELECT + dblink( + 'postgres://user:password@your-production-db.amazonaws.com:5432/production_db', + 'SELECT 1 AS one' +) AS t1(one integer); + +``` + +### Start replicating + +Logical replication works like a pub/sub system: your primary database decides which tables it would like to publish, and PostgresML subscribes to those changes and downloads them as they are made. + +#### Create a publication + +A publication is a set of tables that your primary database would like to share with your PostgresML database. To create a publication, connect to your primary database as a superuser and create the publication for your tables of choice: + +```postgresql +CREATE PUBLICATION postgresml +FOR TABLE your_list_of_tables; +``` + +where `your_list_of_tables` are the tables you'd like to replicate. For example, if you have two tables, _users_ and _blog_posts_, you can create a publication for those two tables using this command: + + +{% tabs %} +{% tab title="SQL" %} + +```postgresql +CREATE PUBLICATION postgresml_users +FOR TABLE users, blog_posts; +``` + +{% endtab %} + +{% tab title="Output" %} + +``` +CREATE PUBLICATION +``` + +{% endtab %} +{% endtabs %} + +#### Subscribe to changes + +Now that we have a list of tables we want to replicate, we need to make sure those tables exist in your PostgresML database. Logical replication only sends over the data, without knowing anything else about your databases. Therefore, we need to make sure both the tables in your primary database and in your PostgresML databases match. + +You can get the schema for your tables either by using a PostgreSQL client like pgAdmin or, more easily, by using _pg_dump_ and then importing it into PostgresML using _psql_: + +{% tabs %} +{% tab title="Export the schema" %} + +```bash +pg_dump \ + postgres://user:password@yyour-production-db.amazonaws.com:5432/prodution_db \ + --schema-only \ + --no-owner \ + --no-privileges \ + -t users \ + -t blog_posts \ +> schema.sql +``` + +{% endtab %} +{% tab title="Import the schema" %} + +```bash +psql \ + postgres://user:password@db.cloud.postgresml.org:6432/your_postgresml_database \ + -f schema.sql +``` + +{% endtab %} +{% endtabs %} + +Once you have the tables created, we can start replicating data: + +{% tabs %} +{% tab title="SQL" %} + +```postgresql +CREATE SUBSCRIPTION postgresml +CONNECTION 'postgres://user:password@your-production-db.amazonaws.com:5432/prodution_db' +PUBLICATION postgresml; +``` + +{% endtab %} +{% tab title="Output" %} + +``` +CREATE SUBSCRIPTION +``` + +{% endtab %} +{% endtabs %} + +As soon you run this command, the PostgresML database will create a connection to your production database and copy the data from your tables into your PostgresML tables. Once that's done, the replication will start in real time and individual changes will be sent one row at a time. diff --git a/pgml-cms/docs/introduction/getting-started/import-your-data/logical-replication/inside-a-vpc.md b/pgml-cms/docs/introduction/getting-started/import-your-data/logical-replication/inside-a-vpc.md new file mode 100644 index 000000000..9f77b300a --- /dev/null +++ b/pgml-cms/docs/introduction/getting-started/import-your-data/logical-replication/inside-a-vpc.md @@ -0,0 +1,13 @@ +# Connect your VPC to PostgresML + +If your database doesn't have Internet access, PostgresML will need a service to proxy connections to your database. Any TCP proxy will do, +and we also provide an nginx-based Docker image than can be used without any additional configuration. + +
+ + +## PostgresML IPs by region + +| Region | List of IP addresses | +|-------------------------|----------------| +| AWS US West 2 | 100.20.31.186, 44.228.201.73, 44.238.193.82 | diff --git a/pgml-cms/docs/introduction/getting-started/import-your-data/pg-dump.md b/pgml-cms/docs/introduction/getting-started/import-your-data/pg-dump.md new file mode 100644 index 000000000..61cf688f6 --- /dev/null +++ b/pgml-cms/docs/introduction/getting-started/import-your-data/pg-dump.md @@ -0,0 +1,45 @@ +# Migrate with pg_dump + +_pg_dump_ is a command-line PostgreSQL tool that can move data between PostgreSQL databases. If you're planning a migration from your database to PostgresML, _pg_dump_ is a good tool to get you going quickly. + +## Getting started + +If your database is reasonably small (10 GB or less), you can just run _pg_dump_ in one command: + +{% tabs %} +{% tab title="pg_dump" %} + +```bash +pg_dump \ + --no-owner \ + --clean \ + --no-privileges \ + postgres://user:password@your-production-database.amazonaws.com/production_db | \ +psql postgres://user:password@sql.cloud.postgresml.org:6432/your_pgml_db +``` + +{% endtab %} +{% endtabs %} + +This will take a few minutes, and once the command completes, all your data, including indexes, will be in your PostgresML database. + +## Migrating one table at a time + +If your database is larger, you can split the migration into multiple steps, migrating one or more tables at a time. + +{% tabs %} +{% tab title="pg_dump" %} + +```bash +pg_dump \ + --no-owner \ + --clean \ + --no-privileges \ + -t users \ + -t orders \ + postgres://user:password@your-production-database.amazonaws.com/production_db | \ +psql postgres://user:password@sql.cloud.postgresml.org:6432/your_pgml_db +``` + +{% endtab %} +{% endtabs %} diff --git a/pgml-cms/docs/product/cloud-database/README.md b/pgml-cms/docs/product/cloud-database/README.md index 5956ef5dc..515aaed4d 100644 --- a/pgml-cms/docs/product/cloud-database/README.md +++ b/pgml-cms/docs/product/cloud-database/README.md @@ -1,19 +1,19 @@ -# Cloud Database +# Cloud database -PostgresML Cloud databases can be deployed using three (3) configurations: serverless, dedicated and enterprise. Each has its advantages and are tailored for companies of all sizes. +PostgresML cloud databases can be deployed using three (3) configurations: serverless, dedicated and enterprise. Each has its advantages and are tailored for companies of all sizes.

Plans available on PostgresML Cloud

-### Serverless +### [Serverless](serverless) The Serverless plan allows to quickly and easily create PostgresML databases that can scale from very little capacity to gigabytes of GPU cache and terabytes of disk storage. Their main use case is for teams that want to start small and grow as their usage of PostgresML increases. It has no fixed costs, starts at $0 with a generous free tier, and scales instantly to add more capacity. -### Dedicated +### [Dedicated](dedicated) The Dedicated plan is for larger startups and enterprises that have established PostgresML as their AI database of choice. It provides a large assortment of hardware, including CPU and GPU configurations, basically bottomless storage capacity and horizontal scaling into millions of queries per second. The Dedicated plan gives users access to Postgres settings, PgCat settings, replication configuration, tuning, horizontal scalability configuration, metrics, logs, and many more tools and knobs expected from enterprise-grade hosted PostgreSQL deployments. -### Enterprise +### [Enterprise](plans) The Enterprise plan is for large companies that have special compliance needs and deployment configurations. The plan includes support for cloud-prem and on-prem deployments, ACLs, Single Sign On and a dedicated solutions architect who will ensure that the enterprise users have a successful onboarding and integration experience with PostgresML. diff --git a/pgml-cms/docs/product/cloud-database/serverless-databases.md b/pgml-cms/docs/product/cloud-database/serverless.md similarity index 100% rename from pgml-cms/docs/product/cloud-database/serverless-databases.md rename to pgml-cms/docs/product/cloud-database/serverless.md diff --git a/pgml-cms/docs/use-cases/test-hello.md b/pgml-cms/docs/use-cases/test-hello.md new file mode 100644 index 000000000..18f2549a5 --- /dev/null +++ b/pgml-cms/docs/use-cases/test-hello.md @@ -0,0 +1 @@ +# Test hekki diff --git a/pgml-dashboard/Cargo.lock b/pgml-dashboard/Cargo.lock index 774c60038..0fe0b481d 100644 --- a/pgml-dashboard/Cargo.lock +++ b/pgml-dashboard/Cargo.lock @@ -1247,9 +1247,9 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] @@ -1707,9 +1707,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -2538,13 +2538,13 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pgml" -version = "1.0.1" +version = "1.0.2" dependencies = [ "anyhow", "async-trait", @@ -2572,6 +2572,7 @@ dependencies = [ "tokio", "tracing", "tracing-subscriber", + "url", "uuid", "walkdir", ] @@ -4956,9 +4957,9 @@ dependencies = [ [[package]] name = "url" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", "idna", diff --git a/pgml-dashboard/package-lock.json b/pgml-dashboard/package-lock.json index c7f315dec..86436dfd3 100644 --- a/pgml-dashboard/package-lock.json +++ b/pgml-dashboard/package-lock.json @@ -9,13 +9,13 @@ "@codemirror/lang-json": "^6.0.1", "@codemirror/lang-python": "^6.1.3", "@codemirror/lang-rust": "^6.0.1", - "@codemirror/lang-sql": "^6.5.4", "@codemirror/state": "^6.2.1", "@codemirror/view": "^6.21.0", "autosize": "^6.0.1", "codemirror": "^6.0.1", "dompurify": "^3.0.6", - "marked": "^9.1.0" + "marked": "^9.1.0", + "postgresml-lang-sql": "^6.6.3-3" } }, "node_modules/@codemirror/autocomplete": { @@ -88,19 +88,6 @@ "@lezer/rust": "^1.0.0" } }, - "node_modules/@codemirror/lang-sql": { - "version": "6.5.5", - "resolved": "https://registry.npmjs.org/@codemirror/lang-sql/-/lang-sql-6.5.5.tgz", - "integrity": "sha512-DvOaP2RXLb2xlxJxxydTFfwyYw5YDqEFea6aAfgh9UH0kUD6J1KFZ0xPgPpw1eo/5s2w3L6uh5PVR7GM23GxkQ==", - "dependencies": { - "@codemirror/autocomplete": "^6.0.0", - "@codemirror/language": "^6.0.0", - "@codemirror/state": "^6.0.0", - "@lezer/common": "^1.2.0", - "@lezer/highlight": "^1.0.0", - "@lezer/lr": "^1.0.0" - } - }, "node_modules/@codemirror/language": { "version": "6.10.0", "resolved": "https://registry.npmjs.org/@codemirror/language/-/language-6.10.0.tgz", @@ -249,6 +236,19 @@ "node": ">= 16" } }, + "node_modules/postgresml-lang-sql": { + "version": "6.6.3-3", + "resolved": "https://registry.npmjs.org/postgresml-lang-sql/-/postgresml-lang-sql-6.6.3-3.tgz", + "integrity": "sha512-nwKvOqJnEEGsrPE2M4p36TODbKojAjfxlioLWRlLWG2WeZcYVRe+ZDSiJ6n66LuDwbJh/r6vPWNSH/7u2SiovQ==", + "dependencies": { + "@codemirror/autocomplete": "^6.0.0", + "@codemirror/language": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@lezer/common": "^1.2.0", + "@lezer/highlight": "^1.0.0", + "@lezer/lr": "^1.0.0" + } + }, "node_modules/style-mod": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/style-mod/-/style-mod-4.1.0.tgz", diff --git a/pgml-dashboard/package.json b/pgml-dashboard/package.json index 3dfc7d703..c5a41ead6 100644 --- a/pgml-dashboard/package.json +++ b/pgml-dashboard/package.json @@ -3,7 +3,7 @@ "@codemirror/lang-javascript": "^6.2.1", "@codemirror/lang-python": "^6.1.3", "@codemirror/lang-rust": "^6.0.1", - "@codemirror/lang-sql": "^6.5.4", + "postgresml-lang-sql": "^6.6.3-3", "@codemirror/lang-json": "^6.0.1", "@codemirror/state": "^6.2.1", "@codemirror/view": "^6.21.0", diff --git a/pgml-dashboard/src/api/cms.rs b/pgml-dashboard/src/api/cms.rs index e5580e025..e376d7e9a 100644 --- a/pgml-dashboard/src/api/cms.rs +++ b/pgml-dashboard/src/api/cms.rs @@ -351,7 +351,7 @@ pub struct Collection { impl Collection { pub fn new(name: &str, hide_root: bool, redirects: HashMap<&'static str, &'static str>) -> Collection { - info!("Loading collection: {name}"); + debug!("Loading collection: {name}"); let name = name.to_owned(); let slug = name.to_lowercase(); let root_dir = config::cms_dir().join(&slug); @@ -423,7 +423,7 @@ impl Collection { // Docs gets a home link added to the index match self.name.as_str() { "Docs" => { - index.push(IndexLink::new("Docs Home", indent_level).href("/docs")); + index.push(IndexLink::new("Documentation", indent_level).href("/docs")); } _ => {} } diff --git a/pgml-dashboard/src/components/code_block/code_block_controller.js b/pgml-dashboard/src/components/code_block/code_block_controller.js index 127d4a6b5..8817ea08c 100644 --- a/pgml-dashboard/src/components/code_block/code_block_controller.js +++ b/pgml-dashboard/src/components/code_block/code_block_controller.js @@ -1,6 +1,6 @@ import { Controller } from "@hotwired/stimulus"; import { basicSetup } from "codemirror"; -import { sql } from "@codemirror/lang-sql"; +import { sql } from "postgresml-lang-sql"; import { python } from "@codemirror/lang-python"; import { javascript } from "@codemirror/lang-javascript"; import { rust } from "@codemirror/lang-rust"; diff --git a/pgml-dashboard/src/utils/markdown.rs b/pgml-dashboard/src/utils/markdown.rs index 424dc81e0..0b42a9121 100644 --- a/pgml-dashboard/src/utils/markdown.rs +++ b/pgml-dashboard/src/utils/markdown.rs @@ -513,15 +513,34 @@ pub fn get_toc<'a>(root: &'a AstNode<'a>) -> anyhow::Result> { return Ok(false); } }; - if let NodeValue::Text(text) = &sibling.data.borrow().value { - let index = match header_count.get(text) { + + let text = if let NodeValue::Text(text) = &sibling.data.borrow().value { + Some(text.clone()) + } else if let NodeValue::Link(_link) = &sibling.data.borrow().value { + let text = sibling.children() + .into_iter() + .map(|child| { + if let NodeValue::Text(text) = &child.data.borrow().value { + text.clone() + } else { + "".to_string() + } + }) + .join(""); + Some(text) + } else { + None + }; + + if let Some(text) = text { + let index = match header_count.get(&text) { Some(index) => index + 1, _ => 0, }; header_count.insert(text.clone(), index); - links.push(TocLink::new(text, index).level(header.level)); + links.push(TocLink::new(&text, index).level(header.level)); return Ok(false); } } diff --git a/pgml-dashboard/static/css/scss/pages/_docs.scss b/pgml-dashboard/static/css/scss/pages/_docs.scss index e5c36d7cc..7741ce643 100644 --- a/pgml-dashboard/static/css/scss/pages/_docs.scss +++ b/pgml-dashboard/static/css/scss/pages/_docs.scss @@ -1,4 +1,6 @@ .docs { + color: #{$white}; + div.results { overflow-x: auto; margin: 24px 24px; @@ -87,16 +89,17 @@ } table { - top: -1rem; - margin-bottom: 0; - white-space: nowrap; - color: #{$gray-100}; - th { - font-weight: 800; - } - tr:hover > * { - color: #{$gray-100}; - background-color: rgba(50, 54, 58, 0.3); + @extend .table-sm; + color: #{$white}; + + tbody { + tr { + font-weight: #{$font-weight-normal} !important; + &:hover { + color: #{$white}; + background-color: transparent; + } + } } } diff --git a/pgml-dashboard/static/js/utilities/code_mirror_theme.js b/pgml-dashboard/static/js/utilities/code_mirror_theme.js index c74801489..d546d3578 100644 --- a/pgml-dashboard/static/js/utilities/code_mirror_theme.js +++ b/pgml-dashboard/static/js/utilities/code_mirror_theme.js @@ -29,6 +29,7 @@ const editorTheme = { ".cm-content": { caretColor: cursor, + paddingBottom: '1rem', }, ".cm-cursor, .cm-dropCursor": { borderLeftColor: cursor },