Merge branch 'main' into PG-1244-Doc-KMS-update-limitations

.github/workflows/docker.yaml

@@ -17,7 +17,7 @@ jobs:
 
     steps:
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1
+        uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v3.8.0
 
       - name: Build
         uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 # v6.10.0

.github/workflows/docs.yaml

@@ -1,13 +1,14 @@
 name: Docs
 on:
+  workflow_dispatch: {}
   push:
     branches:
       - main
     paths:
       - "documentation/**"
 
 permissions:
-  contents: read
+  contents: write
 
 jobs:
   release:
@@ -20,21 +21,18 @@ jobs:
     steps:
       - name: Chekout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0 # fetch all commits/branches
 
       - name: Set up Python
         uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
         with:
           python-version: "3.x"
 
       - name: Configure git
-        env:
-          ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }}
         run: |
-          git config --global url."https://percona-platform-robot:${ROBOT_TOKEN}@github.com".insteadOf "https://github.com"
           git config user.name "GitHub Action"
           git config user.email "github-action@users.noreply.github.com"
-          git config user.password "${ROBOT_TOKEN}"
-          echo "GIT_USER=percona-platform-robot:${ROBOT_TOKEN}" >> $GITHUB_ENV
 
       - name: Install MkDocs
         run: |
@@ -44,6 +42,6 @@ jobs:
 
       - name: Deploy
         run: |
-          mike deploy main -p
           mike set-default main -p
           mike retitle main "Beta" -p
+          mike deploy main -p

.github/workflows/postgresql-16-ppg-package-pgxs.yml

@@ -107,7 +107,7 @@ jobs:
         working-directory: src/pg_tde
 
       - name: Report on test fail
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         if: ${{ failure() }}
         with:
           name: Regressions diff and postgresql log
@@ -130,7 +130,7 @@ jobs:
           sudo cp /usr/lib/postgresql/16/lib/pg_tde* pgtde-ppg16/usr/lib/postgresql/16/lib/
 
       - name: Upload tgz
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         with:
           name: pg_tde_ppg16_binary
           path: pgtde-ppg16
@@ -152,7 +152,7 @@ jobs:
           sudo dpkg -i --debug=7777 pgtde-ppg16.deb
 
       - name: Upload deb
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         with:
           name: pg_tde_deb
           path: pgtde-ppg16.deb

.github/workflows/postgresql-16-src-make-ssl11.yml

@@ -97,7 +97,7 @@ jobs:
         working-directory: src/contrib/pg_tde
 
       - name: Report on test fail
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         if: ${{ failure() }}
         with:
           name: Regressions diff and postgresql log

.github/workflows/postgresql-16-src-make.yml

@@ -97,7 +97,7 @@ jobs:
         working-directory: src/contrib/pg_tde
 
       - name: Report on test fail
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         if: ${{ failure() }}
         with:
           name: Regressions diff and postgresql log

.github/workflows/postgresql-16-src-meson.yml

@@ -85,7 +85,7 @@ jobs:
         working-directory: src/build
 
       - name: Report on test fail
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         if: ${{ failure() }}
         with:
           name: Regressions diff and postgresql log

.github/workflows/postgresql-17-src-make.yml

@@ -97,7 +97,7 @@ jobs:
         working-directory: src/contrib/pg_tde
 
       - name: Report on test fail
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         if: ${{ failure() }}
         with:
           name: Regressions diff and postgresql log

.github/workflows/postgresql-17-src-meson-perf.yml

@@ -1,9 +1,7 @@
 name: Perf test
 on: [pull_request]
 permissions:
-  contents: write
-  pull-requests: write
-  repository-projects: write
+  contents: read
 
 jobs:
   build:
@@ -81,7 +79,7 @@ jobs:
         working-directory: src/build
 
       - name: Report on test fail
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         if: ${{ failure() }}
         with:
           name: Regressions diff and postgresql log
@@ -141,7 +139,7 @@ jobs:
           echo "EOF" >> $GITHUB_ENV
         working-directory: inst
 
-      - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+      - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         with:
           name: pr_perf_results
           path: inst/pr_perf_results

.github/workflows/postgresql-17-src-meson.yml

@@ -80,7 +80,7 @@ jobs:
         working-directory: src/build
 
       - name: Report on test fail
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         if: ${{ failure() }}
         with:
           name: Regressions diff and postgresql log

.github/workflows/postgresql-perf-results.yml

@@ -6,6 +6,10 @@ on:
     types:
       - completed
 
+permissions:
+  contents: read
+  pull-requests: write
+
 jobs:
   download:
     runs-on: ubuntu-latest
@@ -35,17 +39,10 @@ jobs:
         run: |
           unzip pr_perf_results.zip
 
-      - name: Clone pg_tde repository
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          path: 'src'
-          ref: ${{ github.event.workflow_run.head_branch }}
-
       - name: 'Create comment'
         run: |
-          gh pr comment ${PR_NUMBER} -F ../pr_perf_results --edit-last || \
-            gh pr comment ${PR_NUMBER} -F ../pr_perf_results
+          gh pr comment ${PR_NUMBER} -F pr_perf_results --repo ${{ github.repository }}  --edit-last || \
+            gh pr comment ${PR_NUMBER} -F pr_perf_results --repo ${{ github.repository }}
         env:
           PR_NUMBER: ${{ github.event.number }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        working-directory: src

.github/workflows/postgresql-pgdg-package-pgxs.yml

@@ -104,7 +104,7 @@ jobs:
         working-directory: src/pg_tde
 
       - name: Report on test fail
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         if: ${{ failure() }}
         with:
           name: Regressions diff and postgresql log
@@ -131,7 +131,7 @@ jobs:
       - name: Upload tgz
         env:
           POSTGRESQL_VERSION: ${{ matrix.postgresql-version }}
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         with:
           name: pg_tde_pgdg$POSTGRESQL_VERSION_binary
           path: pgtde-pgdg$POSTGRESQL_VERSION
@@ -159,7 +159,7 @@ jobs:
       - name: Upload deb
         env:
           POSTGRESQL_VERSION: ${{ matrix.postgresql-version }}
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         with:
           name: pg_tde_deb
           path: pgtde-pgdg$POSTGRESQL_VERSION.deb

.github/workflows/scorecard.yml

@@ -35,14 +35,14 @@ jobs:
           publish_results: true
 
       - name: Upload results
-        uses: actions/upload-artifact@97a0fba1372883ab732affbe8f94b823f91727db # v3.pre.node20
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         with:
           name: SARIF file
           path: results.sarif
           retention-days: 5
 
       # Upload the results to GitHub's code scanning dashboard (optional).
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6
+        uses: github/codeql-action/upload-sarif@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # v3.28.0
         with:
           sarif_file: results.sarif

.gitignore

@@ -1,5 +1,6 @@
 *.so
 *.o
+*.frontend
 __pycache__
 
 /config.cache

Makefile

@@ -3,7 +3,7 @@
 PGFILEDESC = "pg_tde access method"
 MODULE_big = pg_tde
 EXTENSION = pg_tde
-DATA = pg_tde--1.0.sql
+DATA = pg_tde--1.0-beta2.sql
 
 REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/pg_tde/pg_tde.conf
 REGRESS = toast_decrypt_basic \
@@ -67,12 +67,12 @@ src/libkmip/libkmip/src/kmip_memset.o
 ifdef USE_PGXS
 PG_CONFIG = pg_config
 PGXS := $(shell $(PG_CONFIG) --pgxs)
-override PG_CPPFLAGS += -I$(CURDIR)/src/include -I$(CURDIR)/src/libkmip/libkmip/include -I$(CURDIR)/src$(MAJORVERSION)/include -fPIC
+override PG_CPPFLAGS += -I$(CURDIR)/src/include -I$(CURDIR)/src/libkmip/libkmip/include -I$(CURDIR)/src$(MAJORVERSION)/include
 include $(PGXS)
 else
 subdir = contrib/pg_tde
 top_builddir = ../..
-override PG_CPPFLAGS += -I$(top_srcdir)/$(subdir)/src/include  -I$(top_srcdir)/$(subdir)/src/libkmip/libkmip/include -I$(top_srcdir)/$(subdir)/src$(MAJORVERSION)/include -fPIC
+override PG_CPPFLAGS += -I$(top_srcdir)/$(subdir)/src/include  -I$(top_srcdir)/$(subdir)/src/libkmip/libkmip/include -I$(top_srcdir)/$(subdir)/src$(MAJORVERSION)/include
 include $(top_builddir)/src/Makefile.global
 include $(top_srcdir)/contrib/contrib-global.mk
 endif

Makefile.tools

@@ -1,20 +1,23 @@
 TDE_OBJS = \
- 	src/access/pg_tde_tdemap.o \
- 	src/access/pg_tde_xlog_encrypt.o \
- 	src/catalog/tde_global_space.o \
- 	src/catalog/tde_keyring.o \
- 	src/catalog/tde_keyring_parse_opts.o \
- 	src/catalog/tde_principal_key.o \
- 	src/common/pg_tde_utils.o \
- 	src/encryption/enc_aes.o \
- 	src/encryption/enc_tde.o \
- 	src/keyring/keyring_api.o \
- 	src/keyring/keyring_curl.o \
- 	src/keyring/keyring_file.o \
- 	src/keyring/keyring_vault.o \
-	src/keyring/keyring_kmip.o \
-	src/keyring/keyring_kmip_ereport.o \
-	src/libkmip/libkmip/src/kmip.o \
-	src/libkmip/libkmip/src/kmip_bio.o \
-	src/libkmip/libkmip/src/kmip_locate.o \
-	src/libkmip/libkmip/src/kmip_memset.o
+ 	src/access/pg_tde_tdemap.frontend \
+ 	src/access/pg_tde_xlog_encrypt.frontend \
+ 	src/catalog/tde_global_space.frontend \
+ 	src/catalog/tde_keyring.frontend \
+ 	src/catalog/tde_keyring_parse_opts.frontend \
+ 	src/catalog/tde_principal_key.frontend \
+ 	src/common/pg_tde_utils.frontend \
+ 	src/encryption/enc_aes.frontend \
+ 	src/encryption/enc_tde.frontend \
+ 	src/keyring/keyring_api.frontend \
+ 	src/keyring/keyring_curl.frontend \
+ 	src/keyring/keyring_file.frontend \
+ 	src/keyring/keyring_vault.frontend \
+	src/keyring/keyring_kmip.frontend \
+	src/keyring/keyring_kmip_ereport.frontend \
+	src/libkmip/libkmip/src/kmip.frontend \
+	src/libkmip/libkmip/src/kmip_bio.frontend \
+	src/libkmip/libkmip/src/kmip_locate.frontend \
+	src/libkmip/libkmip/src/kmip_memset.frontend
+
+%.frontend: %.c
+	$(CC) $(CPPFLAGS) -c $< -o $@

documentation/docs/faq.md

@@ -0,0 +1,30 @@
+# FAQ
+
+## Why do I need TDE?
+
+- Compliance to security and legal regulations like GDPR, PCI DSS and others
+- Encryption of backups
+- Granular encryption of specific data sets and reducing the performance overhead that encryption brings
+- Additional layer of security to existing security measures
+
+## I use disk-level encryption. Why should I care about TDE?
+
+Encrypting a hard drive encrypts all data including system and application files that are there. However, disk encryption doesn’t protect your data after the boot-up of your system. During runtime, the files are decrypted with disk-encryption.
+
+TDE focuses specifically on data files and offers a more granular control over encrypted data. It also ensures that files are encrypted on disk during runtime and when moved to another system or storage.
+
+Consider using TDE and storage-level encryption together to add another layer of data security
+
+## Is TDE enough to ensure data security?
+
+No. TDE is an additional layer to ensure data security. It protects data at rest. Consider introducing also these measures:
+
+* Access control and authentication
+* Strong network security like TLS
+* Disk encryption
+* Regular monitoring and auditing
+* Additional data protection for sensitive fields (e.g., application-layer encryption)
+
+## What happens to my data if I lose a principal key?
+
+If you lose encryption keys, especially, the principal key, the data is lost. That's why it's critical to back up your encryption keys securely.

documentation/docs/functions.md

@@ -21,7 +21,7 @@ Creates a new key provider for the database using a remote HashiCorp Vault serve
 The specified access parameters require permission to read and write keys at the location.
 
 ```
-SELECT pg_tde_add_key_provider_vault_v2('provider-name',:'secret_token','url','mount','ca_path');
+SELECT pg_tde_add_key_provider_vault_v2('provider-name','secret_token','url','mount','ca_path');
 ```
 
 where:
@@ -33,6 +33,24 @@ where:
 
 All parameters can be either strings, or JSON objects [referencing remote parameters](external-parameters.md).
 
+## pg_tde_add_key_provider_kmip
+
+Creates a new key provider for the database using a remote KMIP server.
+
+The specified access parameters require permission to read and write keys at the server.
+
+```
+SELECT pg_tde_add_key_provider_kmip('provider-name','kmip-IP', 5696, '/path_to/server_certificate.pem', '/path_to/client_key.pem');
+```
+
+where:
+
+* `provider-name` is the name of the provider. You can specify any name, it's for you to identify the provider.
+* `kmip-IP` is the IP address of a domain name of the KMIP server
+* The port to communicate with the KMIP server. The default port is `5696`.
+* `server-certificate` is the path to the certificate file for the KMIP server.
+* `client key` is the path to the client key.
+
 ## pg_tde_set_principal_key
 
 Sets the principal key for the database using the specified key provider.
@@ -72,12 +90,19 @@ SELECT pg_tde_rotate_principal_key('name-of-the-new-principal-key', NULL);
 SELECT pg_tde_rotate_principal_key(NULL, 'name-of-the-new-provider');
 ```
 
+
 ## pg_tde_is_encrypted
 
-Tells if a table is using the `pg_tde` access method or not.
+Tells if a table is encrypted using the `tde_heap` access method or not.
+
+To verify a table encryption, run the following statement:
 
 ```
 SELECT pg_tde_is_encrypted('table_name');
 ```
 
+You can also verify if the table in a custom schema is encrypted. Pass teh schema name for the function as follows:
 
+```
+SELECT pg_tde_is_encrypted('schema.table_name');
+```