From 2f52b2193f8d0a5f2338f48c093010ddcee0bf63 Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoy Date: Thu, 9 May 2024 18:47:18 +0300 Subject: [PATCH 1/8] Prepare for XLog keyring * Make the *.map *.dat processing code aware of custom databases and table spaces * Add XLog GUC and init the keyring based on that. Only FS for now * Make the internal/external key infrastructure work with custom (not stored in the database) keyrings. --- src/access/pg_tde_tdemap.c | 224 +++++++++++++++++---------- src/access/pg_tde_xlog.c | 149 ++++++++++++++++-- src/catalog/tde_keyring.c | 3 +- src/catalog/tde_master_key.c | 49 +++--- src/include/access/pg_tde_tdemap.h | 22 ++- src/include/access/pg_tde_xlog.h | 1 + src/include/catalog/tde_keyring.h | 7 + src/include/catalog/tde_master_key.h | 6 +- src/transam/pg_tde_xact_handler.c | 2 - 9 files changed, 334 insertions(+), 129 deletions(-) diff --git a/src/access/pg_tde_tdemap.c b/src/access/pg_tde_tdemap.c index ced8f6a2..8c4e345f 100644 --- a/src/access/pg_tde_tdemap.c +++ b/src/access/pg_tde_tdemap.c @@ -73,10 +73,11 @@ typedef struct TDEMapEntry int32 key_index; } TDEMapEntry; -/* Global variables */ -static char db_path[MAXPGPATH] = {0}; -static char db_map_path[MAXPGPATH] = {0}; -static char db_keydata_path[MAXPGPATH] = {0}; +typedef struct TDEMapFilePath +{ + char map_path[MAXPGPATH]; + char keydata_path[MAXPGPATH]; +} TDEMapFilePath; static void put_key_into_map(Oid rel_id, RelKeyData *key); @@ -84,11 +85,6 @@ static File pg_tde_open_file_basic(char *tde_filename, int fileFlags, bool ignor static File pg_tde_file_header_write(char *tde_filename, File tde_file, TDEMasterKeyInfo *master_key_info, off_t *bytes_written); static File pg_tde_file_header_read(char *tde_filename, File tde_file, TDEFileHeader *fheader, bool *is_new_file, off_t *bytes_read); -static RelKeyData* tde_create_rel_key(const RelFileLocator *rlocator, InternalKey *key, TDEMasterKeyInfo *master_key_info); -static RelKeyData *tde_encrypt_rel_key(TDEMasterKey *master_key, RelKeyData *rel_key_data, const RelFileLocator *rlocator); -static RelKeyData *tde_decrypt_rel_key(TDEMasterKey *master_key, RelKeyData *enc_rel_key_data, const RelFileLocator *rlocator); - -static void pg_tde_set_db_file_paths(Oid dbOid); static File pg_tde_open_file(char *tde_filename, TDEMasterKeyInfo *master_key_info, bool should_fill_info, int fileFlags, bool *is_new_file, off_t *offset); static int32 pg_tde_write_map_entry(const RelFileLocator *rlocator, char *db_map_path, TDEMasterKeyInfo *master_key_info); @@ -98,7 +94,7 @@ static bool pg_tde_read_one_map_entry(File map_file, const RelFileLocator *rloca static void pg_tde_write_keydata(char *db_keydata_path, TDEMasterKeyInfo *master_key_info, int32 key_index, RelKeyData *enc_rel_key_data); static void pg_tde_write_one_keydata(File keydata_file, int32 key_index, RelKeyData *enc_rel_key_data); -static RelKeyData* pg_tde_get_key_from_file(const RelFileLocator *rlocator); +static RelKeyData* pg_tde_get_key_from_file(const RelFileLocator *rlocator, GenericKeyring *keyring); static RelKeyData* pg_tde_read_keydata(char *db_keydata_path, int32 key_index, TDEMasterKey *master_key); static RelKeyData* pg_tde_read_one_keydata(File keydata_file, int32 key_index, TDEMasterKey *master_key); @@ -140,7 +136,7 @@ pg_tde_create_key_map_entry(const RelFileLocator *newrlocator) } /* Encrypt the key */ - rel_key_data = tde_create_rel_key(newrlocator, &int_key, &master_key->keyInfo); + rel_key_data = tde_create_rel_key(newrlocator->relNumber, &int_key, &master_key->keyInfo); enc_rel_key_data = tde_encrypt_rel_key(master_key, rel_key_data, newrlocator); /* @@ -184,7 +180,29 @@ GetRelationKey(RelFileLocator rel) } } - key = pg_tde_get_key_from_file(&rel); + key = pg_tde_get_key_from_file(&rel, NULL); + + put_key_into_map(rel.relNumber, key); + + return key; +} + +RelKeyData * +GetInternalKey(RelFileLocator rel, GenericKeyring *keyring) +{ + RelKey *curr; + RelKeyData *key; + + Oid rel_id = rel.relNumber; + for (curr = tde_rel_key_map; curr != NULL; curr = curr->next) + { + if (curr->rel_id == rel_id) + { + return curr->key; + } + } + + key = pg_tde_get_key_from_file(&rel, keyring); if (key != NULL) { @@ -226,8 +244,8 @@ tde_sprint_key(InternalKey *k) * Creates a key for a relation identified by rlocator. Returns the newly * created key. */ -static RelKeyData * -tde_create_rel_key(const RelFileLocator *rlocator, InternalKey *key, TDEMasterKeyInfo *master_key_info) +RelKeyData * +tde_create_rel_key(Oid rel_id, InternalKey *key, TDEMasterKeyInfo *master_key_info) { RelKeyData *rel_key_data; @@ -238,7 +256,7 @@ tde_create_rel_key(const RelFileLocator *rlocator, InternalKey *key, TDEMasterKe rel_key_data->internal_key.ctx = NULL; /* Add to the decrypted key to cache */ - put_key_into_map(rlocator->relNumber, rel_key_data); + put_key_into_map(rel_id, rel_key_data); return rel_key_data; } @@ -246,7 +264,7 @@ tde_create_rel_key(const RelFileLocator *rlocator, InternalKey *key, TDEMasterKe /* * Encrypts a given key and returns the encrypted one. */ -static RelKeyData * +RelKeyData * tde_encrypt_rel_key(TDEMasterKey *master_key, RelKeyData *rel_key_data, const RelFileLocator *rlocator) { RelKeyData *enc_rel_key_data; @@ -260,7 +278,7 @@ tde_encrypt_rel_key(TDEMasterKey *master_key, RelKeyData *rel_key_data, const Re /* * Decrypts a given key and returns the decrypted one. */ -static RelKeyData * +RelKeyData * tde_decrypt_rel_key(TDEMasterKey *master_key, RelKeyData *enc_rel_key_data, const RelFileLocator *rlocator) { RelKeyData *rel_key_data = NULL; @@ -271,33 +289,17 @@ tde_decrypt_rel_key(TDEMasterKey *master_key, RelKeyData *enc_rel_key_data, cons return rel_key_data; } -/* - * Sets the global variables so that we don't have to do this again for this - * backend lifetime. - */ -static void -pg_tde_set_db_file_paths(Oid dbOid) +inline void +pg_tde_set_db_file_paths(const RelFileLocator *rlocator, char *map_path, char *keydata_path) { - /* Return if the values are already set */ - // TODO: remove this entire global state thing - //if (*db_path && *db_map_path && *db_keydata_path) - // return; - - /* Fill in the values */ - snprintf(db_path, MAXPGPATH, "%s", GetDatabasePath(dbOid, DEFAULTTABLESPACE_OID)); - - /* Set the file nanes for map and keydata */ - join_path_components(db_map_path, db_path, PG_TDE_MAP_FILENAME); - join_path_components(db_keydata_path, db_path, PG_TDE_KEYDATA_FILENAME); -} - -/* - * Path data clean up once the transaction is done. - */ -void -pg_tde_cleanup_path_vars(void) -{ - *db_path = *db_map_path = *db_keydata_path = 0; + if (map_path) + join_path_components(map_path, + GetDatabasePath(rlocator->dbOid, rlocator->spcOid), + PG_TDE_MAP_FILENAME); + if (keydata_path) + join_path_components(keydata_path, + GetDatabasePath(rlocator->dbOid, rlocator->spcOid), + PG_TDE_KEYDATA_FILENAME); } /* @@ -305,10 +307,17 @@ pg_tde_cleanup_path_vars(void) * Returns true if both map and key data files are created. */ void -pg_tde_delete_tde_files(Oid dbOid) +pg_tde_delete_tde_files(Oid dbOid, Oid spcOid) { + char db_map_path[MAXPGPATH] = {0}; + char db_keydata_path[MAXPGPATH] = {0}; + /* Set the file paths */ - pg_tde_set_db_file_paths(dbOid); + pg_tde_set_db_file_paths(&(RelFileLocator) { + spcOid, + dbOid, + 0}, + db_map_path, db_keydata_path); /* Remove these files without emitting any error */ PathNameDeleteTemporaryFile(db_map_path, false); @@ -332,9 +341,15 @@ pg_tde_save_master_key(TDEMasterKeyInfo *master_key_info) off_t curr_pos = 0; bool is_new_map = false; bool is_new_key_data = false; + char db_map_path[MAXPGPATH] = {0}; + char db_keydata_path[MAXPGPATH] = {0}; /* Set the file paths */ - pg_tde_set_db_file_paths(master_key_info->databaseId); + pg_tde_set_db_file_paths(&(RelFileLocator) { + master_key_info->tablespaceId, + master_key_info->databaseId, + 0}, + db_map_path, db_keydata_path); ereport(LOG, (errmsg("pg_tde_save_master_key"))); @@ -354,16 +369,22 @@ pg_tde_save_master_key(TDEMasterKeyInfo *master_key_info) * a LW_SHARED or higher lock on files before calling this function. */ TDEMasterKeyInfo * -pg_tde_get_master_key(Oid dbOid) +pg_tde_get_master_key(Oid dbOid, Oid spcOid) { File tde_file = -1; TDEFileHeader fheader; TDEMasterKeyInfo *master_key_info = NULL; bool is_new_file = false; off_t bytes_read = 0; + char db_map_path[MAXPGPATH] = {0}; + char db_keydata_path[MAXPGPATH] = {0}; /* Set the file paths */ - pg_tde_set_db_file_paths(dbOid); + pg_tde_set_db_file_paths(&(RelFileLocator) { + spcOid, + dbOid, + 0}, + db_map_path, db_keydata_path); /* * Ensuring that we always open the file in binary mode. The caller must @@ -409,7 +430,7 @@ pg_tde_open_file_basic(char *tde_filename, int fileFlags, bool ignore_missing) tde_file = PathNameOpenFile(tde_filename, fileFlags | PG_BINARY); if (tde_file < 0 && !(errno == ENOENT && ignore_missing == true)) { - ereport(ERROR, + ereport(PANIC, (errcode_for_file_access(), errmsg("Could not open tde file \"%s\": %m", tde_filename))); @@ -594,6 +615,8 @@ pg_tde_write_one_map_entry(File map_file, const RelFileLocator *rlocator, int fl /* Add the entry to the file */ if (bytes_written != MAP_ENTRY_SIZE) { + char db_map_path[MAXPGPATH] = {0}; + pg_tde_set_db_file_paths(rlocator, db_map_path, NULL); ereport(FATAL, (errcode_for_file_access(), errmsg("Could not write tde map file \"%s\": %m", @@ -772,8 +795,7 @@ pg_tde_write_one_keydata(File keydata_file, int32 key_index, RelKeyData *enc_rel { ereport(FATAL, (errcode_for_file_access(), - errmsg("Could not write tde key data file \"%s\": %m", - db_keydata_path))); + errmsg("Could not write tde key data file: %m"))); } } @@ -823,6 +845,12 @@ pg_tde_read_one_keydata(File keydata_file, int32 key_index, TDEMasterKey *master /* Check if the file has a valid key */ if ((read_pos + INTERNAL_KEY_LEN) > FileSize(keydata_file)) { + char db_keydata_path[MAXPGPATH] = {0}; + pg_tde_set_db_file_paths(&(RelFileLocator) { + master_key->keyInfo.tablespaceId, + master_key->keyInfo.databaseId, + 0}, + NULL, db_keydata_path); ereport(FATAL, (errcode(ERRCODE_NO_DATA_FOUND), errmsg("Could not find the required key at index %d in tde data file \"%s\": %m", @@ -833,6 +861,12 @@ pg_tde_read_one_keydata(File keydata_file, int32 key_index, TDEMasterKey *master /* Read the encrypted key */ if (FileRead(keydata_file, &(enc_rel_key_data->internal_key), INTERNAL_KEY_LEN, read_pos, WAIT_EVENT_DATA_FILE_READ) != INTERNAL_KEY_LEN) { + char db_keydata_path[MAXPGPATH] = {0}; + pg_tde_set_db_file_paths(&(RelFileLocator) { + master_key->keyInfo.tablespaceId, + master_key->keyInfo.databaseId, + 0}, + NULL, db_keydata_path); ereport(FATAL, (errcode_for_file_access(), errmsg("Could not read key at index %d in tde key data file \"%s\": %m", @@ -853,13 +887,15 @@ pg_tde_read_one_keydata(File keydata_file, int32 key_index, TDEMasterKey *master void pg_tde_write_key_map_entry(const RelFileLocator *rlocator, RelKeyData *enc_rel_key_data, TDEMasterKeyInfo *master_key_info) { - int32 key_index = 0; - LWLock *lock_files = tde_lwlock_mk_files(); + int32 key_index = 0; + LWLock *lock_files = tde_lwlock_mk_files(); + char db_map_path[MAXPGPATH] = {0}; + char db_keydata_path[MAXPGPATH] = {0}; Assert(rlocator); /* Set the file paths */ - pg_tde_set_db_file_paths(rlocator->dbOid); + pg_tde_set_db_file_paths(rlocator, db_map_path, db_keydata_path); /* Create the map entry and then add the encrypted key to the data file */ LWLockAcquire(lock_files, LW_EXCLUSIVE); @@ -877,14 +913,16 @@ pg_tde_write_key_map_entry(const RelFileLocator *rlocator, RelKeyData *enc_rel_k void pg_tde_delete_key_map_entry(const RelFileLocator *rlocator) { - int32 key_index = 0; - off_t offset = 0; - LWLock *lock_files = tde_lwlock_mk_files(); + int32 key_index = 0; + off_t offset = 0; + LWLock *lock_files = tde_lwlock_mk_files(); + char db_map_path[MAXPGPATH] = {0}; + char db_keydata_path[MAXPGPATH] = {0}; Assert(rlocator); /* Get the file paths */ - pg_tde_set_db_file_paths(rlocator->dbOid); + pg_tde_set_db_file_paths(rlocator, db_map_path, db_keydata_path); /* Remove the map entry if found */ LWLockAcquire(lock_files, LW_EXCLUSIVE); @@ -920,13 +958,15 @@ pg_tde_delete_key_map_entry(const RelFileLocator *rlocator) void pg_tde_free_key_map_entry(const RelFileLocator *rlocator, off_t offset) { - int32 key_index = 0; - LWLock *lock_files = tde_lwlock_mk_files(); + int32 key_index = 0; + LWLock *lock_files = tde_lwlock_mk_files(); + char db_map_path[MAXPGPATH] = {0}; + char db_keydata_path[MAXPGPATH] = {0}; Assert(rlocator); /* Get the file paths */ - pg_tde_set_db_file_paths(rlocator->dbOid); + pg_tde_set_db_file_paths(rlocator, db_map_path, db_keydata_path); /* Remove the map entry if found */ LWLockAcquire(lock_files, LW_EXCLUSIVE); @@ -950,22 +990,23 @@ pg_tde_free_key_map_entry(const RelFileLocator *rlocator, off_t offset) * reads the key data from the keydata file. */ static RelKeyData * -pg_tde_get_key_from_file(const RelFileLocator *rlocator) +pg_tde_get_key_from_file(const RelFileLocator *rlocator, GenericKeyring *keyring) { - int32 key_index = 0; - TDEMasterKey *master_key; - RelKeyData *rel_key_data; - RelKeyData *enc_rel_key_data; - off_t offset = 0; - LWLock *lock_files = tde_lwlock_mk_files(); + int32 key_index = 0; + TDEMasterKey *master_key; + RelKeyData *rel_key_data; + RelKeyData *enc_rel_key_data; + off_t offset = 0; + LWLock *lock_files = tde_lwlock_mk_files(); + char db_map_path[MAXPGPATH] = {0}; + char db_keydata_path[MAXPGPATH] = {0}; Assert(rlocator); LWLockAcquire(lock_files, LW_SHARED); /* Get/generate a master, create the key for relation and get the encrypted key with bytes to write */ - pg_tde_set_db_file_paths(rlocator->dbOid); - master_key = GetMasterKey(rlocator->dbOid); + master_key = GetMasterKey(rlocator->dbOid, rlocator->spcOid, keyring); if (master_key == NULL) { LWLockRelease(lock_files); @@ -973,6 +1014,9 @@ pg_tde_get_key_from_file(const RelFileLocator *rlocator) (errmsg("failed to retrieve master key"))); } + /* Get the file paths */ + pg_tde_set_db_file_paths(rlocator, db_map_path, db_keydata_path); + /* Read the map entry and get the index of the relation key */ key_index = pg_tde_process_map_entry(rlocator, db_map_path, &offset, false); @@ -1049,12 +1093,18 @@ pg_tde_perform_rotate_key(TDEMasterKey *master_key, TDEMasterKey *new_master_key off_t map_size; off_t keydata_size; XLogMasterKeyRotate *xlrec; - off_t xlrec_size; - LWLock *lock_files = tde_lwlock_mk_files(); - LWLock *lock_cache = tde_lwlock_mk_cache(); + off_t xlrec_size; + LWLock *lock_files = tde_lwlock_mk_files(); + LWLock *lock_cache = tde_lwlock_mk_cache(); + char db_map_path[MAXPGPATH] = {0}; + char db_keydata_path[MAXPGPATH] = {0}; /* Set the file paths */ - pg_tde_set_db_file_paths(master_key->keyInfo.databaseId); + pg_tde_set_db_file_paths(&(RelFileLocator) { + master_key->keyInfo.tablespaceId, + master_key->keyInfo.databaseId, + 0}, + db_map_path, db_keydata_path); /* Let's update the pathnames in the local variable for ease of use/readability */ strncpy(m_path[OLD_MASTER_KEY], db_map_path, MAXPGPATH); @@ -1158,21 +1208,27 @@ bool pg_tde_write_map_keydata_files(off_t map_size, char *m_file_data, off_t keydata_size, char *k_file_data) { TDEFileHeader *fheader; - char m_path_new[MAXPGPATH]; - char k_path_new[MAXPGPATH]; - File m_file_new; - File k_file_new; - bool is_new_file; - off_t curr_pos = 0; - off_t read_pos_tmp = 0; - LWLock *lock_files = tde_lwlock_mk_files(); - LWLock *lock_cache = tde_lwlock_mk_cache(); + char m_path_new[MAXPGPATH]; + char k_path_new[MAXPGPATH]; + File m_file_new; + File k_file_new; + bool is_new_file; + off_t curr_pos = 0; + off_t read_pos_tmp = 0; + LWLock *lock_files = tde_lwlock_mk_files(); + LWLock *lock_cache = tde_lwlock_mk_cache(); + char db_map_path[MAXPGPATH] = {0}; + char db_keydata_path[MAXPGPATH] = {0}; /* Let's get the header. Buff should start with the map file header. */ fheader = (TDEFileHeader *) m_file_data; /* Set the file paths */ - pg_tde_set_db_file_paths(fheader->master_key_info.databaseId); + pg_tde_set_db_file_paths(&(RelFileLocator) { + fheader->master_key_info.tablespaceId, + fheader->master_key_info.databaseId, + 0}, + db_map_path, db_keydata_path); LWLockAcquire(lock_files, LW_EXCLUSIVE); LWLockAcquire(lock_cache, LW_EXCLUSIVE); diff --git a/src/access/pg_tde_xlog.c b/src/access/pg_tde_xlog.c index b3fd4ec5..bdbbb553 100644 --- a/src/access/pg_tde_xlog.c +++ b/src/access/pg_tde_xlog.c @@ -16,6 +16,7 @@ #include "access/xlog.h" #include "access/xlog_internal.h" #include "access/xloginsert.h" +#include "catalog/pg_tablespace_d.h" #include "storage/bufmgr.h" #include "storage/shmem.h" #include "utils/guc.h" @@ -23,6 +24,7 @@ #include "access/pg_tde_tdemap.h" #include "access/pg_tde_xlog.h" +#include "catalog/tde_keyring.h" #include "catalog/tde_master_key.h" #include "encryption/enc_tde.h" @@ -30,12 +32,36 @@ static char *TDEXLogEncryptBuf = NULL; bool EncryptXLog = false; +/* GUC */ +static char *KRingProviderType = NULL; +static char *KRingProviderFilePath = NULL; + static XLogPageHeaderData EncryptCurrentPageHrd; static XLogPageHeaderData DecryptCurrentPageHrd; static ssize_t TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset); static void SetXLogPageIVPrefix(TimeLineID tli, XLogRecPtr lsn, char* iv_prefix); static int XLOGChooseNumBuffers(void); + +typedef enum +{ + TDE_GCAT_KEY_XLOG, + + /* must be last */ + TDE_GCAT_KEYS_COUNT +} GlobalCatalogKeyTypes; + +/* TODO: move TDEXLogEncryptBuf here*/ +typedef struct XLogEncryptionState +{ + GenericKeyring *keyring; + /* TODO: locking */ + TDEMasterKey master_keys[TDE_GCAT_KEYS_COUNT]; + +} XLogEncryptionState; + +static XLogEncryptionState *EncryptionState = NULL; + /* * TDE fork XLog */ @@ -123,16 +149,54 @@ pg_tde_rmgr_identify(uint8 info) #ifdef PERCONA_FORK /* + * ------------------------- * XLog Storage Manager - * TODO: - * - Should be a config option "on/off"? - * - Currently it encrypts WAL XLog Pages, should we encrypt whole Segments? `initdb` for - * example generates a write of 312 pages - so 312 "gen IV" and "encrypt" runs instead of one. - * Would require though an extra read() during recovery/was_send etc to check `XLogPageHeader` - * if segment is encrypted. - * We could also encrypt Records while adding them to the XLog Buf but it'll be the slowest (?). */ +static GenericKeyring *xlog_keyring; + +static void +pg_tde_init_xlog_kring(void) +{ + EncryptionState->keyring->type = get_keyring_provider_from_typename(KRingProviderType); + switch (EncryptionState->keyring->type) + { + case FILE_KEY_PROVIDER: + FileKeyring *kring = (FileKeyring *) EncryptionState->keyring; + strncpy(kring->file_name, KRingProviderFilePath, sizeof(kring->file_name)); + break; + } +} + +static void +pg_tde_create_xlog_key(void) +{ + InternalKey int_key; + RelKeyData *rel_key_data; + RelKeyData *enc_rel_key_data; + RelFileLocator *rlocator = &GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID); + TDEMasterKey *master_key; + + master_key = set_master_key_with_keyring("xlog-master-key", xlog_keyring, + rlocator->dbOid, rlocator->spcOid, false); + + memset(&int_key, 0, sizeof(InternalKey)); + + if (!RAND_bytes(int_key.key, INTERNAL_KEY_LEN)) + { + ereport(FATAL, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("could not generate internal key for \"WAL\": %s", + ERR_error_string(ERR_get_error(), NULL)))); + } + + rel_key_data = tde_create_rel_key(rlocator->relNumber, &int_key, &master_key->keyInfo); + enc_rel_key_data = tde_encrypt_rel_key(master_key, rel_key_data, rlocator); + + pg_tde_write_key_map_entry(rlocator, enc_rel_key_data, &master_key->keyInfo); + memcpy(EncryptionState->master_keys + TDE_GCAT_KEY_XLOG, master_key, sizeof(TDEMasterKey)); +} + void xlogInitGUC(void) { @@ -147,6 +211,28 @@ xlogInitGUC(void) NULL, /* assign_hook */ NULL /* show_hook */ ); + DefineCustomStringVariable("pg_tde.wal_keyring_type", + "Keyring type for XLog", + NULL, + &KRingProviderType, + NULL, + PGC_POSTMASTER, + 0, /* no flags required */ + NULL, + NULL, + NULL + ); + DefineCustomStringVariable("pg_tde.wal_keyring_file_path", + "Keyring file options for XLog", + NULL, + &KRingProviderFilePath, + NULL, + PGC_POSTMASTER, + 0, /* no flags required */ + NULL, + NULL, + NULL + ); } static int @@ -174,6 +260,17 @@ TDEXLogEncryptBuffSize() return (Size) XLOG_BLCKSZ * xbuffers; } +Size +XLogEncStateSize() +{ + Size size; + + size = sizeof(XLogEncryptionState); + size = add_size(size, sizeof(KeyringProviders)); + + return MAXALIGN(size); +} + /* * Alloc memory for the encryption buffer. * @@ -187,10 +284,11 @@ TDEXLogEncryptBuffSize() void TDEXLogShmemInit(void) { + bool foundBuf; + char *allocptr; + if (EncryptXLog) { - bool foundBuf; - TDEXLogEncryptBuf = (char *) TYPEALIGN(PG_IO_ALIGN_SIZE, ShmemInitStruct("TDE XLog Encryption Buffer", @@ -199,12 +297,21 @@ TDEXLogShmemInit(void) elog(DEBUG1, "pg_tde: initialized encryption buffer %lu bytes", XLOG_TDE_ENC_BUFF_ALIGNED_SIZE); } + + EncryptionState = (XLogEncryptionState *) + ShmemInitStruct("TDE XLog Encryption State", + XLogEncStateSize(), &foundBuf); + + allocptr = ((char *) EncryptionState) + MAXALIGN(sizeof(XLogEncryptionState)); + EncryptionState->keyring = allocptr; } void TDEInitXLogSmgr(void) { SetXLogSmgr(&tde_xlog_smgr); + pg_tde_init_xlog_kring(); + pg_tde_create_xlog_key(); } /* @@ -232,11 +339,15 @@ TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset) size_t data_size = 0; XLogPageHeader curr_page_hdr = &EncryptCurrentPageHrd; XLogPageHeader enc_buf_page; - RelKeyData key = {.internal_key = XLogInternalKey}; + // RelKeyData key = {.internal_key = XLogInternalKey}; + RelKeyData *key = NULL; off_t enc_off; size_t page_size = XLOG_BLCKSZ - offset % XLOG_BLCKSZ; uint32 iv_ctr = 0; + pg_tde_init_xlog_kring(); + key = GetInternalKey(GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID), xlog_keyring); + #ifdef TDE_XLOG_DEBUG elog(DEBUG1, "write encrypted WAL, pages amount: %d, size: %lu offset: %ld", count / (Size) XLOG_BLCKSZ, count, offset); @@ -300,7 +411,7 @@ TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset) { SetXLogPageIVPrefix(curr_page_hdr->xlp_tli, curr_page_hdr->xlp_pageaddr, iv_prefix); PG_TDE_ENCRYPT_DATA(iv_prefix, iv_ctr, (char *) buf + enc_off, data_size, - TDEXLogEncryptBuf + enc_off, &key); + TDEXLogEncryptBuf + enc_off, key); } page_size = XLOG_BLCKSZ; @@ -320,7 +431,8 @@ pg_tde_xlog_seg_read(int fd, void *buf, size_t count, off_t offset) char iv_prefix[16] = {0,}; size_t data_size = 0; XLogPageHeader curr_page_hdr = &DecryptCurrentPageHrd; - RelKeyData key = {.internal_key = XLogInternalKey}; + // RelKeyData key = {.internal_key = XLogInternalKey}; + RelKeyData *key = NULL; size_t page_size = XLOG_BLCKSZ - offset % XLOG_BLCKSZ; off_t dec_off; uint32 iv_ctr = 0; @@ -329,6 +441,17 @@ pg_tde_xlog_seg_read(int fd, void *buf, size_t count, off_t offset) elog(DEBUG1, "read from a WAL segment, pages amount: %d, size: %lu offset: %ld", count / (Size) XLOG_BLCKSZ, count, offset); #endif + pg_tde_init_xlog_kring(); + { + char db_map_path[MAXPGPATH] = {0}; + + pg_tde_set_db_file_paths(&GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID), + db_map_path, NULL); + if (access(db_map_path, F_OK) == -1) + pg_tde_create_xlog_key(); + } + key = GetInternalKey(GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID), xlog_keyring); + readsz = pg_pread(fd, buf, count, offset); /* @@ -373,7 +496,7 @@ pg_tde_xlog_seg_read(int fd, void *buf, size_t count, off_t offset) SetXLogPageIVPrefix(curr_page_hdr->xlp_tli, curr_page_hdr->xlp_pageaddr, iv_prefix); PG_TDE_DECRYPT_DATA( iv_prefix, iv_ctr, - (char *) buf + dec_off, data_size, (char *) buf + dec_off, &key); + (char *) buf + dec_off, data_size, (char *) buf + dec_off, key); } page_size = XLOG_BLCKSZ; diff --git a/src/catalog/tde_keyring.c b/src/catalog/tde_keyring.c index fc56deee..5397dcb7 100644 --- a/src/catalog/tde_keyring.c +++ b/src/catalog/tde_keyring.c @@ -50,13 +50,12 @@ PG_FUNCTION_INFO_V1(keyring_delete_dependency_check_trigger); #define FILE_KEYRING_TYPE_KEY "type" static FileKeyring *load_file_keyring_provider_options(Datum keyring_options); -static ProviderType get_keyring_provider_from_typename(char *provider_type); static GenericKeyring *load_keyring_provider_options(ProviderType provider_type, Datum keyring_options); static VaultV2Keyring *load_vaultV2_keyring_provider_options(Datum keyring_options); static void debug_print_kerying(GenericKeyring *keyring); static GenericKeyring *load_keyring_provider_from_tuple(HeapTuple tuple, TupleDesc tupDesc); -static ProviderType +ProviderType get_keyring_provider_from_typename(char *provider_type) { if (provider_type == NULL) diff --git a/src/catalog/tde_master_key.c b/src/catalog/tde_master_key.c index 618e85de..e1b5d913 100644 --- a/src/catalog/tde_master_key.c +++ b/src/catalog/tde_master_key.c @@ -72,7 +72,6 @@ static void clear_master_key_cache(Oid databaseId, Oid tablespaceId) ; static inline dshash_table *get_master_key_Hash(void); static TDEMasterKey *get_master_key_from_cache(Oid dbOid); static void push_master_key_to_cache(TDEMasterKey *masterKey); -static TDEMasterKey *set_master_key_with_keyring(const char *key_name, GenericKeyring *keyring, bool ensure_new_key); static const TDEShmemSetupRoutine master_key_info_shmem_routine = { .init_shared_state = initialize_shared_state, @@ -214,11 +213,10 @@ save_master_key_info(TDEMasterKeyInfo *master_key_info) * throws an error. */ TDEMasterKey * -GetMasterKey(Oid dbOid) +GetMasterKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring) { TDEMasterKey *masterKey = NULL; TDEMasterKeyInfo *masterKeyInfo = NULL; - GenericKeyring *keyring = NULL; const keyInfo *keyInfo = NULL; KeyringReturnCodes keyring_ret; LWLock *lock_files = tde_lwlock_mk_files(); @@ -263,7 +261,7 @@ GetMasterKey(Oid dbOid) } /* Master key not present in cache. Load from the keyring */ - masterKeyInfo = pg_tde_get_master_key(dbOid); + masterKeyInfo = pg_tde_get_master_key(dbOid, spcOid); if (masterKeyInfo == NULL) { LWLockRelease(lock_cache); @@ -273,12 +271,12 @@ GetMasterKey(Oid dbOid) return NULL; } - /* Load the master key from keyring and store it in cache */ - keyring = GetKeyProviderByID(masterKeyInfo->keyringId); - if (keyring == NULL) - { - LWLockRelease(lock_cache); - LWLockRelease(lock_files); + if (keyring == NULL) { + keyring = GetKeyProviderByID(masterKeyInfo->keyringId); + if (keyring == NULL) + { + LWLockRelease(lock_cache); + LWLockRelease(lock_files); recursion--; return NULL; @@ -324,12 +322,11 @@ GetMasterKey(Oid dbOid) * to make sure if some other caller has not added a master key for * same database while we were waiting for the lock. */ - -static TDEMasterKey * -set_master_key_with_keyring(const char *key_name, GenericKeyring *keyring, bool ensure_new_key) +TDEMasterKey * +set_master_key_with_keyring(const char *key_name, GenericKeyring *keyring, + Oid dbOid, Oid spcOid, bool ensure_new_key) { TDEMasterKey *masterKey = NULL; - Oid dbOid = MyDatabaseId; LWLock *lock_files = tde_lwlock_mk_files(); LWLock *lock_cache = tde_lwlock_mk_cache(); bool is_dup_key = false; @@ -345,14 +342,15 @@ set_master_key_with_keyring(const char *key_name, GenericKeyring *keyring, bool /* TODO: Add the key in the cache? */ if (is_dup_key == false) - is_dup_key = (pg_tde_get_master_key(dbOid) != NULL); + is_dup_key = (pg_tde_get_master_key(dbOid, spcOid) != NULL); if (is_dup_key == false) { const keyInfo *keyInfo = NULL; masterKey = palloc(sizeof(TDEMasterKey)); - masterKey->keyInfo.databaseId = MyDatabaseId; + masterKey->keyInfo.databaseId = dbOid; + masterKey->keyInfo.tablespaceId = spcOid; masterKey->keyInfo.keyId.version = DEFAULT_MASTER_KEY_VERSION; masterKey->keyInfo.keyringId = keyring->key_id; strncpy(masterKey->keyInfo.keyId.name, key_name, TDE_KEY_NAME_LEN); @@ -378,9 +376,9 @@ set_master_key_with_keyring(const char *key_name, GenericKeyring *keyring, bool save_master_key_info(&masterKey->keyInfo); /* XLog the new key*/ - XLogBeginInsert(); - XLogRegisterData((char *) &masterKey->keyInfo, sizeof(TDEMasterKeyInfo)); - XLogInsert(RM_TDERMGR_ID, XLOG_TDE_ADD_MASTER_KEY); + // XLogBeginInsert(); + // XLogRegisterData((char *) &masterKey->keyInfo, sizeof(TDEMasterKeyInfo)); + // XLogInsert(RM_TDERMGR_ID, XLOG_TDE_ADD_MASTER_KEY); push_master_key_to_cache(masterKey); } @@ -407,7 +405,10 @@ set_master_key_with_keyring(const char *key_name, GenericKeyring *keyring, bool bool SetMasterKey(const char *key_name, const char *provider_name, bool ensure_new_key) { - TDEMasterKey *master_key = set_master_key_with_keyring(key_name, GetKeyProviderByName(provider_name), ensure_new_key); + TDEMasterKey *master_key = set_master_key_with_keyring(key_name, + GetKeyProviderByName(provider_name), + MyDatabaseId, MyDatabaseTableSpace, + ensure_new_key); return (master_key != NULL); } @@ -415,7 +416,7 @@ SetMasterKey(const char *key_name, const char *provider_name, bool ensure_new_ke bool RotateMasterKey(const char *new_key_name, const char *new_provider_name, bool ensure_new_key) { - TDEMasterKey *master_key = GetMasterKey(MyDatabaseId); + TDEMasterKey *master_key = GetMasterKey(MyDatabaseId, MyDatabaseTableSpace, NULL); TDEMasterKey new_master_key; const keyInfo *keyInfo = NULL; GenericKeyring *keyring; @@ -564,7 +565,7 @@ GetMasterKeyProviderId(void) } { /* Master key not present in cache. Try Loading it from the info file */ - masterKeyInfo = pg_tde_get_master_key(dbOid); + masterKeyInfo = pg_tde_get_master_key(dbOid, MyDatabaseTableSpace); if (masterKeyInfo) { keyringId = masterKeyInfo->keyringId; @@ -671,7 +672,7 @@ cleanup_master_key_info(Oid databaseId, Oid tablespaceId) */ /* Remove the tde files */ - pg_tde_delete_tde_files(databaseId); + pg_tde_delete_tde_files(databaseId, tablespaceId); } static void @@ -748,7 +749,7 @@ Datum pg_tde_master_key_info(PG_FUNCTION_ARGS) (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function returning record called in context that cannot accept type record"))); - master_key = GetMasterKey(MyDatabaseId); + master_key = GetMasterKey(MyDatabaseId, MyDatabaseTableSpace, NULL); if (master_key == NULL) { ereport(ERROR, diff --git a/src/include/access/pg_tde_tdemap.h b/src/include/access/pg_tde_tdemap.h index a0b4a899..4b72bf5e 100644 --- a/src/include/access/pg_tde_tdemap.h +++ b/src/include/access/pg_tde_tdemap.h @@ -10,10 +10,21 @@ #include "utils/rel.h" #include "access/xlog_internal.h" +#include "catalog/pg_tablespace_d.h" #include "catalog/tde_master_key.h" #include "storage/fd.h" #include "storage/relfilelocator.h" +/* + * Neeed for glogbal data (WAL etc) keys identification in caches and storage. + * We take IDs the oid type operators, so there is no overlap with the "real" + * catalog object possible. + */ +#define GLOBAL_DATA_TDE_OID 607 /* Global objects fake "db" */ +#define XLOG_TDE_OID 608 + +#define GLOBAL_SPACE_RLOCATOR(_obj_oid) (RelFileLocator) {GLOBALTABLESPACE_OID, 0, _obj_oid} + typedef struct InternalKey { uint8 key[INTERNAL_KEY_LEN]; @@ -52,14 +63,19 @@ extern void pg_tde_delete_key_map_entry(const RelFileLocator *rlocator); extern void pg_tde_free_key_map_entry(const RelFileLocator *rlocator, off_t offset); extern RelKeyData *GetRelationKey(RelFileLocator rel); +extern RelKeyData *GetInternalKey(RelFileLocator rel, GenericKeyring *keyring); -extern void pg_tde_cleanup_path_vars(void); -extern void pg_tde_delete_tde_files(Oid dbOid); +extern void pg_tde_delete_tde_files(Oid dbOid, Oid spcOid); -extern TDEMasterKeyInfo *pg_tde_get_master_key(Oid dbOid); +extern TDEMasterKeyInfo *pg_tde_get_master_key(Oid dbOid, Oid spcOid); extern bool pg_tde_save_master_key(TDEMasterKeyInfo *master_key_info); extern bool pg_tde_perform_rotate_key(TDEMasterKey *master_key, TDEMasterKey *new_master_key); extern bool pg_tde_write_map_keydata_files(off_t map_size, char *m_file_data, off_t keydata_size, char *k_file_data); +extern RelKeyData* tde_create_rel_key(Oid rel_id, InternalKey *key, TDEMasterKeyInfo *master_key_info); +extern RelKeyData *tde_encrypt_rel_key(TDEMasterKey *master_key, RelKeyData *rel_key_data, const RelFileLocator *rlocator); +extern RelKeyData *tde_decrypt_rel_key(TDEMasterKey *master_key, RelKeyData *enc_rel_key_data, const RelFileLocator *rlocator); + +extern void pg_tde_set_db_file_paths(const RelFileLocator *rlocator, char *map_path, char *keydata_path); const char * tde_sprint_key(InternalKey *k); diff --git a/src/include/access/pg_tde_xlog.h b/src/include/access/pg_tde_xlog.h index f7b37dbe..c380b8b2 100644 --- a/src/include/access/pg_tde_xlog.h +++ b/src/include/access/pg_tde_xlog.h @@ -45,6 +45,7 @@ static const RmgrData pg_tde_rmgr = { extern bool EncryptXLog; extern Size TDEXLogEncryptBuffSize(); +extern Size XLogEncStateSize(); #define XLOG_TDE_ENC_BUFF_ALIGNED_SIZE add_size(TDEXLogEncryptBuffSize(), PG_IO_ALIGN_SIZE) diff --git a/src/include/catalog/tde_keyring.h b/src/include/catalog/tde_keyring.h index 29b2100e..5eb310cd 100644 --- a/src/include/catalog/tde_keyring.h +++ b/src/include/catalog/tde_keyring.h @@ -54,8 +54,15 @@ typedef struct VaultV2Keyring char vault_mount_path[MAXPGPATH]; } VaultV2Keyring; +typedef union KeyringProviders +{ + FileKeyring file; + VaultV2Keyring vault; +} KeyringProviders; + extern List *GetAllKeyringProviders(void); extern GenericKeyring *GetKeyProviderByName(const char *provider_name); extern GenericKeyring *GetKeyProviderByID(int provider_id); +extern ProviderType get_keyring_provider_from_typename(char *provider_type); #endif /*TDE_KEYRING_H*/ diff --git a/src/include/catalog/tde_master_key.h b/src/include/catalog/tde_master_key.h index 77bad07f..a6f82123 100644 --- a/src/include/catalog/tde_master_key.h +++ b/src/include/catalog/tde_master_key.h @@ -68,9 +68,13 @@ extern LWLock *tde_lwlock_mk_cache(void); extern bool save_master_key_info(TDEMasterKeyInfo *masterKeyInfo); extern Oid GetMasterKeyProviderId(void); -extern TDEMasterKey* GetMasterKey(Oid); +extern TDEMasterKey* GetMasterKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring); extern bool SetMasterKey(const char *key_name, const char *provider_name, bool ensure_new_key); extern bool RotateMasterKey(const char *new_key_name, const char *new_provider_name, bool ensure_new_key); extern bool xl_tde_perform_rotate_key(XLogMasterKeyRotate *xlrec); +extern TDEMasterKey *set_master_key_with_keyring(const char *key_name, + GenericKeyring *keyring, + Oid dbOid, Oid spcOid, + bool ensure_new_key); #endif /*PG_TDE_MASTER_KEY_H*/ diff --git a/src/transam/pg_tde_xact_handler.c b/src/transam/pg_tde_xact_handler.c index ff84c3d9..4b0576a0 100644 --- a/src/transam/pg_tde_xact_handler.c +++ b/src/transam/pg_tde_xact_handler.c @@ -52,8 +52,6 @@ pg_tde_xact_callback(XactEvent event, void *arg) { pending_delete_cleanup(); } - - pg_tde_cleanup_path_vars(); } void From 594a582dc43a2b34e1d579976a85b9234263d4c1 Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoy Date: Thu, 16 May 2024 18:39:59 +0300 Subject: [PATCH 2/8] Create and use global catalog key * Check and create an internal key for XLog during the server start. If the key is created (not the first start with the EncryptWAL), then upload it into the cache. We can't read the key from files while writing the XLog to the disk as it happens in the critical section and no palloc is allowed. * Create a custom cache for the global catalog external key as we can't use PG's hashmap during the (again, no pallocs in critical section). --- Makefile.in | 1 + meson.build | 1 + src/access/pg_tde_tdemap.c | 19 +- src/access/pg_tde_xlog.c | 133 +------------- src/catalog/tde_global_catalog.c | 225 +++++++++++++++++++++++ src/catalog/tde_master_key.c | 25 ++- src/include/access/pg_tde_tdemap.h | 10 - src/include/access/pg_tde_xlog.h | 14 +- src/include/catalog/tde_global_catalog.h | 37 ++++ src/include/catalog/tde_master_key.h | 6 +- src/keyring/.keyring_api.c.swp | Bin 12288 -> 0 bytes src/pg_tde.c | 14 +- 12 files changed, 321 insertions(+), 164 deletions(-) create mode 100644 src/catalog/tde_global_catalog.c create mode 100644 src/include/catalog/tde_global_catalog.h delete mode 100644 src/keyring/.keyring_api.c.swp diff --git a/Makefile.in b/Makefile.in index 8f530f2d..7671467a 100644 --- a/Makefile.in +++ b/Makefile.in @@ -40,6 +40,7 @@ src/keyring/keyring_curl.o \ src/keyring/keyring_file.o \ src/keyring/keyring_vault.o \ src/keyring/keyring_api.o \ +src/catalog/tde_global_catalog.o \ src/catalog/tde_keyring.o \ src/catalog/tde_master_key.o \ src/common/pg_tde_shmem.o \ diff --git a/meson.build b/meson.build index b9cb1e87..01ba44b4 100644 --- a/meson.build +++ b/meson.build @@ -40,6 +40,7 @@ pg_tde_sources = files( 'src/smgr/pg_tde_smgr.c', + 'src/catalog/tde_global_catalog.c', 'src/catalog/tde_keyring.c', 'src/catalog/tde_master_key.c', 'src/common/pg_tde_shmem.c', diff --git a/src/access/pg_tde_tdemap.c b/src/access/pg_tde_tdemap.c index 8c4e345f..13ac03d7 100644 --- a/src/access/pg_tde_tdemap.c +++ b/src/access/pg_tde_tdemap.c @@ -292,14 +292,21 @@ tde_decrypt_rel_key(TDEMasterKey *master_key, RelKeyData *enc_rel_key_data, cons inline void pg_tde_set_db_file_paths(const RelFileLocator *rlocator, char *map_path, char *keydata_path) { + char *db_path; + + /* We use dbOid for the global space for key caches but for the backend + * it should be 0. + */ + if (rlocator->spcOid == GLOBALTABLESPACE_OID) + db_path = GetDatabasePath(0, rlocator->spcOid); + else + db_path = GetDatabasePath(rlocator->dbOid, rlocator->spcOid); + + if (map_path) - join_path_components(map_path, - GetDatabasePath(rlocator->dbOid, rlocator->spcOid), - PG_TDE_MAP_FILENAME); + join_path_components(map_path, db_path, PG_TDE_MAP_FILENAME); if (keydata_path) - join_path_components(keydata_path, - GetDatabasePath(rlocator->dbOid, rlocator->spcOid), - PG_TDE_KEYDATA_FILENAME); + join_path_components(keydata_path, db_path, PG_TDE_KEYDATA_FILENAME); } /* diff --git a/src/access/pg_tde_xlog.c b/src/access/pg_tde_xlog.c index bdbbb553..c3c661b6 100644 --- a/src/access/pg_tde_xlog.c +++ b/src/access/pg_tde_xlog.c @@ -24,17 +24,14 @@ #include "access/pg_tde_tdemap.h" #include "access/pg_tde_xlog.h" -#include "catalog/tde_keyring.h" -#include "catalog/tde_master_key.h" +#include "catalog/tde_global_catalog.h" #include "encryption/enc_tde.h" static char *TDEXLogEncryptBuf = NULL; -bool EncryptXLog = false; /* GUC */ -static char *KRingProviderType = NULL; -static char *KRingProviderFilePath = NULL; +static bool EncryptXLog = false; static XLogPageHeaderData EncryptCurrentPageHrd; static XLogPageHeaderData DecryptCurrentPageHrd; @@ -52,15 +49,13 @@ typedef enum } GlobalCatalogKeyTypes; /* TODO: move TDEXLogEncryptBuf here*/ -typedef struct XLogEncryptionState +typedef struct EncryptionStateData { GenericKeyring *keyring; /* TODO: locking */ TDEMasterKey master_keys[TDE_GCAT_KEYS_COUNT]; -} XLogEncryptionState; - -static XLogEncryptionState *EncryptionState = NULL; +} EncryptionStateData; /* * TDE fork XLog @@ -153,52 +148,8 @@ pg_tde_rmgr_identify(uint8 info) * XLog Storage Manager */ -static GenericKeyring *xlog_keyring; - -static void -pg_tde_init_xlog_kring(void) -{ - EncryptionState->keyring->type = get_keyring_provider_from_typename(KRingProviderType); - switch (EncryptionState->keyring->type) - { - case FILE_KEY_PROVIDER: - FileKeyring *kring = (FileKeyring *) EncryptionState->keyring; - strncpy(kring->file_name, KRingProviderFilePath, sizeof(kring->file_name)); - break; - } -} - -static void -pg_tde_create_xlog_key(void) -{ - InternalKey int_key; - RelKeyData *rel_key_data; - RelKeyData *enc_rel_key_data; - RelFileLocator *rlocator = &GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID); - TDEMasterKey *master_key; - - master_key = set_master_key_with_keyring("xlog-master-key", xlog_keyring, - rlocator->dbOid, rlocator->spcOid, false); - - memset(&int_key, 0, sizeof(InternalKey)); - - if (!RAND_bytes(int_key.key, INTERNAL_KEY_LEN)) - { - ereport(FATAL, - (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("could not generate internal key for \"WAL\": %s", - ERR_error_string(ERR_get_error(), NULL)))); - } - - rel_key_data = tde_create_rel_key(rlocator->relNumber, &int_key, &master_key->keyInfo); - enc_rel_key_data = tde_encrypt_rel_key(master_key, rel_key_data, rlocator); - - pg_tde_write_key_map_entry(rlocator, enc_rel_key_data, &master_key->keyInfo); - memcpy(EncryptionState->master_keys + TDE_GCAT_KEY_XLOG, master_key, sizeof(TDEMasterKey)); -} - void -xlogInitGUC(void) +XLogInitGUC(void) { DefineCustomBoolVariable("pg_tde.wal_encrypt", /* name */ "Enable/Disable encryption of WAL.", /* short_desc */ @@ -211,28 +162,6 @@ xlogInitGUC(void) NULL, /* assign_hook */ NULL /* show_hook */ ); - DefineCustomStringVariable("pg_tde.wal_keyring_type", - "Keyring type for XLog", - NULL, - &KRingProviderType, - NULL, - PGC_POSTMASTER, - 0, /* no flags required */ - NULL, - NULL, - NULL - ); - DefineCustomStringVariable("pg_tde.wal_keyring_file_path", - "Keyring file options for XLog", - NULL, - &KRingProviderFilePath, - NULL, - PGC_POSTMASTER, - 0, /* no flags required */ - NULL, - NULL, - NULL - ); } static int @@ -252,7 +181,7 @@ XLOGChooseNumBuffers(void) * Defines the size of the XLog encryption buffer */ Size -TDEXLogEncryptBuffSize() +TDEXLogEncryptBuffSize(void) { int xbuffers; @@ -260,17 +189,6 @@ TDEXLogEncryptBuffSize() return (Size) XLOG_BLCKSZ * xbuffers; } -Size -XLogEncStateSize() -{ - Size size; - - size = sizeof(XLogEncryptionState); - size = add_size(size, sizeof(KeyringProviders)); - - return MAXALIGN(size); -} - /* * Alloc memory for the encryption buffer. * @@ -285,7 +203,6 @@ void TDEXLogShmemInit(void) { bool foundBuf; - char *allocptr; if (EncryptXLog) { @@ -297,29 +214,14 @@ TDEXLogShmemInit(void) elog(DEBUG1, "pg_tde: initialized encryption buffer %lu bytes", XLOG_TDE_ENC_BUFF_ALIGNED_SIZE); } - - EncryptionState = (XLogEncryptionState *) - ShmemInitStruct("TDE XLog Encryption State", - XLogEncStateSize(), &foundBuf); - - allocptr = ((char *) EncryptionState) + MAXALIGN(sizeof(XLogEncryptionState)); - EncryptionState->keyring = allocptr; } void -TDEInitXLogSmgr(void) +TDEXLogSmgrInit(void) { SetXLogSmgr(&tde_xlog_smgr); - pg_tde_init_xlog_kring(); - pg_tde_create_xlog_key(); } -/* - * TODO: proper key management - * where to store refs to the master and internal keys? - */ -static InternalKey XLogInternalKey = {.key = {0xD,}}; - ssize_t pg_tde_xlog_seg_write(int fd, const void *buf, size_t count, off_t offset) { @@ -339,16 +241,11 @@ TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset) size_t data_size = 0; XLogPageHeader curr_page_hdr = &EncryptCurrentPageHrd; XLogPageHeader enc_buf_page; - // RelKeyData key = {.internal_key = XLogInternalKey}; - RelKeyData *key = NULL; + RelKeyData *key = GetGlCatInternalKey(XLOG_TDE_OID); off_t enc_off; size_t page_size = XLOG_BLCKSZ - offset % XLOG_BLCKSZ; uint32 iv_ctr = 0; - pg_tde_init_xlog_kring(); - key = GetInternalKey(GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID), xlog_keyring); - - #ifdef TDE_XLOG_DEBUG elog(DEBUG1, "write encrypted WAL, pages amount: %d, size: %lu offset: %ld", count / (Size) XLOG_BLCKSZ, count, offset); #endif @@ -431,8 +328,7 @@ pg_tde_xlog_seg_read(int fd, void *buf, size_t count, off_t offset) char iv_prefix[16] = {0,}; size_t data_size = 0; XLogPageHeader curr_page_hdr = &DecryptCurrentPageHrd; - // RelKeyData key = {.internal_key = XLogInternalKey}; - RelKeyData *key = NULL; + RelKeyData *key = GetGlCatInternalKey(XLOG_TDE_OID); size_t page_size = XLOG_BLCKSZ - offset % XLOG_BLCKSZ; off_t dec_off; uint32 iv_ctr = 0; @@ -441,17 +337,6 @@ pg_tde_xlog_seg_read(int fd, void *buf, size_t count, off_t offset) elog(DEBUG1, "read from a WAL segment, pages amount: %d, size: %lu offset: %ld", count / (Size) XLOG_BLCKSZ, count, offset); #endif - pg_tde_init_xlog_kring(); - { - char db_map_path[MAXPGPATH] = {0}; - - pg_tde_set_db_file_paths(&GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID), - db_map_path, NULL); - if (access(db_map_path, F_OK) == -1) - pg_tde_create_xlog_key(); - } - key = GetInternalKey(GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID), xlog_keyring); - readsz = pg_pread(fd, buf, count, offset); /* diff --git a/src/catalog/tde_global_catalog.c b/src/catalog/tde_global_catalog.c new file mode 100644 index 00000000..a6c69418 --- /dev/null +++ b/src/catalog/tde_global_catalog.c @@ -0,0 +1,225 @@ +/*------------------------------------------------------------------------- + * + * tde_global_catalog.c + * Global catalog key management + * + * + * IDENTIFICATION + * src/catalog/tde_global_catalog.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "storage/shmem.h" +#include "utils/guc.h" + +#include "access/pg_tde_tdemap.h" +#include "catalog/tde_global_catalog.h" +#include "catalog/tde_keyring.h" +#include "catalog/tde_master_key.h" + +#include +#include +#include + +typedef enum +{ + TDE_GCAT_KEY_XLOG, + + /* must be last */ + TDE_GCAT_KEYS_COUNT +} GlobalCatalogKeyTypes; + +typedef struct EncryptionStateData +{ + GenericKeyring *keyring; + TDEMasterKey master_keys[TDE_GCAT_KEYS_COUNT]; +} EncryptionStateData; + +static EncryptionStateData *EncryptionState = NULL; + +/* GUC */ +static char *KRingProviderType = NULL; +static char *KRingProviderFilePath = NULL; + +static void init_gl_catalog_keys(void); +static void init_keyring(void); +static TDEMasterKey *create_master_key(const char *key_name, + GenericKeyring *keyring, Oid dbOid, Oid spcOid, + bool ensure_new_key); + +void +TDEGlCatInitGUC(void) +{ + DefineCustomStringVariable("pg_tde.global_keyring_type", + "Keyring type for global catalog", + NULL, + &KRingProviderType, + NULL, + PGC_POSTMASTER, + 0, /* no flags required */ + NULL, + NULL, + NULL + ); + DefineCustomStringVariable("pg_tde.global_keyring_file_path", + "Keyring file options for global catalog", + NULL, + &KRingProviderFilePath, + NULL, + PGC_POSTMASTER, + 0, /* no flags required */ + NULL, + NULL, + NULL + ); +} + + +Size +TDEGlCatEncStateSize(void) +{ + Size size; + + size = sizeof(EncryptionStateData); + size = add_size(size, sizeof(KeyringProviders)); + + return MAXALIGN(size); +} + +void +TDEGlCatShmemInit(void) +{ + bool foundBuf; + char *allocptr; + + EncryptionState = (EncryptionStateData *) + ShmemInitStruct("TDE XLog Encryption State", + TDEGlCatEncStateSize(), &foundBuf); + + allocptr = ((char *) EncryptionState) + MAXALIGN(sizeof(EncryptionStateData)); + EncryptionState->keyring = (GenericKeyring *) allocptr; + memset(EncryptionState->keyring, 0, sizeof(KeyringProviders)); + memset(EncryptionState->master_keys, 0, sizeof(TDEMasterKey) * TDE_GCAT_KEYS_COUNT); +} + +void +TDEGlCatKeyInit(void) +{ + char db_map_path[MAXPGPATH] = {0}; + + init_keyring(); + + pg_tde_set_db_file_paths(&GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID), + db_map_path, NULL); + if (access(db_map_path, F_OK) == -1) + { + init_gl_catalog_keys(); + } +} + +TDEMasterKey * +TDEGetGlCatKeyFromCache(void) +{ + TDEMasterKey *mkey; + + mkey = &EncryptionState->master_keys[TDE_GCAT_KEY_XLOG]; + if (mkey->keyLength == 0) + return NULL; + + return mkey; +} + +void +TDEPutGlCatKeyInCache(TDEMasterKey *mkey) +{ + memcpy(EncryptionState->master_keys + TDE_GCAT_KEY_XLOG, mkey, sizeof(TDEMasterKey)); +} + +RelKeyData * +GetGlCatInternalKey(Oid obj_id) +{ + return GetInternalKey(GLOBAL_SPACE_RLOCATOR(obj_id), EncryptionState->keyring); +} + +static void +init_keyring(void) +{ + EncryptionState->keyring->type = get_keyring_provider_from_typename(KRingProviderType); + switch (EncryptionState->keyring->type) + { + case FILE_KEY_PROVIDER: + FileKeyring *kring = (FileKeyring *) EncryptionState->keyring; + strncpy(kring->file_name, KRingProviderFilePath, sizeof(kring->file_name)); + break; + } +} + +/* + * Keys are created during the cluster start only, so no locks needed here. + */ +static void +init_gl_catalog_keys(void) +{ + InternalKey int_key; + RelKeyData *rel_key_data; + RelKeyData *enc_rel_key_data; + RelFileLocator *rlocator; + TDEMasterKey *mkey; + + mkey = create_master_key("global-catalog-master-key", + EncryptionState->keyring, + GLOBAL_DATA_TDE_OID, GLOBALTABLESPACE_OID, false); + + memset(&int_key, 0, sizeof(InternalKey)); + + /* Create and store an internal key for XLog */ + if (!RAND_bytes(int_key.key, INTERNAL_KEY_LEN)) + { + ereport(FATAL, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("could not generate internal key for \"WAL\": %s", + ERR_error_string(ERR_get_error(), NULL)))); + } + + rlocator = &GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID); + rel_key_data = tde_create_rel_key(rlocator->relNumber, &int_key, &mkey->keyInfo); + enc_rel_key_data = tde_encrypt_rel_key(mkey, rel_key_data, rlocator); + pg_tde_write_key_map_entry(rlocator, enc_rel_key_data, &mkey->keyInfo); + + TDEPutGlCatKeyInCache(mkey); +} + +static TDEMasterKey * +create_master_key(const char *key_name, GenericKeyring *keyring, + Oid dbOid, Oid spcOid, bool ensure_new_key) +{ + TDEMasterKey *masterKey; + keyInfo *keyInfo = NULL; + + masterKey = palloc(sizeof(TDEMasterKey)); + masterKey->keyInfo.databaseId = dbOid; + masterKey->keyInfo.tablespaceId = spcOid; + masterKey->keyInfo.keyId.version = DEFAULT_MASTER_KEY_VERSION; + masterKey->keyInfo.keyringId = keyring->key_id; + strncpy(masterKey->keyInfo.keyId.name, key_name, TDE_KEY_NAME_LEN); + gettimeofday(&masterKey->keyInfo.creationTime, NULL); + + keyInfo = load_latest_versioned_key_name(&masterKey->keyInfo, keyring, ensure_new_key); + + if (keyInfo == NULL) + keyInfo = KeyringGenerateNewKeyAndStore(keyring, masterKey->keyInfo.keyId.versioned_name, INTERNAL_KEY_LEN, false); + + if (keyInfo == NULL) + { + ereport(ERROR, + (errmsg("failed to retrieve master key"))); + } + + masterKey->keyLength = keyInfo->data.len; + memcpy(masterKey->keyData, keyInfo->data.data, keyInfo->data.len); + + return masterKey; +} \ No newline at end of file diff --git a/src/catalog/tde_master_key.c b/src/catalog/tde_master_key.c index e1b5d913..2b97ee05 100644 --- a/src/catalog/tde_master_key.c +++ b/src/catalog/tde_master_key.c @@ -29,8 +29,7 @@ #include #include "access/pg_tde_tdemap.h" - -#define DEFAULT_MASTER_KEY_VERSION 1 +#include "catalog/tde_global_catalog.h" typedef struct TdeMasterKeySharedState { @@ -67,7 +66,6 @@ static Size required_shared_mem_size(void); static int required_locks_count(void); static void shared_memory_shutdown(int code, Datum arg); static void master_key_startup_cleanup(int tde_tbl_count, void *arg); -static keyInfo *load_latest_versioned_key_name(TDEMasterKeyInfo *mastere_key_info, GenericKeyring *keyring, bool ensure_new_key); static void clear_master_key_cache(Oid databaseId, Oid tablespaceId) ; static inline dshash_table *get_master_key_Hash(void); static TDEMasterKey *get_master_key_from_cache(Oid dbOid); @@ -250,7 +248,11 @@ GetMasterKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring) LWLockAcquire(lock_files, LW_SHARED); LWLockAcquire(lock_cache, LW_EXCLUSIVE); - masterKey = get_master_key_from_cache(dbOid); + /* Global catalog has its own cache */ + if (spcOid == GLOBALTABLESPACE_OID) + masterKey = TDEGetGlCatKeyFromCache(); + else + masterKey = get_master_key_from_cache(dbOid); if (masterKey) { @@ -299,7 +301,10 @@ GetMasterKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring) masterKey->keyLength = keyInfo->data.len; Assert(MyDatabaseId == masterKey->keyInfo.databaseId); - push_master_key_to_cache(masterKey); + if (spcOid == GLOBALTABLESPACE_OID) + TDEPutGlCatKeyInCache(masterKey); + else + push_master_key_to_cache(masterKey); /* Release the exclusive locks here */ LWLockRelease(lock_cache); @@ -376,10 +381,10 @@ set_master_key_with_keyring(const char *key_name, GenericKeyring *keyring, save_master_key_info(&masterKey->keyInfo); /* XLog the new key*/ - // XLogBeginInsert(); - // XLogRegisterData((char *) &masterKey->keyInfo, sizeof(TDEMasterKeyInfo)); - // XLogInsert(RM_TDERMGR_ID, XLOG_TDE_ADD_MASTER_KEY); - + XLogBeginInsert(); + XLogRegisterData((char *) &masterKey->keyInfo, sizeof(TDEMasterKeyInfo)); + XLogInsert(RM_TDERMGR_ID, XLOG_TDE_ADD_MASTER_KEY); + push_master_key_to_cache(masterKey); } @@ -481,7 +486,7 @@ xl_tde_perform_rotate_key(XLogMasterKeyRotate *xlrec) * If ensure_new_key is true, then we will keep on incrementing the version number * till we get a key name that is not present in the keyring */ -static keyInfo * +keyInfo * load_latest_versioned_key_name(TDEMasterKeyInfo *mastere_key_info, GenericKeyring *keyring, bool ensure_new_key) { KeyringReturnCodes kr_ret; diff --git a/src/include/access/pg_tde_tdemap.h b/src/include/access/pg_tde_tdemap.h index 4b72bf5e..471deb64 100644 --- a/src/include/access/pg_tde_tdemap.h +++ b/src/include/access/pg_tde_tdemap.h @@ -15,16 +15,6 @@ #include "storage/fd.h" #include "storage/relfilelocator.h" -/* - * Neeed for glogbal data (WAL etc) keys identification in caches and storage. - * We take IDs the oid type operators, so there is no overlap with the "real" - * catalog object possible. - */ -#define GLOBAL_DATA_TDE_OID 607 /* Global objects fake "db" */ -#define XLOG_TDE_OID 608 - -#define GLOBAL_SPACE_RLOCATOR(_obj_oid) (RelFileLocator) {GLOBALTABLESPACE_OID, 0, _obj_oid} - typedef struct InternalKey { uint8 key[INTERNAL_KEY_LEN]; diff --git a/src/include/access/pg_tde_xlog.h b/src/include/access/pg_tde_xlog.h index c380b8b2..740eda8d 100644 --- a/src/include/access/pg_tde_xlog.h +++ b/src/include/access/pg_tde_xlog.h @@ -37,15 +37,15 @@ static const RmgrData pg_tde_rmgr = { .rm_identify = pg_tde_rmgr_identify }; +<<<<<<< HEAD #ifdef PERCONA_FORK /* XLog encryption staff */ +======= +/* XLog encryption stuff */ +>>>>>>> e9805ad (Create and use global catalog key) -/* GUC */ -extern bool EncryptXLog; - -extern Size TDEXLogEncryptBuffSize(); -extern Size XLogEncStateSize(); +extern Size TDEXLogEncryptBuffSize(void); #define XLOG_TDE_ENC_BUFF_ALIGNED_SIZE add_size(TDEXLogEncryptBuffSize(), PG_IO_ALIGN_SIZE) @@ -59,9 +59,9 @@ static const XLogSmgr tde_xlog_smgr = { .seg_write = pg_tde_xlog_seg_write, }; -extern void TDEInitXLogSmgr(void); +extern void TDEXLogSmgrInit(void); -extern void xlogInitGUC(void); +extern void XLogInitGUC(void); #endif diff --git a/src/include/catalog/tde_global_catalog.h b/src/include/catalog/tde_global_catalog.h new file mode 100644 index 00000000..1e32e408 --- /dev/null +++ b/src/include/catalog/tde_global_catalog.h @@ -0,0 +1,37 @@ +/*------------------------------------------------------------------------- + * + * tde_global_catalog.h + * Global catalog key management + * + * src/include/catalog/tde_global_catalog.h + * + *------------------------------------------------------------------------- + */ + +#ifndef TDE_GLOBAL_CATALOG_H +#define TDE_GLOBAL_CATALOG_H + +#include "postgres.h" + +#include "catalog/tde_master_key.h" + +/* + * Needed for glogbal data (WAL etc) keys identification in caches and storage. + * We take IDs the oid type operators, so there is no overlap with the "real" + * catalog object possible. + */ +#define GLOBAL_DATA_TDE_OID 607 /* Global objects fake "db" */ +#define XLOG_TDE_OID 608 + +#define GLOBAL_SPACE_RLOCATOR(_obj_oid) (RelFileLocator) {GLOBALTABLESPACE_OID, 0, _obj_oid} + +extern void TDEGlCatInitGUC(void); +extern Size TDEGlCatEncStateSize(void); +extern void TDEGlCatShmemInit(void); +extern void TDEGlCatKeyInit(void); + +extern TDEMasterKey *TDEGetGlCatKeyFromCache(void); +extern void TDEPutGlCatKeyInCache(TDEMasterKey *mkey); +extern RelKeyData *GetGlCatInternalKey(Oid obj_id); + +#endif /*TDE_GLOBAL_CATALOG_H*/ diff --git a/src/include/catalog/tde_master_key.h b/src/include/catalog/tde_master_key.h index a6f82123..2f70c9c9 100644 --- a/src/include/catalog/tde_master_key.h +++ b/src/include/catalog/tde_master_key.h @@ -17,6 +17,7 @@ #include "nodes/pg_list.h" #include "storage/lwlock.h" +#define DEFAULT_MASTER_KEY_VERSION 1 #define MASTER_KEY_NAME_LEN TDE_KEY_NAME_LEN #define MAX_MASTER_KEY_VERSION_NUM 100000 @@ -76,5 +77,8 @@ extern TDEMasterKey *set_master_key_with_keyring(const char *key_name, GenericKeyring *keyring, Oid dbOid, Oid spcOid, bool ensure_new_key); - +extern keyInfo *load_latest_versioned_key_name(TDEMasterKeyInfo *mastere_key_info, + GenericKeyring *keyring, + bool ensure_new_key); + #endif /*PG_TDE_MASTER_KEY_H*/ diff --git a/src/keyring/.keyring_api.c.swp b/src/keyring/.keyring_api.c.swp deleted file mode 100644 index e730fc72a57d6acba563fe1e2dd8fd54b82c05d1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2O>7%Q6vro}e3(+&Qd9|n&@`yXHg?>mEg)K_vSL$(q)z3eEeKI}vpcp|+3c=% z$4%3!fW(Clfy50|;DQi`3JFeJ5EUvQ4u}Js5K#4q5aj@O%KwdbFHiDbL zwatuu1-=9ygV(`p;2BT>C9oG1Kn~mw?gQW4!PwW}Bk&G*8@vFX1P_5M*aS9!>zf$+ z4O|2lz#G5-MX(k8c{^j5zyc$ElnH zEQRa1p?Yj)1}!PS5eSh+cLrDISjukj^mOMa&z$rEf#1|iM|*NKOODG{;4Rlh5O_g` z?=14^%2Tze>f!q2Y`IdOJ2p94nVZWDTq~7I>G7s;MPNullaFRH zWJXE^!uJB1PAtU_@k*^WTjP)M6Qx=e3+7hs#htuZVjXq7<-7&!5CcohU0i zGTjF(^u0@X4V}oLkqeT;{uWtOI8IO5Z_t>Dk z#`@$>O%&k|uzf)9#BQ^+-Oh;~w<4CJ626*QWY@+J|OxT0mdBjEP`AY6xqgx-I* zJoOi&TwEy=LE{N6hwLgDv<2dDL@as1$|Mem;ymcwvMtTlyqESIi_VOqiDK2wc$nKw z&mwmWAJxf94<3Ns;j0pXtWg`NsP+*iTpcWUQ?KoW~l1a zwHIRtyEO0lJso{X*f5-s^jJ?7Fq}Xb)=FPAYEcnP+#4170l|!%$?-UM=Bb=2 zmlIFVs0>hHAmaBOWe>gJs5czS9J-N8W@m!b8v7}w5_X}EW}Jo{gfi`z6m`tiz{A5Z zTO2J^nBsz-ybH0KL7krzj>EH#NuIB*!~+}(r6FyT_GH`%0~j0Fyq*aUFfxNS^dKwY zYjq0mgY10IitsXCeN}6;4UY}EM|xVK!SMpqHf7IzILm6grqi|rKN!7`6avGw##$4J z9z(q&P+PZg6whrZEG&s7^<%6xn&@-s0fS@ebq6l?cG2_QXo^DE!mYhshG~j0ESzz? xW^YG)yq#JTt1Nb&X{!g^QP~=H->v;)P8)5f&!YYz4ZH~VuHQd`PSTSpe*ybIVLbo< diff --git a/src/pg_tde.c b/src/pg_tde.c index 563b60ac..bac49ea6 100644 --- a/src/pg_tde.c +++ b/src/pg_tde.c @@ -32,6 +32,7 @@ #include "utils/builtins.h" #include "pg_tde_defs.h" #include "smgr/pg_tde_smgr.h" +#include "catalog/tde_global_catalog.h" #define MAX_ON_INSTALLS 5 @@ -80,10 +81,12 @@ tde_shmem_startup(void) TdeShmemInit(); AesInit(); -#ifdef PERCONA_FORK + + TDEGlCatShmemInit(); + TDEGlCatKeyInit(); + TDEXLogShmemInit(); - TDEInitXLogSmgr(); -#endif + TDEXLogSmgrInit(); } void @@ -96,9 +99,8 @@ _PG_init(void) keyringRegisterVariables(); InitializeMasterKeyInfo(); -#ifdef PERCONA_FORK - xlogInitGUC(); -#endif + XLogInitGUC(); + TDEGlCatInitGUC(); prev_shmem_request_hook = shmem_request_hook; shmem_request_hook = tde_shmem_request; From 5b4672a026fb644554df62de5460e30473b2b092 Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoy Date: Wed, 5 Jun 2024 18:40:38 +0300 Subject: [PATCH 3/8] Use Linux fd syscalls instead of PG's File Vfd During the server start, when pg_tde module is loading and it needs to read *.map, *.dat file, InitFileAccess is yet to be called, hence Vfd isn't ready to use. The same gonna happen during recovery. So use raw pread/pwrite calls istead. --- src/access/pg_tde_tdemap.c | 276 +++++++++++++---------- src/access/pg_tde_xlog.c | 16 -- src/catalog/tde_global_catalog.c | 83 ++++--- src/catalog/tde_master_key.c | 11 +- src/include/access/pg_tde_tdemap.h | 2 +- src/include/catalog/tde_global_catalog.h | 6 +- src/keyring/keyring_file.c | 69 +++--- 7 files changed, 261 insertions(+), 202 deletions(-) diff --git a/src/access/pg_tde_tdemap.c b/src/access/pg_tde_tdemap.c index 13ac03d7..9520e4c7 100644 --- a/src/access/pg_tde_tdemap.c +++ b/src/access/pg_tde_tdemap.c @@ -81,24 +81,24 @@ typedef struct TDEMapFilePath static void put_key_into_map(Oid rel_id, RelKeyData *key); -static File pg_tde_open_file_basic(char *tde_filename, int fileFlags, bool ignore_missing); -static File pg_tde_file_header_write(char *tde_filename, File tde_file, TDEMasterKeyInfo *master_key_info, off_t *bytes_written); -static File pg_tde_file_header_read(char *tde_filename, File tde_file, TDEFileHeader *fheader, bool *is_new_file, off_t *bytes_read); +static int pg_tde_open_file_basic(char *tde_filename, int fileFlags, bool ignore_missing); +static int pg_tde_file_header_write(char *tde_filename, int fd, TDEMasterKeyInfo *master_key_info, off_t *bytes_written); +static int pg_tde_file_header_read(char *tde_filename, int fd, TDEFileHeader *fheader, bool *is_new_file, off_t *bytes_read); -static File pg_tde_open_file(char *tde_filename, TDEMasterKeyInfo *master_key_info, bool should_fill_info, int fileFlags, bool *is_new_file, off_t *offset); +static int pg_tde_open_file(char *tde_filename, TDEMasterKeyInfo *master_key_info, bool should_fill_info, int fileFlags, bool *is_new_file, off_t *offset); static int32 pg_tde_write_map_entry(const RelFileLocator *rlocator, char *db_map_path, TDEMasterKeyInfo *master_key_info); -static off_t pg_tde_write_one_map_entry(File map_file, const RelFileLocator *rlocator, int flags, int32 key_index, TDEMapEntry *map_entry, off_t *offset); +static off_t pg_tde_write_one_map_entry(int fd, const RelFileLocator *rlocator, int flags, int32 key_index, TDEMapEntry *map_entry, off_t *offset); static int32 pg_tde_process_map_entry(const RelFileLocator *rlocator, char *db_map_path, off_t *offset, bool should_delete); -static bool pg_tde_read_one_map_entry(File map_file, const RelFileLocator *rlocator, int flags, TDEMapEntry *map_entry, off_t *offset); +static bool pg_tde_read_one_map_entry(int fd, const RelFileLocator *rlocator, int flags, TDEMapEntry *map_entry, off_t *offset); static void pg_tde_write_keydata(char *db_keydata_path, TDEMasterKeyInfo *master_key_info, int32 key_index, RelKeyData *enc_rel_key_data); -static void pg_tde_write_one_keydata(File keydata_file, int32 key_index, RelKeyData *enc_rel_key_data); +static void pg_tde_write_one_keydata(int keydata_fd, int32 key_index, RelKeyData *enc_rel_key_data); static RelKeyData* pg_tde_get_key_from_file(const RelFileLocator *rlocator, GenericKeyring *keyring); static RelKeyData* pg_tde_read_keydata(char *db_keydata_path, int32 key_index, TDEMasterKey *master_key); -static RelKeyData* pg_tde_read_one_keydata(File keydata_file, int32 key_index, TDEMasterKey *master_key); +static RelKeyData* pg_tde_read_one_keydata(int keydata_fd, int32 key_index, TDEMasterKey *master_key); -static File keyrotation_init_file(TDEMasterKeyInfo *new_master_key_info, char *rotated_filename, char *filename, bool *is_new_file, off_t *curr_pos); +static int keyrotation_init_file(TDEMasterKeyInfo *new_master_key_info, char *rotated_filename, char *filename, bool *is_new_file, off_t *curr_pos); static void finalize_key_rotation(char *m_path_old, char *k_path_old, char *m_path_new, char *k_path_new); /* @@ -188,7 +188,7 @@ GetRelationKey(RelFileLocator rel) } RelKeyData * -GetInternalKey(RelFileLocator rel, GenericKeyring *keyring) +GetRelationKeyWithKeyring(RelFileLocator rel, GenericKeyring *keyring) { RelKey *curr; RelKeyData *key; @@ -343,8 +343,8 @@ pg_tde_delete_tde_files(Oid dbOid, Oid spcOid) bool pg_tde_save_master_key(TDEMasterKeyInfo *master_key_info) { - File map_file = -1; - File keydata_file = -1; + int map_fd = -1; + int keydata_fd = -1; off_t curr_pos = 0; bool is_new_map = false; bool is_new_key_data = false; @@ -361,12 +361,12 @@ pg_tde_save_master_key(TDEMasterKeyInfo *master_key_info) ereport(LOG, (errmsg("pg_tde_save_master_key"))); /* Create or truncate these map and keydata files. */ - map_file = pg_tde_open_file(db_map_path, master_key_info, false, O_RDWR | O_CREAT | O_TRUNC, &is_new_map, &curr_pos); - keydata_file = pg_tde_open_file(db_keydata_path, master_key_info, false, O_RDWR | O_CREAT | O_TRUNC, &is_new_key_data, &curr_pos); + map_fd = pg_tde_open_file(db_map_path, master_key_info, false, O_RDWR | O_CREAT | O_TRUNC, &is_new_map, &curr_pos); + keydata_fd = pg_tde_open_file(db_keydata_path, master_key_info, false, O_RDWR | O_CREAT | O_TRUNC, &is_new_key_data, &curr_pos); /* Closing files. */ - FileClose(map_file); - FileClose(keydata_file); + close(map_fd); + close(keydata_fd); return (is_new_map && is_new_key_data); } @@ -378,7 +378,7 @@ pg_tde_save_master_key(TDEMasterKeyInfo *master_key_info) TDEMasterKeyInfo * pg_tde_get_master_key(Oid dbOid, Oid spcOid) { - File tde_file = -1; + int fd = -1; TDEFileHeader fheader; TDEMasterKeyInfo *master_key_info = NULL; bool is_new_file = false; @@ -397,15 +397,15 @@ pg_tde_get_master_key(Oid dbOid, Oid spcOid) * Ensuring that we always open the file in binary mode. The caller must * specify other flags for reading, writing or creating the file. */ - tde_file = pg_tde_open_file_basic(db_map_path, O_RDONLY, true); + fd = pg_tde_open_file_basic(db_map_path, O_RDONLY, true); /* The file does not exist. */ - if (tde_file < 0) + if (fd < 0) return NULL; - pg_tde_file_header_read(db_map_path, tde_file, &fheader, &is_new_file, &bytes_read); + pg_tde_file_header_read(db_map_path, fd, &fheader, &is_new_file, &bytes_read); - FileClose(tde_file); + close(fd); /* It's not a new file. So we can memcpy the master key info from the header */ if (!is_new_file) @@ -425,32 +425,32 @@ pg_tde_get_master_key(Oid dbOid, Oid spcOid) * Returns the file descriptor in case of a success. Otherwise, fatal error * is raised except when ignore_missing is true and the file does not exit. */ -static File +static int pg_tde_open_file_basic(char *tde_filename, int fileFlags, bool ignore_missing) { - File tde_file = -1; + int fd = -1; /* * Ensuring that we always open the file in binary mode. The caller must * specify other flags for reading, writing or creating the file. */ - tde_file = PathNameOpenFile(tde_filename, fileFlags | PG_BINARY); - if (tde_file < 0 && !(errno == ENOENT && ignore_missing == true)) + fd = BasicOpenFile(tde_filename, fileFlags | PG_BINARY); + if (fd < 0 && !(errno == ENOENT && ignore_missing == true)) { ereport(PANIC, (errcode_for_file_access(), - errmsg("Could not open tde file \"%s\": %m", + errmsg("could not open tde file \"%s\": %m", tde_filename))); } - return tde_file; + return fd; } /* * Write TDE file header to a TDE file. */ -static File -pg_tde_file_header_write(char *tde_filename, File tde_file, TDEMasterKeyInfo *master_key_info, off_t *bytes_written) +static int +pg_tde_file_header_write(char *tde_filename, int fd, TDEMasterKeyInfo *master_key_info, off_t *bytes_written) { TDEFileHeader fheader; size_t sz = sizeof(TDEMasterKeyInfo); @@ -464,33 +464,42 @@ pg_tde_file_header_write(char *tde_filename, File tde_file, TDEMasterKeyInfo *ma memset(&fheader.master_key_info, 0, sz); memcpy(&fheader.master_key_info, master_key_info, sz); - *bytes_written = FileWrite(tde_file, &fheader, TDE_FILE_HEADER_SIZE, 0, WAIT_EVENT_DATA_FILE_WRITE); + /* TODO: pgstat_report_wait_start / pgstat_report_wait_end */ + *bytes_written = pg_pwrite(fd, &fheader, TDE_FILE_HEADER_SIZE, 0); if (*bytes_written != TDE_FILE_HEADER_SIZE) { ereport(ERROR, (errcode_for_file_access(), - errmsg("Could not write tde file \"%s\": %m", + errmsg("could not write tde file \"%s\": %m", tde_filename))); } - return tde_file; + if (pg_fsync(fd) != 0) + { + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", tde_filename))); + } + + return fd; } /* * Read TDE file header from a TDE file and fill in the fheader data structure. */ -static File -pg_tde_file_header_read(char *tde_filename, File tde_file, TDEFileHeader *fheader, bool *is_new_file, off_t *bytes_read) +static int +pg_tde_file_header_read(char *tde_filename, int fd, TDEFileHeader *fheader, bool *is_new_file, off_t *bytes_read) { Assert(fheader); - *bytes_read = FileRead(tde_file, fheader, TDE_FILE_HEADER_SIZE, 0, WAIT_EVENT_DATA_FILE_READ); + /* TODO: pgstat_report_wait_start / pgstat_report_wait_end */ + *bytes_read = pg_pread(fd, fheader, TDE_FILE_HEADER_SIZE, 0); *is_new_file = (*bytes_read == 0); /* File doesn't exist */ if (*bytes_read == 0) - return tde_file; + return fd; if (*bytes_read != TDE_FILE_HEADER_SIZE || fheader->file_version != PG_TDE_FILEMAGIC) @@ -502,7 +511,7 @@ pg_tde_file_header_read(char *tde_filename, File tde_file, TDEFileHeader *fheade tde_filename))); } - return tde_file; + return fd; } /* @@ -522,10 +531,10 @@ pg_tde_file_header_read(char *tde_filename, File tde_file, TDEFileHeader *fheade * The caller can pass the required flags to ensure that file is created * or an error is thrown if the file does not exist. */ -File +int pg_tde_open_file(char *tde_filename, TDEMasterKeyInfo *master_key_info, bool should_fill_info, int fileFlags, bool *is_new_file, off_t *curr_pos) { - File tde_file = -1; + int fd = -1; TDEFileHeader fheader; off_t bytes_read = 0; off_t bytes_written = 0; @@ -534,16 +543,16 @@ pg_tde_open_file(char *tde_filename, TDEMasterKeyInfo *master_key_info, bool sho * Ensuring that we always open the file in binary mode. The caller must * specify other flags for reading, writing or creating the file. */ - tde_file = pg_tde_open_file_basic(tde_filename, fileFlags, false); + fd = pg_tde_open_file_basic(tde_filename, fileFlags, false); - pg_tde_file_header_read(tde_filename, tde_file, &fheader, is_new_file, &bytes_read); + pg_tde_file_header_read(tde_filename, fd, &fheader, is_new_file, &bytes_read); /* In case it's a new file, let's add the header now. */ if (*is_new_file && master_key_info) - pg_tde_file_header_write(tde_filename, tde_file, master_key_info, &bytes_written); + pg_tde_file_header_write(tde_filename, fd, master_key_info, &bytes_written); *curr_pos = bytes_read + bytes_written; - return tde_file; + return fd; } /* @@ -558,7 +567,7 @@ pg_tde_open_file(char *tde_filename, TDEMasterKeyInfo *master_key_info, bool sho static int32 pg_tde_write_map_entry(const RelFileLocator *rlocator, char *db_map_path, TDEMasterKeyInfo *master_key_info) { - File map_file = -1; + int map_fd = -1; int32 key_index = 0; TDEMapEntry map_entry; bool is_new_file; @@ -567,7 +576,7 @@ pg_tde_write_map_entry(const RelFileLocator *rlocator, char *db_map_path, TDEMas bool found = false; /* Open and vaidate file for basic correctness. */ - map_file = pg_tde_open_file(db_map_path, master_key_info, false, O_RDWR | O_CREAT, &is_new_file, &curr_pos); + map_fd = pg_tde_open_file(db_map_path, master_key_info, false, O_RDWR | O_CREAT, &is_new_file, &curr_pos); prev_pos = curr_pos; /* @@ -578,7 +587,7 @@ pg_tde_write_map_entry(const RelFileLocator *rlocator, char *db_map_path, TDEMas while(1) { prev_pos = curr_pos; - found = pg_tde_read_one_map_entry(map_file, NULL, MAP_ENTRY_FREE, &map_entry, &curr_pos); + found = pg_tde_read_one_map_entry(map_fd, NULL, MAP_ENTRY_FREE, &map_entry, &curr_pos); /* We either reach EOF or found an empty slot in the middle of the file */ if (prev_pos == curr_pos || found) @@ -590,10 +599,10 @@ pg_tde_write_map_entry(const RelFileLocator *rlocator, char *db_map_path, TDEMas /* Write the given entry at the location pointed by prev_pos; i.e. the free entry */ curr_pos = prev_pos; - pg_tde_write_one_map_entry(map_file, rlocator, MAP_ENTRY_VALID, key_index, &map_entry, &prev_pos); + pg_tde_write_one_map_entry(map_fd, rlocator, MAP_ENTRY_VALID, key_index, &map_entry, &prev_pos); /* Let's close the file. */ - FileClose(map_file); + close(map_fd); /* Register the entry to be freed in case the transaction aborts */ RegisterEntryForDeletion(rlocator, curr_pos, false); @@ -606,7 +615,7 @@ pg_tde_write_map_entry(const RelFileLocator *rlocator, char *db_map_path, TDEMas * map file. */ static off_t -pg_tde_write_one_map_entry(File map_file, const RelFileLocator *rlocator, int flags, int32 key_index, TDEMapEntry *map_entry, off_t *offset) +pg_tde_write_one_map_entry(int fd, const RelFileLocator *rlocator, int flags, int32 key_index, TDEMapEntry *map_entry, off_t *offset) { int bytes_written = 0; @@ -617,7 +626,8 @@ pg_tde_write_one_map_entry(File map_file, const RelFileLocator *rlocator, int fl map_entry->flags = flags; map_entry->key_index = key_index; - bytes_written = FileWrite(map_file, map_entry, MAP_ENTRY_SIZE, *offset, WAIT_EVENT_DATA_FILE_WRITE); + /* TODO: pgstat_report_wait_start / pgstat_report_wait_end */ + bytes_written = pg_pwrite(fd, map_entry, MAP_ENTRY_SIZE, *offset); /* Add the entry to the file */ if (bytes_written != MAP_ENTRY_SIZE) @@ -626,9 +636,17 @@ pg_tde_write_one_map_entry(File map_file, const RelFileLocator *rlocator, int fl pg_tde_set_db_file_paths(rlocator, db_map_path, NULL); ereport(FATAL, (errcode_for_file_access(), - errmsg("Could not write tde map file \"%s\": %m", + errmsg("could not write tde map file \"%s\": %m", db_map_path))); } + if (pg_fsync(fd) != 0) + { + char db_map_path[MAXPGPATH] = {0}; + pg_tde_set_db_file_paths(rlocator, db_map_path, NULL); + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", db_map_path))); + } return (*offset + bytes_written); } @@ -645,7 +663,7 @@ pg_tde_write_one_map_entry(File map_file, const RelFileLocator *rlocator, int fl static int32 pg_tde_process_map_entry(const RelFileLocator *rlocator, char *db_map_path, off_t *offset, bool should_delete) { - File map_file = -1; + File map_fd = -1; int32 key_index = 0; TDEMapEntry map_entry; bool is_new_file; @@ -659,7 +677,7 @@ pg_tde_process_map_entry(const RelFileLocator *rlocator, char *db_map_path, off_ * Open and validate file for basic correctness. DO NOT create it. * The file should pre-exist otherwise we should never be here. */ - map_file = pg_tde_open_file(db_map_path, NULL, false, O_RDWR, &is_new_file, &curr_pos); + map_fd = pg_tde_open_file(db_map_path, NULL, false, O_RDWR, &is_new_file, &curr_pos); /* * If we need to delete an entry, we expect an offset value to the start @@ -668,13 +686,13 @@ pg_tde_process_map_entry(const RelFileLocator *rlocator, char *db_map_path, off_ */ if (should_delete == true && *offset > 0) { - curr_pos = lseek(map_file, *offset, SEEK_SET); + curr_pos = lseek(map_fd, *offset, SEEK_SET); if (curr_pos == -1) { ereport(FATAL, (errcode_for_file_access(), - errmsg("Could not seek in tde map file \"%s\": %m", + errmsg("could not seek in tde map file \"%s\": %m", db_map_path))); } } @@ -692,7 +710,7 @@ pg_tde_process_map_entry(const RelFileLocator *rlocator, char *db_map_path, off_ while(1) { prev_pos = curr_pos; - found = pg_tde_read_one_map_entry(map_file, rlocator, MAP_ENTRY_VALID, &map_entry, &curr_pos); + found = pg_tde_read_one_map_entry(map_fd, rlocator, MAP_ENTRY_VALID, &map_entry, &curr_pos); /* We've reached EOF */ if (curr_pos == prev_pos) @@ -704,7 +722,7 @@ pg_tde_process_map_entry(const RelFileLocator *rlocator, char *db_map_path, off_ /* Mark the entry pointed by prev_pos as free */ if (should_delete) { - pg_tde_write_one_map_entry(map_file, NULL, MAP_ENTRY_FREE, 0, &map_entry, &prev_pos); + pg_tde_write_one_map_entry(map_fd, NULL, MAP_ENTRY_FREE, 0, &map_entry, &prev_pos); } break; @@ -715,7 +733,7 @@ pg_tde_process_map_entry(const RelFileLocator *rlocator, char *db_map_path, off_ } /* Let's close the file. */ - FileClose(map_file); + close(map_fd); /* Return -1 indicating that no entry was removed */ return ((found) ? key_index : -1); @@ -742,7 +760,8 @@ pg_tde_read_one_map_entry(File map_file, const RelFileLocator *rlocator, int fla Assert(offset); /* Read the entry at the given offset */ - bytes_read = FileRead(map_file, map_entry, MAP_ENTRY_SIZE, *offset, WAIT_EVENT_DATA_FILE_READ); + /* TODO: pgstat_report_wait_start / pgstat_report_wait_end */ + bytes_read = pg_pread(map_file, map_entry, MAP_ENTRY_SIZE, *offset); /* We've reached the end of the file. */ if (bytes_read != MAP_ENTRY_SIZE) @@ -771,38 +790,46 @@ pg_tde_read_one_map_entry(File map_file, const RelFileLocator *rlocator, int fla static void pg_tde_write_keydata(char *db_keydata_path, TDEMasterKeyInfo *master_key_info, int32 key_index, RelKeyData *enc_rel_key_data) { - File keydata_file = -1; + File fd = -1; bool is_new_file; off_t curr_pos = 0; /* Open and validate file for basic correctness. */ - keydata_file = pg_tde_open_file(db_keydata_path, master_key_info, false, O_RDWR | O_CREAT, &is_new_file, &curr_pos); + fd = pg_tde_open_file(db_keydata_path, master_key_info, false, O_RDWR | O_CREAT, &is_new_file, &curr_pos); /* Write a single key data */ - pg_tde_write_one_keydata(keydata_file, key_index, enc_rel_key_data); + pg_tde_write_one_keydata(fd, key_index, enc_rel_key_data); /* Let's close the file. */ - FileClose(keydata_file); + close(fd); } /* * Function writes a single RelKeyData into the file at the given index. */ static void -pg_tde_write_one_keydata(File keydata_file, int32 key_index, RelKeyData *enc_rel_key_data) +pg_tde_write_one_keydata(int fd, int32 key_index, RelKeyData *enc_rel_key_data) { off_t curr_pos; - Assert(keydata_file != -1); + Assert(fd != -1); /* Calculate the writing position in the file. */ curr_pos = (key_index * INTERNAL_KEY_LEN) + TDE_FILE_HEADER_SIZE; - if (FileWrite(keydata_file, &enc_rel_key_data->internal_key, INTERNAL_KEY_LEN, curr_pos, WAIT_EVENT_DATA_FILE_WRITE) != INTERNAL_KEY_LEN) + /* TODO: pgstat_report_wait_start / pgstat_report_wait_end */ + if (pg_pwrite(fd, &enc_rel_key_data->internal_key, INTERNAL_KEY_LEN, curr_pos) != INTERNAL_KEY_LEN) { ereport(FATAL, (errcode_for_file_access(), - errmsg("Could not write tde key data file: %m"))); + errmsg("could not write tde key data file: %m"))); + } + + if (pg_fsync(fd) != 0) + { + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync file: %m"))); } } @@ -812,7 +839,7 @@ pg_tde_write_one_keydata(File keydata_file, int32 key_index, RelKeyData *enc_rel static RelKeyData * pg_tde_read_keydata(char *db_keydata_path, int32 key_index, TDEMasterKey *master_key) { - File keydata_file = -1; + int fd = -1; RelKeyData *enc_rel_key_data; off_t read_pos = 0; bool is_new_file; @@ -820,13 +847,13 @@ pg_tde_read_keydata(char *db_keydata_path, int32 key_index, TDEMasterKey *master /* Open and validate file for basic correctness. */ LWLockAcquire(lock_files, LW_SHARED); - keydata_file = pg_tde_open_file(db_keydata_path, &master_key->keyInfo, false, O_RDONLY, &is_new_file, &read_pos); + fd = pg_tde_open_file(db_keydata_path, &master_key->keyInfo, false, O_RDONLY, &is_new_file, &read_pos); /* Read the encrypted key from file */ - enc_rel_key_data = pg_tde_read_one_keydata(keydata_file, key_index, master_key); + enc_rel_key_data = pg_tde_read_one_keydata(fd, key_index, master_key); /* Let's close the file. */ - FileClose(keydata_file); + close(fd); LWLockRelease(lock_files); return enc_rel_key_data; @@ -836,7 +863,7 @@ pg_tde_read_keydata(char *db_keydata_path, int32 key_index, TDEMasterKey *master * Reads a single keydata from the file. */ static RelKeyData * -pg_tde_read_one_keydata(File keydata_file, int32 key_index, TDEMasterKey *master_key) +pg_tde_read_one_keydata(int keydata_fd, int32 key_index, TDEMasterKey *master_key) { RelKeyData *enc_rel_key_data; off_t read_pos = 0; @@ -850,7 +877,7 @@ pg_tde_read_one_keydata(File keydata_file, int32 key_index, TDEMasterKey *master read_pos += (key_index * INTERNAL_KEY_LEN) + TDE_FILE_HEADER_SIZE; /* Check if the file has a valid key */ - if ((read_pos + INTERNAL_KEY_LEN) > FileSize(keydata_file)) + if ((read_pos + INTERNAL_KEY_LEN) > lseek(keydata_fd, 0, SEEK_END)) { char db_keydata_path[MAXPGPATH] = {0}; pg_tde_set_db_file_paths(&(RelFileLocator) { @@ -860,13 +887,14 @@ pg_tde_read_one_keydata(File keydata_file, int32 key_index, TDEMasterKey *master NULL, db_keydata_path); ereport(FATAL, (errcode(ERRCODE_NO_DATA_FOUND), - errmsg("Could not find the required key at index %d in tde data file \"%s\": %m", + errmsg("could not find the required key at index %d in tde data file \"%s\": %m", key_index, db_keydata_path))); } /* Read the encrypted key */ - if (FileRead(keydata_file, &(enc_rel_key_data->internal_key), INTERNAL_KEY_LEN, read_pos, WAIT_EVENT_DATA_FILE_READ) != INTERNAL_KEY_LEN) + /* TODO: pgstat_report_wait_start / pgstat_report_wait_end */ + if (pg_pread(keydata_fd, &(enc_rel_key_data->internal_key), INTERNAL_KEY_LEN, read_pos) != INTERNAL_KEY_LEN) { char db_keydata_path[MAXPGPATH] = {0}; pg_tde_set_db_file_paths(&(RelFileLocator) { @@ -876,7 +904,7 @@ pg_tde_read_one_keydata(File keydata_file, int32 key_index, TDEMasterKey *master NULL, db_keydata_path); ereport(FATAL, (errcode_for_file_access(), - errmsg("Could not read key at index %d in tde key data file \"%s\": %m", + errmsg("could not read key at index %d in tde key data file \"%s\": %m", key_index, db_keydata_path))); } @@ -940,7 +968,7 @@ pg_tde_delete_key_map_entry(const RelFileLocator *rlocator) { ereport(WARNING, (errcode(ERRCODE_NO_DATA_FOUND), - errmsg("Could not find the required map entry for deletion of relation %d in tde map file \"%s\": %m", + errmsg("could not find the required map entry for deletion of relation %d in tde map file \"%s\": %m", rlocator->relNumber, db_map_path))); @@ -984,7 +1012,7 @@ pg_tde_free_key_map_entry(const RelFileLocator *rlocator, off_t offset) { ereport(WARNING, (errcode(ERRCODE_NO_DATA_FOUND), - errmsg("Could not find the required map entry for deletion of relation %d in tde map file \"%s\": %m", + errmsg("could not find the required map entry for deletion of relation %d in tde map file \"%s\": %m", rlocator->relNumber, db_map_path))); @@ -1088,8 +1116,8 @@ pg_tde_perform_rotate_key(TDEMasterKey *master_key, TDEMasterKey *new_master_key int32 key_index[MASTER_KEY_COUNT] = {0}; RelKeyData *rel_key_data[MASTER_KEY_COUNT]; RelKeyData *enc_rel_key_data[MASTER_KEY_COUNT]; - File m_file[MASTER_KEY_COUNT] = {-1}; - File k_file[MASTER_KEY_COUNT] = {-1}; + int m_fd[MASTER_KEY_COUNT] = {-1}; + int k_fd[MASTER_KEY_COUNT] = {-1}; char m_path[MASTER_KEY_COUNT][MAXPGPATH]; char k_path[MASTER_KEY_COUNT][MAXPGPATH]; TDEMapEntry map_entry; @@ -1121,17 +1149,17 @@ pg_tde_perform_rotate_key(TDEMasterKey *master_key, TDEMasterKey *new_master_key LWLockAcquire(lock_cache, LW_EXCLUSIVE); /* Open both files in read only mode. We don't need to track the current position of the keydata file. We always use the key index */ - m_file[OLD_MASTER_KEY] = pg_tde_open_file(m_path[OLD_MASTER_KEY], &master_key->keyInfo, false, O_RDONLY, &is_new_file, &curr_pos[OLD_MASTER_KEY]); - k_file[OLD_MASTER_KEY] = pg_tde_open_file(k_path[OLD_MASTER_KEY], &master_key->keyInfo, false, O_RDONLY, &is_new_file, &read_pos_tmp); + m_fd[OLD_MASTER_KEY] = pg_tde_open_file(m_path[OLD_MASTER_KEY], &master_key->keyInfo, false, O_RDONLY, &is_new_file, &curr_pos[OLD_MASTER_KEY]); + k_fd[OLD_MASTER_KEY] = pg_tde_open_file(k_path[OLD_MASTER_KEY], &master_key->keyInfo, false, O_RDONLY, &is_new_file, &read_pos_tmp); - m_file[NEW_MASTER_KEY] = keyrotation_init_file(&new_master_key->keyInfo, m_path[NEW_MASTER_KEY], m_path[OLD_MASTER_KEY], &is_new_file, &curr_pos[NEW_MASTER_KEY]); - k_file[NEW_MASTER_KEY] = keyrotation_init_file(&new_master_key->keyInfo, k_path[NEW_MASTER_KEY], k_path[OLD_MASTER_KEY], &is_new_file, &read_pos_tmp); + m_fd[NEW_MASTER_KEY] = keyrotation_init_file(&new_master_key->keyInfo, m_path[NEW_MASTER_KEY], m_path[OLD_MASTER_KEY], &is_new_file, &curr_pos[NEW_MASTER_KEY]); + k_fd[NEW_MASTER_KEY] = keyrotation_init_file(&new_master_key->keyInfo, k_path[NEW_MASTER_KEY], k_path[OLD_MASTER_KEY], &is_new_file, &read_pos_tmp); /* Read all entries until EOF */ for(key_index[OLD_MASTER_KEY] = 0; ; key_index[OLD_MASTER_KEY]++) { prev_pos[OLD_MASTER_KEY] = curr_pos[OLD_MASTER_KEY]; - found = pg_tde_read_one_map_entry(m_file[OLD_MASTER_KEY], NULL, MAP_ENTRY_VALID, &map_entry, &curr_pos[OLD_MASTER_KEY]); + found = pg_tde_read_one_map_entry(m_fd[OLD_MASTER_KEY], NULL, MAP_ENTRY_VALID, &map_entry, &curr_pos[OLD_MASTER_KEY]); /* We either reach EOF */ if (prev_pos[OLD_MASTER_KEY] == curr_pos[OLD_MASTER_KEY]) @@ -1147,7 +1175,7 @@ pg_tde_perform_rotate_key(TDEMasterKey *master_key, TDEMasterKey *new_master_key rloc.spcOid = DEFAULTTABLESPACE_OID; /* Let's get the decrypted key and re-encrypt it with the new key. */ - enc_rel_key_data[OLD_MASTER_KEY] = pg_tde_read_one_keydata(k_file[OLD_MASTER_KEY], key_index[OLD_MASTER_KEY], master_key); + enc_rel_key_data[OLD_MASTER_KEY] = pg_tde_read_one_keydata(k_fd[OLD_MASTER_KEY], key_index[OLD_MASTER_KEY], master_key); /* Decrypt and re-encrypt keys */ rel_key_data[OLD_MASTER_KEY] = tde_decrypt_rel_key(master_key, enc_rel_key_data[OLD_MASTER_KEY], &rloc); @@ -1155,20 +1183,20 @@ pg_tde_perform_rotate_key(TDEMasterKey *master_key, TDEMasterKey *new_master_key /* Write the given entry at the location pointed by prev_pos */ prev_pos[NEW_MASTER_KEY] = curr_pos[NEW_MASTER_KEY]; - curr_pos[NEW_MASTER_KEY] = pg_tde_write_one_map_entry(m_file[NEW_MASTER_KEY], &rloc, MAP_ENTRY_VALID, key_index[NEW_MASTER_KEY], &map_entry, &prev_pos[NEW_MASTER_KEY]); - pg_tde_write_one_keydata(k_file[NEW_MASTER_KEY], key_index[NEW_MASTER_KEY], enc_rel_key_data[NEW_MASTER_KEY]); + curr_pos[NEW_MASTER_KEY] = pg_tde_write_one_map_entry(m_fd[NEW_MASTER_KEY], &rloc, MAP_ENTRY_VALID, key_index[NEW_MASTER_KEY], &map_entry, &prev_pos[NEW_MASTER_KEY]); + pg_tde_write_one_keydata(k_fd[NEW_MASTER_KEY], key_index[NEW_MASTER_KEY], enc_rel_key_data[NEW_MASTER_KEY]); /* Increment the key index for the new master key */ key_index[NEW_MASTER_KEY]++; } /* Close unrotated files */ - FileClose(m_file[OLD_MASTER_KEY]); - FileClose(k_file[OLD_MASTER_KEY]); + close(m_fd[OLD_MASTER_KEY]); + close(k_fd[OLD_MASTER_KEY]); /* Let's calculate sizes */ - map_size = FileSize(m_file[NEW_MASTER_KEY]); - keydata_size = FileSize(k_file[NEW_MASTER_KEY]); + map_size = lseek(m_fd[NEW_MASTER_KEY], 0, SEEK_END); + keydata_size = lseek(k_fd[NEW_MASTER_KEY], 0, SEEK_END); xlrec_size = map_size + keydata_size + SizeoOfXLogMasterKeyRotate; /* palloc and fill in the structure */ @@ -1178,12 +1206,13 @@ pg_tde_perform_rotate_key(TDEMasterKey *master_key, TDEMasterKey *new_master_key xlrec->map_size = map_size; xlrec->keydata_size = keydata_size; - FileRead(m_file[NEW_MASTER_KEY], xlrec->buff, xlrec->map_size, 0, WAIT_EVENT_DATA_FILE_READ); - FileRead(k_file[NEW_MASTER_KEY], &xlrec->buff[xlrec->map_size], xlrec->keydata_size, 0, WAIT_EVENT_DATA_FILE_READ); + /* TODO: pgstat_report_wait_start / pgstat_report_wait_end */ + pg_pread(m_fd[NEW_MASTER_KEY], xlrec->buff, xlrec->map_size, 0); + pg_pread(k_fd[NEW_MASTER_KEY], &xlrec->buff[xlrec->map_size], xlrec->keydata_size, 0); /* Close the files */ - FileClose(m_file[NEW_MASTER_KEY]); - FileClose(k_file[NEW_MASTER_KEY]); + close(m_fd[NEW_MASTER_KEY]); + close(k_fd[NEW_MASTER_KEY]); /* Insert the XLog record */ XLogBeginInsert(); @@ -1217,8 +1246,8 @@ pg_tde_write_map_keydata_files(off_t map_size, char *m_file_data, off_t keydata_ TDEFileHeader *fheader; char m_path_new[MAXPGPATH]; char k_path_new[MAXPGPATH]; - File m_file_new; - File k_file_new; + int m_fd_new; + int k_fd_new; bool is_new_file; off_t curr_pos = 0; off_t read_pos_tmp = 0; @@ -1226,6 +1255,7 @@ pg_tde_write_map_keydata_files(off_t map_size, char *m_file_data, off_t keydata_ LWLock *lock_cache = tde_lwlock_mk_cache(); char db_map_path[MAXPGPATH] = {0}; char db_keydata_path[MAXPGPATH] = {0}; + bool is_err = false; /* Let's get the header. Buff should start with the map file header. */ fheader = (TDEFileHeader *) m_file_data; @@ -1241,38 +1271,56 @@ pg_tde_write_map_keydata_files(off_t map_size, char *m_file_data, off_t keydata_ LWLockAcquire(lock_cache, LW_EXCLUSIVE); /* Initialize the new files and set the names */ - m_file_new = keyrotation_init_file(&fheader->master_key_info, m_path_new, db_map_path, &is_new_file, &curr_pos); - k_file_new = keyrotation_init_file(&fheader->master_key_info, k_path_new, db_keydata_path, &is_new_file, &read_pos_tmp); + m_fd_new = keyrotation_init_file(&fheader->master_key_info, m_path_new, db_map_path, &is_new_file, &curr_pos); + k_fd_new = keyrotation_init_file(&fheader->master_key_info, k_path_new, db_keydata_path, &is_new_file, &read_pos_tmp); - if (FileWrite(m_file_new, m_file_data, map_size, 0, WAIT_EVENT_DATA_FILE_WRITE) != map_size) + /* TODO: pgstat_report_wait_start / pgstat_report_wait_end */ + if (pg_pwrite(m_fd_new, m_file_data, map_size, 0) != map_size) { - LWLockRelease(lock_cache); - LWLockRelease(lock_files); - ereport(WARNING, (errcode_for_file_access(), - errmsg("Could not write tde file \"%s\": %m", + errmsg("could not write tde file \"%s\": %m", m_path_new))); + is_err = true; + goto FINALIZE; } - - if (FileWrite(k_file_new, k_file_data, keydata_size, 0, WAIT_EVENT_DATA_FILE_WRITE) != keydata_size) + if (pg_fsync(m_fd_new) != 0) { - LWLockRelease(lock_cache); - LWLockRelease(lock_files); + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", m_path_new))); + is_err = true; + goto FINALIZE; + } + + if (pg_pwrite(k_fd_new, k_file_data, keydata_size, 0) != keydata_size) + { ereport(WARNING, (errcode_for_file_access(), - errmsg("Could not write tde file \"%s\": %m", + errmsg("could not write tde file \"%s\": %m", k_path_new))); + is_err = true; + goto FINALIZE; + } + if (pg_fsync(k_fd_new) != 0) + { + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", k_path_new))); + is_err = true; + goto FINALIZE; } - FileClose(m_file_new); - FileClose(k_file_new); +FINALIZE: + close(m_fd_new); + close(k_fd_new); - finalize_key_rotation(db_map_path, db_keydata_path, m_path_new, k_path_new); + if (!is_err) + finalize_key_rotation(db_map_path, db_keydata_path, m_path_new, k_path_new); LWLockRelease(lock_cache); LWLockRelease(lock_files); - return true; + return !is_err; } diff --git a/src/access/pg_tde_xlog.c b/src/access/pg_tde_xlog.c index c3c661b6..c8a1b48f 100644 --- a/src/access/pg_tde_xlog.c +++ b/src/access/pg_tde_xlog.c @@ -40,22 +40,6 @@ static ssize_t TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, static void SetXLogPageIVPrefix(TimeLineID tli, XLogRecPtr lsn, char* iv_prefix); static int XLOGChooseNumBuffers(void); -typedef enum -{ - TDE_GCAT_KEY_XLOG, - - /* must be last */ - TDE_GCAT_KEYS_COUNT -} GlobalCatalogKeyTypes; - -/* TODO: move TDEXLogEncryptBuf here*/ -typedef struct EncryptionStateData -{ - GenericKeyring *keyring; - /* TODO: locking */ - TDEMasterKey master_keys[TDE_GCAT_KEYS_COUNT]; - -} EncryptionStateData; /* * TDE fork XLog diff --git a/src/catalog/tde_global_catalog.c b/src/catalog/tde_global_catalog.c index a6c69418..429c065e 100644 --- a/src/catalog/tde_global_catalog.c +++ b/src/catalog/tde_global_catalog.c @@ -24,21 +24,24 @@ #include #include +#define MASTER_KEY_DEFAULT_NAME "tde-global-catalog-key" + +/* TODO: not sure if we need an option of multiple master keys for the global catalog */ typedef enum { - TDE_GCAT_KEY_XLOG, + TDE_GCAT_XLOG_KEY, /* must be last */ TDE_GCAT_KEYS_COUNT -} GlobalCatalogKeyTypes; +} GlobalCatalogKeyTypes; typedef struct EncryptionStateData { GenericKeyring *keyring; - TDEMasterKey master_keys[TDE_GCAT_KEYS_COUNT]; -} EncryptionStateData; + TDEMasterKey master_keys[TDE_GCAT_KEYS_COUNT]; +} EncryptionStateData; -static EncryptionStateData *EncryptionState = NULL; +static EncryptionStateData * EncryptionState = NULL; /* GUC */ static char *KRingProviderType = NULL; @@ -46,9 +49,9 @@ static char *KRingProviderFilePath = NULL; static void init_gl_catalog_keys(void); static void init_keyring(void); -static TDEMasterKey *create_master_key(const char *key_name, - GenericKeyring *keyring, Oid dbOid, Oid spcOid, - bool ensure_new_key); +static TDEMasterKey * create_master_key(const char *key_name, + GenericKeyring * keyring, Oid dbOid, Oid spcOid, + bool ensure_new_key); void TDEGlCatInitGUC(void) @@ -81,7 +84,7 @@ TDEGlCatInitGUC(void) Size TDEGlCatEncStateSize(void) { - Size size; + Size size; size = sizeof(EncryptionStateData); size = add_size(size, sizeof(KeyringProviders)); @@ -92,12 +95,12 @@ TDEGlCatEncStateSize(void) void TDEGlCatShmemInit(void) { - bool foundBuf; - char *allocptr; + bool foundBuf; + char *allocptr; EncryptionState = (EncryptionStateData *) - ShmemInitStruct("TDE XLog Encryption State", - TDEGlCatEncStateSize(), &foundBuf); + ShmemInitStruct("TDE XLog Encryption State", + TDEGlCatEncStateSize(), &foundBuf); allocptr = ((char *) EncryptionState) + MAXALIGN(sizeof(EncryptionStateData)); EncryptionState->keyring = (GenericKeyring *) allocptr; @@ -108,24 +111,29 @@ TDEGlCatShmemInit(void) void TDEGlCatKeyInit(void) { - char db_map_path[MAXPGPATH] = {0}; + char db_map_path[MAXPGPATH] = {0}; init_keyring(); pg_tde_set_db_file_paths(&GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID), - db_map_path, NULL); + db_map_path, NULL); if (access(db_map_path, F_OK) == -1) { init_gl_catalog_keys(); } + else + { + /* put an internal key into the cache */ + GetGlCatInternalKey(XLOG_TDE_OID); + } } TDEMasterKey * TDEGetGlCatKeyFromCache(void) { TDEMasterKey *mkey; - - mkey = &EncryptionState->master_keys[TDE_GCAT_KEY_XLOG]; + + mkey = &EncryptionState->master_keys[TDE_GCAT_XLOG_KEY]; if (mkey->keyLength == 0) return NULL; @@ -133,17 +141,18 @@ TDEGetGlCatKeyFromCache(void) } void -TDEPutGlCatKeyInCache(TDEMasterKey *mkey) +TDEPutGlCatKeyInCache(TDEMasterKey * mkey) { - memcpy(EncryptionState->master_keys + TDE_GCAT_KEY_XLOG, mkey, sizeof(TDEMasterKey)); + memcpy(EncryptionState->master_keys + TDE_GCAT_XLOG_KEY, mkey, sizeof(TDEMasterKey)); } RelKeyData * GetGlCatInternalKey(Oid obj_id) { - return GetInternalKey(GLOBAL_SPACE_RLOCATOR(obj_id), EncryptionState->keyring); + return GetRelationKeyWithKeyring(GLOBAL_SPACE_RLOCATOR(obj_id), EncryptionState->keyring); } +/* TODO: add Vault */ static void init_keyring(void) { @@ -151,27 +160,27 @@ init_keyring(void) switch (EncryptionState->keyring->type) { case FILE_KEY_PROVIDER: - FileKeyring *kring = (FileKeyring *) EncryptionState->keyring; + FileKeyring * kring = (FileKeyring *) EncryptionState->keyring; strncpy(kring->file_name, KRingProviderFilePath, sizeof(kring->file_name)); break; } } -/* +/* * Keys are created during the cluster start only, so no locks needed here. */ static void init_gl_catalog_keys(void) { - InternalKey int_key; - RelKeyData *rel_key_data; - RelKeyData *enc_rel_key_data; - RelFileLocator *rlocator; - TDEMasterKey *mkey; + InternalKey int_key; + RelKeyData *rel_key_data; + RelKeyData *enc_rel_key_data; + RelFileLocator *rlocator; + TDEMasterKey *mkey; - mkey = create_master_key("global-catalog-master-key", - EncryptionState->keyring, - GLOBAL_DATA_TDE_OID, GLOBALTABLESPACE_OID, false); + mkey = create_master_key(MASTER_KEY_DEFAULT_NAME, + EncryptionState->keyring, + GLOBAL_DATA_TDE_OID, GLOBALTABLESPACE_OID, false); memset(&int_key, 0, sizeof(InternalKey)); @@ -180,8 +189,8 @@ init_gl_catalog_keys(void) { ereport(FATAL, (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("could not generate internal key for \"WAL\": %s", - ERR_error_string(ERR_get_error(), NULL)))); + errmsg("could not generate internal key for \"WAL\": %s", + ERR_error_string(ERR_get_error(), NULL)))); } rlocator = &GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID); @@ -193,11 +202,11 @@ init_gl_catalog_keys(void) } static TDEMasterKey * -create_master_key(const char *key_name, GenericKeyring *keyring, - Oid dbOid, Oid spcOid, bool ensure_new_key) +create_master_key(const char *key_name, GenericKeyring * keyring, + Oid dbOid, Oid spcOid, bool ensure_new_key) { - TDEMasterKey *masterKey; - keyInfo *keyInfo = NULL; + TDEMasterKey *masterKey; + keyInfo *keyInfo = NULL; masterKey = palloc(sizeof(TDEMasterKey)); masterKey->keyInfo.databaseId = dbOid; @@ -222,4 +231,4 @@ create_master_key(const char *key_name, GenericKeyring *keyring, memcpy(masterKey->keyData, keyInfo->data.data, keyInfo->data.len); return masterKey; -} \ No newline at end of file +} diff --git a/src/catalog/tde_master_key.c b/src/catalog/tde_master_key.c index 2b97ee05..dd4b7360 100644 --- a/src/catalog/tde_master_key.c +++ b/src/catalog/tde_master_key.c @@ -232,7 +232,11 @@ GetMasterKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring) recursion++; LWLockAcquire(lock_cache, LW_SHARED); - masterKey = get_master_key_from_cache(dbOid); + /* Global catalog has its own cache */ + if (spcOid == GLOBALTABLESPACE_OID) + masterKey = TDEGetGlCatKeyFromCache(); + else + masterKey = get_master_key_from_cache(dbOid); LWLockRelease(lock_cache); if (masterKey) @@ -273,7 +277,8 @@ GetMasterKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring) return NULL; } - if (keyring == NULL) { + if (keyring == NULL) + { keyring = GetKeyProviderByID(masterKeyInfo->keyringId); if (keyring == NULL) { @@ -300,7 +305,7 @@ GetMasterKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring) memcpy(masterKey->keyData, keyInfo->data.data, keyInfo->data.len); masterKey->keyLength = keyInfo->data.len; - Assert(MyDatabaseId == masterKey->keyInfo.databaseId); + Assert(dbOid == masterKey->keyInfo.databaseId); if (spcOid == GLOBALTABLESPACE_OID) TDEPutGlCatKeyInCache(masterKey); else diff --git a/src/include/access/pg_tde_tdemap.h b/src/include/access/pg_tde_tdemap.h index 471deb64..16e0bb83 100644 --- a/src/include/access/pg_tde_tdemap.h +++ b/src/include/access/pg_tde_tdemap.h @@ -53,7 +53,7 @@ extern void pg_tde_delete_key_map_entry(const RelFileLocator *rlocator); extern void pg_tde_free_key_map_entry(const RelFileLocator *rlocator, off_t offset); extern RelKeyData *GetRelationKey(RelFileLocator rel); -extern RelKeyData *GetInternalKey(RelFileLocator rel, GenericKeyring *keyring); +extern RelKeyData *GetRelationKeyWithKeyring(RelFileLocator rel, GenericKeyring *keyring); extern void pg_tde_delete_tde_files(Oid dbOid, Oid spcOid); diff --git a/src/include/catalog/tde_global_catalog.h b/src/include/catalog/tde_global_catalog.h index 1e32e408..5dd44b9e 100644 --- a/src/include/catalog/tde_global_catalog.h +++ b/src/include/catalog/tde_global_catalog.h @@ -23,7 +23,11 @@ #define GLOBAL_DATA_TDE_OID 607 /* Global objects fake "db" */ #define XLOG_TDE_OID 608 -#define GLOBAL_SPACE_RLOCATOR(_obj_oid) (RelFileLocator) {GLOBALTABLESPACE_OID, 0, _obj_oid} +#define GLOBAL_SPACE_RLOCATOR(_obj_oid) (RelFileLocator) { \ + GLOBALTABLESPACE_OID, \ + GLOBAL_DATA_TDE_OID, \ + _obj_oid \ +} extern void TDEGlCatInitGUC(void); extern Size TDEGlCatEncStateSize(void); diff --git a/src/keyring/keyring_file.c b/src/keyring/keyring_file.c index 812e9fab..f5d8648d 100644 --- a/src/keyring/keyring_file.c +++ b/src/keyring/keyring_file.c @@ -1,11 +1,11 @@ /*------------------------------------------------------------------------- * * keyring_file.c - * Implements the file provider keyring - * routines. + * Implements the file provider keyring + * routines. * * IDENTIFICATION - * contrib/pg_tde/src/keyring/keyring_file.c + * contrib/pg_tde/src/keyring/keyring_file.c * *------------------------------------------------------------------------- */ @@ -40,21 +40,21 @@ static keyInfo* get_key_by_name(GenericKeyring* keyring, const char* key_name, bool throw_error, KeyringReturnCodes *return_code) { keyInfo* key = NULL; - File file = -1; + int fd = -1; FileKeyring* file_keyring = (FileKeyring*)keyring; off_t bytes_read = 0; off_t curr_pos = 0; *return_code = KEYRING_CODE_SUCCESS; - file = PathNameOpenFile(file_keyring->file_name, PG_BINARY); - if (file < 0) + fd = BasicOpenFile(file_keyring->file_name, PG_BINARY); + if (fd < 0) return NULL; key = palloc(sizeof(keyInfo)); while(true) { - bytes_read = FileRead(file, key, sizeof(keyInfo), curr_pos, WAIT_EVENT_DATA_FILE_READ); + bytes_read = pg_pread(fd, key, sizeof(keyInfo), curr_pos); curr_pos += bytes_read; if (bytes_read == 0 ) @@ -62,13 +62,13 @@ get_key_by_name(GenericKeyring* keyring, const char* key_name, bool throw_error, /* * Empty keyring file is considered as a valid keyring file that has no keys */ - FileClose(file); + close(fd); pfree(key); return NULL; } if (bytes_read != sizeof(keyInfo)) { - FileClose(file); + close(fd); pfree(key); /* Corrupt file */ *return_code = KEYRING_CODE_DATA_CORRUPTED; @@ -81,21 +81,21 @@ get_key_by_name(GenericKeyring* keyring, const char* key_name, bool throw_error, } if (strncasecmp(key->name.name, key_name, sizeof(key->name.name)) == 0) { - FileClose(file); + close(fd); return key; } } - FileClose(file); + close(fd); pfree(key); - return NULL; + return NULL; } static KeyringReturnCodes set_key_by_name(GenericKeyring* keyring, keyInfo *key, bool throw_error) { - off_t bytes_written = 0; + off_t bytes_written = 0; off_t curr_pos = 0; - File file; + int fd; FileKeyring* file_keyring = (FileKeyring*)keyring; keyInfo *existing_key; KeyringReturnCodes return_code = KEYRING_CODE_SUCCESS; @@ -111,26 +111,35 @@ set_key_by_name(GenericKeyring* keyring, keyInfo *key, bool throw_error) return KEYRING_CODE_INVALID_OPERATION; } - file = PathNameOpenFile(file_keyring->file_name, O_CREAT | O_RDWR | PG_BINARY); - if (file < 0) - { + fd = BasicOpenFile(file_keyring->file_name, O_CREAT | O_RDWR | PG_BINARY); + if (fd < 0) + { ereport(throw_error?ERROR:WARNING, (errcode_for_file_access(), errmsg("Failed to open keyring file %s :%m", file_keyring->file_name))); - return KEYRING_CODE_RESOURCE_NOT_ACCESSABLE; - } + return KEYRING_CODE_RESOURCE_NOT_ACCESSABLE; + } /* Write key to the end of file */ - curr_pos = FileSize(file); - bytes_written = FileWrite(file, key, sizeof(keyInfo), curr_pos, WAIT_EVENT_DATA_FILE_WRITE); + curr_pos = lseek(fd, 0, SEEK_END); + bytes_written = pg_pwrite(fd, key, sizeof(keyInfo), curr_pos); if (bytes_written != sizeof(keyInfo)) - { - FileClose(file); - ereport(throw_error?ERROR:WARNING, - (errcode_for_file_access(), - errmsg("keyring file \"%s\" can't be written: %m", - file_keyring->file_name))); - return KEYRING_CODE_RESOURCE_NOT_ACCESSABLE; - } - FileClose(file); + { + close(fd); + ereport(throw_error?ERROR:WARNING, + (errcode_for_file_access(), + errmsg("keyring file \"%s\" can't be written: %m", + file_keyring->file_name))); + return KEYRING_CODE_RESOURCE_NOT_ACCESSABLE; + } + if (pg_fsync(fd) != 0) + { + close(fd); + ereport(throw_error?ERROR:WARNING, + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", + file_keyring->file_name))); + return KEYRING_CODE_RESOURCE_NOT_ACCESSABLE; + } + close(fd); return KEYRING_CODE_SUCCESS; } From b90bb3a02c48b128c46b2c0a1427a2ba99c31141 Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoy Date: Fri, 14 Jun 2024 17:45:29 +0300 Subject: [PATCH 4/8] Fixes and chores after the rebase with smgr --- src/access/pg_tde_tdemap.c | 40 ++++++++---------------------- src/catalog/tde_global_catalog.c | 1 + src/include/access/pg_tde_tdemap.h | 2 ++ src/smgr/pg_tde_smgr.c | 2 +- 4 files changed, 14 insertions(+), 31 deletions(-) diff --git a/src/access/pg_tde_tdemap.c b/src/access/pg_tde_tdemap.c index 9520e4c7..fcb1bd8e 100644 --- a/src/access/pg_tde_tdemap.c +++ b/src/access/pg_tde_tdemap.c @@ -79,8 +79,6 @@ typedef struct TDEMapFilePath char keydata_path[MAXPGPATH]; } TDEMapFilePath; -static void put_key_into_map(Oid rel_id, RelKeyData *key); - static int pg_tde_open_file_basic(char *tde_filename, int fileFlags, bool ignore_missing); static int pg_tde_file_header_write(char *tde_filename, int fd, TDEMasterKeyInfo *master_key_info, off_t *bytes_written); static int pg_tde_file_header_read(char *tde_filename, int fd, TDEFileHeader *fheader, bool *is_new_file, off_t *bytes_read); @@ -113,8 +111,7 @@ pg_tde_create_key_map_entry(const RelFileLocator *newrlocator) TDEMasterKey *master_key; XLogRelKey xlrec; - pg_tde_set_db_file_paths(newrlocator->dbOid); - master_key = GetMasterKey(newrlocator->dbOid); + master_key = GetMasterKey(newrlocator->dbOid, newrlocator->spcOid, NULL); if (master_key == NULL) { ereport(ERROR, @@ -168,23 +165,7 @@ RelKey *tde_rel_key_map = NULL; RelKeyData * GetRelationKey(RelFileLocator rel) { - RelKey *curr; - RelKeyData *key; - - Oid rel_id = rel.relNumber; - for (curr = tde_rel_key_map; curr != NULL; curr = curr->next) - { - if (curr->rel_id == rel_id) - { - return curr->key; - } - } - - key = pg_tde_get_key_from_file(&rel, NULL); - - put_key_into_map(rel.relNumber, key); - - return key; + return GetRelationKeyWithKeyring(rel, NULL); } RelKeyData * @@ -206,14 +187,14 @@ GetRelationKeyWithKeyring(RelFileLocator rel, GenericKeyring *keyring) if (key != NULL) { - put_key_into_map(rel.relNumber, key); + pg_tde_put_key_into_map(rel.relNumber, key); } return key; } -static void -put_key_into_map(Oid rel_id, RelKeyData *key) { +void +pg_tde_put_key_into_map(Oid rel_id, RelKeyData *key) { RelKey *new; RelKey *prev = NULL; @@ -256,7 +237,7 @@ tde_create_rel_key(Oid rel_id, InternalKey *key, TDEMasterKeyInfo *master_key_in rel_key_data->internal_key.ctx = NULL; /* Add to the decrypted key to cache */ - put_key_into_map(rel_id, rel_key_data); + pg_tde_put_key_into_map(rel_id, rel_key_data); return rel_key_data; } @@ -294,11 +275,11 @@ pg_tde_set_db_file_paths(const RelFileLocator *rlocator, char *map_path, char *k { char *db_path; - /* We use dbOid for the global space for key caches but for the backend - * it should be 0. + /* If this is a global space, than the call might be in a critial section + * (during XLog write) so we can't do GetDatabasePath as it calls palloc() */ if (rlocator->spcOid == GLOBALTABLESPACE_OID) - db_path = GetDatabasePath(0, rlocator->spcOid); + db_path = "global"; else db_path = GetDatabasePath(rlocator->dbOid, rlocator->spcOid); @@ -384,14 +365,13 @@ pg_tde_get_master_key(Oid dbOid, Oid spcOid) bool is_new_file = false; off_t bytes_read = 0; char db_map_path[MAXPGPATH] = {0}; - char db_keydata_path[MAXPGPATH] = {0}; /* Set the file paths */ pg_tde_set_db_file_paths(&(RelFileLocator) { spcOid, dbOid, 0}, - db_map_path, db_keydata_path); + db_map_path, NULL); /* * Ensuring that we always open the file in binary mode. The caller must diff --git a/src/catalog/tde_global_catalog.c b/src/catalog/tde_global_catalog.c index 429c065e..662974a3 100644 --- a/src/catalog/tde_global_catalog.c +++ b/src/catalog/tde_global_catalog.c @@ -198,6 +198,7 @@ init_gl_catalog_keys(void) enc_rel_key_data = tde_encrypt_rel_key(mkey, rel_key_data, rlocator); pg_tde_write_key_map_entry(rlocator, enc_rel_key_data, &mkey->keyInfo); + pg_tde_put_key_into_map(rlocator->relNumber, rel_key_data); TDEPutGlCatKeyInCache(mkey); } diff --git a/src/include/access/pg_tde_tdemap.h b/src/include/access/pg_tde_tdemap.h index 16e0bb83..e0e06b63 100644 --- a/src/include/access/pg_tde_tdemap.h +++ b/src/include/access/pg_tde_tdemap.h @@ -69,4 +69,6 @@ extern void pg_tde_set_db_file_paths(const RelFileLocator *rlocator, char *map_p const char * tde_sprint_key(InternalKey *k); +extern void pg_tde_put_key_into_map(Oid rel_id, RelKeyData *key); + #endif /*PG_TDE_MAP_H*/ diff --git a/src/smgr/pg_tde_smgr.c b/src/smgr/pg_tde_smgr.c index c6c2400e..7c4aa49f 100644 --- a/src/smgr/pg_tde_smgr.c +++ b/src/smgr/pg_tde_smgr.c @@ -35,7 +35,7 @@ tde_smgr_get_key(SMgrRelation reln) recursion++; - if(GetMasterKey(reln->smgr_rlocator.locator.relNumber)==NULL) + if(GetMasterKey(reln->smgr_rlocator.locator.relNumber, reln->smgr_rlocator.locator.spcOid, NULL)==NULL) { recursion--; return NULL; From 33e76da18826438e08b7ecc9a27fed5f90361200 Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoy Date: Mon, 17 Jun 2024 18:32:31 +0300 Subject: [PATCH 5/8] Add todo for key management --- src/catalog/tde_global_catalog.c | 7 ++++++- src/include/access/pg_tde_xlog.h | 4 ---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/catalog/tde_global_catalog.c b/src/catalog/tde_global_catalog.c index 662974a3..844d8c19 100644 --- a/src/catalog/tde_global_catalog.c +++ b/src/catalog/tde_global_catalog.c @@ -152,7 +152,12 @@ GetGlCatInternalKey(Oid obj_id) return GetRelationKeyWithKeyring(GLOBAL_SPACE_RLOCATOR(obj_id), EncryptionState->keyring); } -/* TODO: add Vault */ +/* + * TODO: should be aligned with the rest of the keyring_provider code after its + * refactoring + * + * TODO: add Vault + */ static void init_keyring(void) { diff --git a/src/include/access/pg_tde_xlog.h b/src/include/access/pg_tde_xlog.h index 740eda8d..17a1c65c 100644 --- a/src/include/access/pg_tde_xlog.h +++ b/src/include/access/pg_tde_xlog.h @@ -37,13 +37,9 @@ static const RmgrData pg_tde_rmgr = { .rm_identify = pg_tde_rmgr_identify }; -<<<<<<< HEAD #ifdef PERCONA_FORK /* XLog encryption staff */ -======= -/* XLog encryption stuff */ ->>>>>>> e9805ad (Create and use global catalog key) extern Size TDEXLogEncryptBuffSize(void); From 3e007b20acc7008a33d04b9873ac939d313d7964 Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoy Date: Tue, 18 Jun 2024 18:30:59 +0300 Subject: [PATCH 6/8] Fix builds for with upstream PG --- src/access/pg_tde_tdemap.c | 2 +- src/access/pg_tde_xlog.c | 6 +++--- src/catalog/tde_global_catalog.c | 3 +++ src/catalog/tde_master_key.c | 21 +++++++++++++++------ src/pg_tde.c | 7 ++++++- 5 files changed, 28 insertions(+), 11 deletions(-) diff --git a/src/access/pg_tde_tdemap.c b/src/access/pg_tde_tdemap.c index fcb1bd8e..71896037 100644 --- a/src/access/pg_tde_tdemap.c +++ b/src/access/pg_tde_tdemap.c @@ -417,7 +417,7 @@ pg_tde_open_file_basic(char *tde_filename, int fileFlags, bool ignore_missing) fd = BasicOpenFile(tde_filename, fileFlags | PG_BINARY); if (fd < 0 && !(errno == ENOENT && ignore_missing == true)) { - ereport(PANIC, + ereport(ERROR, (errcode_for_file_access(), errmsg("could not open tde file \"%s\": %m", tde_filename))); diff --git a/src/access/pg_tde_xlog.c b/src/access/pg_tde_xlog.c index c8a1b48f..246ef195 100644 --- a/src/access/pg_tde_xlog.c +++ b/src/access/pg_tde_xlog.c @@ -24,9 +24,9 @@ #include "access/pg_tde_tdemap.h" #include "access/pg_tde_xlog.h" -#include "catalog/tde_global_catalog.h" #include "encryption/enc_tde.h" - +#ifdef PERCONA_FORK +#include "catalog/tde_global_catalog.h" static char *TDEXLogEncryptBuf = NULL; @@ -39,7 +39,7 @@ static XLogPageHeaderData DecryptCurrentPageHrd; static ssize_t TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset); static void SetXLogPageIVPrefix(TimeLineID tli, XLogRecPtr lsn, char* iv_prefix); static int XLOGChooseNumBuffers(void); - +#endif /* * TDE fork XLog diff --git a/src/catalog/tde_global_catalog.c b/src/catalog/tde_global_catalog.c index 844d8c19..2c560699 100644 --- a/src/catalog/tde_global_catalog.c +++ b/src/catalog/tde_global_catalog.c @@ -12,6 +12,8 @@ #include "postgres.h" +#ifdef PERCONA_FORK + #include "storage/shmem.h" #include "utils/guc.h" @@ -238,3 +240,4 @@ create_master_key(const char *key_name, GenericKeyring * keyring, return masterKey; } +#endif /* PERCONA_FORK */ diff --git a/src/catalog/tde_master_key.c b/src/catalog/tde_master_key.c index dd4b7360..4d507011 100644 --- a/src/catalog/tde_master_key.c +++ b/src/catalog/tde_master_key.c @@ -29,7 +29,9 @@ #include #include "access/pg_tde_tdemap.h" +#ifdef PERCONA_FORK #include "catalog/tde_global_catalog.h" +#endif typedef struct TdeMasterKeySharedState { @@ -232,10 +234,12 @@ GetMasterKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring) recursion++; LWLockAcquire(lock_cache, LW_SHARED); - /* Global catalog has its own cache */ +#ifdef PERCONA_FORK + /* Global catalog has its own cache */ if (spcOid == GLOBALTABLESPACE_OID) masterKey = TDEGetGlCatKeyFromCache(); - else + else +#endif masterKey = get_master_key_from_cache(dbOid); LWLockRelease(lock_cache); @@ -252,10 +256,12 @@ GetMasterKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring) LWLockAcquire(lock_files, LW_SHARED); LWLockAcquire(lock_cache, LW_EXCLUSIVE); +#ifdef PERCONA_FORK /* Global catalog has its own cache */ if (spcOid == GLOBALTABLESPACE_OID) masterKey = TDEGetGlCatKeyFromCache(); - else + else +#endif masterKey = get_master_key_from_cache(dbOid); if (masterKey) @@ -285,8 +291,9 @@ GetMasterKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring) LWLockRelease(lock_cache); LWLockRelease(lock_files); - recursion--; - return NULL; + recursion--; + return NULL; + } } keyInfo = KeyringGetKey(keyring, masterKeyInfo->keyId.versioned_name, false, &keyring_ret); @@ -306,9 +313,11 @@ GetMasterKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring) masterKey->keyLength = keyInfo->data.len; Assert(dbOid == masterKey->keyInfo.databaseId); +#ifdef PERCONA_FORK if (spcOid == GLOBALTABLESPACE_OID) TDEPutGlCatKeyInCache(masterKey); - else + else +#endif push_master_key_to_cache(masterKey); /* Release the exclusive locks here */ diff --git a/src/pg_tde.c b/src/pg_tde.c index bac49ea6..2298fe67 100644 --- a/src/pg_tde.c +++ b/src/pg_tde.c @@ -32,7 +32,9 @@ #include "utils/builtins.h" #include "pg_tde_defs.h" #include "smgr/pg_tde_smgr.h" +#ifdef PERCONA_FORK #include "catalog/tde_global_catalog.h" +#endif #define MAX_ON_INSTALLS 5 @@ -82,11 +84,13 @@ tde_shmem_startup(void) TdeShmemInit(); AesInit(); +#ifdef PERCONA_FORK TDEGlCatShmemInit(); TDEGlCatKeyInit(); TDEXLogShmemInit(); TDEXLogSmgrInit(); +#endif } void @@ -99,9 +103,10 @@ _PG_init(void) keyringRegisterVariables(); InitializeMasterKeyInfo(); +#ifdef PERCONA_FORK XLogInitGUC(); TDEGlCatInitGUC(); - +#endif prev_shmem_request_hook = shmem_request_hook; shmem_request_hook = tde_shmem_request; prev_shmem_startup_hook = shmem_startup_hook; From 9302137c128bb378c1971089f9a7cf4c0cef31f9 Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoy Date: Wed, 19 Jun 2024 17:41:01 +0300 Subject: [PATCH 7/8] Fix key rotation --- src/access/pg_tde_tdemap.c | 1 - src/catalog/tde_master_key.c | 20 +++++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/access/pg_tde_tdemap.c b/src/access/pg_tde_tdemap.c index 71896037..5b10f24d 100644 --- a/src/access/pg_tde_tdemap.c +++ b/src/access/pg_tde_tdemap.c @@ -1209,7 +1209,6 @@ pg_tde_perform_rotate_key(TDEMasterKey *master_key, TDEMasterKey *new_master_key /* Free up the palloc'ed data */ pfree(xlrec); - /* TODO: Remove the existing ones from cache etc. */ return true; #undef OLD_MASTER_KEY diff --git a/src/catalog/tde_master_key.c b/src/catalog/tde_master_key.c index 4d507011..8d1fd707 100644 --- a/src/catalog/tde_master_key.c +++ b/src/catalog/tde_master_key.c @@ -68,7 +68,7 @@ static Size required_shared_mem_size(void); static int required_locks_count(void); static void shared_memory_shutdown(int code, Datum arg); static void master_key_startup_cleanup(int tde_tbl_count, void *arg); -static void clear_master_key_cache(Oid databaseId, Oid tablespaceId) ; +static void clear_master_key_cache(Oid databaseId) ; static inline dshash_table *get_master_key_Hash(void); static TDEMasterKey *get_master_key_from_cache(Oid dbOid); static void push_master_key_to_cache(TDEMasterKey *masterKey); @@ -439,6 +439,7 @@ RotateMasterKey(const char *new_key_name, const char *new_provider_name, bool en TDEMasterKey new_master_key; const keyInfo *keyInfo = NULL; GenericKeyring *keyring; + bool is_rotated; /* * Let's set everything the same as the older master key and @@ -477,8 +478,13 @@ RotateMasterKey(const char *new_key_name, const char *new_provider_name, bool en new_master_key.keyLength = keyInfo->data.len; memcpy(new_master_key.keyData, keyInfo->data.data, keyInfo->data.len); - clear_master_key_cache(MyDatabaseId, MyDatabaseTableSpace); - return pg_tde_perform_rotate_key(master_key, &new_master_key); + is_rotated = pg_tde_perform_rotate_key(master_key, &new_master_key); + if (is_rotated) { + clear_master_key_cache(master_key->keyInfo.databaseId); + push_master_key_to_cache(&new_master_key); + } + + return is_rotated; } /* @@ -490,7 +496,7 @@ xl_tde_perform_rotate_key(XLogMasterKeyRotate *xlrec) bool ret; ret = pg_tde_write_map_keydata_files(xlrec->map_size, xlrec->buff, xlrec->keydata_size, &xlrec->buff[xlrec->map_size]); - clear_master_key_cache(MyDatabaseId, MyDatabaseTableSpace); + clear_master_key_cache(MyDatabaseId); return ret; } @@ -640,7 +646,7 @@ static void push_master_key_to_cache(TDEMasterKey *masterKey) { TDEMasterKey *cacheEntry = NULL; - Oid databaseId = MyDatabaseId; + Oid databaseId = masterKey->keyInfo.databaseId; bool found = false; cacheEntry = dshash_find_or_insert(get_master_key_Hash(), &databaseId, &found); @@ -684,7 +690,7 @@ master_key_startup_cleanup(int tde_tbl_count, void* arg) void cleanup_master_key_info(Oid databaseId, Oid tablespaceId) { - clear_master_key_cache(databaseId, tablespaceId); + clear_master_key_cache(databaseId); /* * TODO: Although should never happen. Still verify if any table in the * database is using tde @@ -695,7 +701,7 @@ cleanup_master_key_info(Oid databaseId, Oid tablespaceId) } static void -clear_master_key_cache(Oid databaseId, Oid tablespaceId) +clear_master_key_cache(Oid databaseId) { TDEMasterKey *cache_entry; From d4181b6b75b2e1fd3f90c452eac33b9db20e7050 Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoy Date: Fri, 21 Jun 2024 16:56:35 +0300 Subject: [PATCH 8/8] Add comment on InternalKey caching --- src/catalog/tde_global_catalog.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/catalog/tde_global_catalog.c b/src/catalog/tde_global_catalog.c index 2c560699..4fb72908 100644 --- a/src/catalog/tde_global_catalog.c +++ b/src/catalog/tde_global_catalog.c @@ -205,6 +205,11 @@ init_gl_catalog_keys(void) enc_rel_key_data = tde_encrypt_rel_key(mkey, rel_key_data, rlocator); pg_tde_write_key_map_entry(rlocator, enc_rel_key_data, &mkey->keyInfo); + /* + * TODO: move global catalog internal keys into own cache. This cache should + * be in the TopMemmoryContext because of SSL contexts + * (see https://github.com/Percona-Lab/pg_tde/pull/214#discussion_r1648998317) + */ pg_tde_put_key_into_map(rlocator->relNumber, rel_key_data); TDEPutGlCatKeyInCache(mkey); }