Opengauss database source code analysis series articles -- detailed explanation of dense equivalent query technology (Part 2)

Last issue We introduce the secret equivalent query technology and how to create the client key CMK, This installment continues to explain how to create a column encryption key CEK And creating encrypted tables .

Catalog

Create column encryption key CEK

Create encryption table

---

The client master key creation syntax is essentially to CMK The meta information of is parsed and saved in CreateClientLogicGlobal In the structure . among global_key_name Is the key name ,global_setting_params It's a List structure , Each node is a ClientLogicGlobalParam structure , The key information is saved in the form of key value . The client first passes the parser “fe_raw_parser()” It can be interpreted as CreateClientLogicGlobal Structure , Verify its parameters and send query statements to the server ; The server resolves to CreateClientLogicGlobal Structure and check the user namespace Such as permissions ,CMK Meta information is saved in the system table . establish CMK The overall process of is shown in the figure .

Create column encryption key CEK

  With the master key CMK, You can create... Based on this CEK, The following will be true of CREATE COLUMN ENCRYPTION KEY The syntax structure definitions involved in the statement are introduced one by one .
CREATE COLUMN ENCRYPTION KEY Syntax related data structures ：

/*   Save syntax information for creating column encryption keys   */
typedef struct CreateClientLogicColumn {
    NodeTag type;
    List *column_key_name;        /*   Column encryption key name   */
    List *column_setting_params; /*   Column encryption key parameters   */
} CreateClientLogicColumn;

/*   Save column encryption key parameters , Save in CreateClientLogicColumn Of column_setting_params in   */
typedef struct ClientLogicColumnParam {
    NodeTag type;
    ClientLogicColumnProperty key;
    char *value;
    unsigned int len;
    List *qualname;
    int location;
} ClientLogicColumnParam;

/*   Save column encryption key parameters key Enumerated type of   */
typedef enum class ClientLogicColumnProperty {
    CLIENT_GLOBAL_SETTING,    /*   encryption CEK Of CMK  */
    CEK_ALGORITHM,             /*   An algorithm for encrypting user data   */
    CEK_EXPECTED_VALUE,       /*  CEK Key plaintext , Optional parameters   */
    COLUMN_COLUMN_FUNCTION,  /*   The default is encryption  */
} ClientLogicColumnProperty;

CREATE COLUMN ENCRYPTION KEY cek_1 WITH VALUES (CLIENT_MASTER_KEY = cmk_1, ALGORITHM = AEAD_AES_256_CBC_HMAC_SHA256);

The parameter description of the above command is ：
（1） CLIENT_MASTER_KEY： Specified for encryption CEK Of CMK object .
（2） ALGORITHM：CEK Used to encrypt user data , This parameter specifies the algorithm for encrypting user data , Designated CEK Key type of .
（3） ENCRYPTED_VALUE： Column the plaintext of the encryption key , Default random generation , It can also be specified by the user , When specified by the user, the key length range is 28～256 position .
The column encryption key creation syntax is to parse the parameters into... Through the front-end parser CreateClientLogicColumn After structure , Specify for encryption by checking CEK Of CMK Load after object exists CMK cache , And then through “HooksManager::ColumnSettings::pre_create” Statement to call the cryptographic function “EncryptionColumnHookExecutor::pre_create” To verify the parameters and generate or encrypt ENCRYPTED_VALUE value , Last in “EncryptionPreProcess::set_new_query” Function ENCRYPTED_VALUE Parameter is CEK Ciphertext , restructure SQL Query statement . After the reconstruction SQL After the statement is sent to the server, the server resolves to CreateClientLogicColumn Structure and check the user namespace Such as permissions , take CEK The information of is saved in the system table . establish CEK The overall process of is shown in the figure 9-40 Shown , The organizational structure is shown in the figure 9-41 Shown .

  chart 9-40   Column encryption key CEK Create a process

chart 9-41   Client master key CMK Organizational structure of  

In the face of CEK Parameters are parsed , Use CMK Yes ENCRYPTED_VALUE Parameters are encrypted , After encryption, use the encrypted ENCRYPTED_VALUE Parameter and other parameter pairs create CEK Refactoring the syntax of . The main function entry codes for converting the input query statements into encrypted query statements are as follows ：

void EncryptionPreProcess::set_new_query(char **query, size_t query_size, StringArgs string_args, int location,
    int encrypted_value_location, size_t encrypted_value_size, size_t quote_num)
{
for (size_t i = 0; i < string_args.Size(); i++) {
    /*   from string_args Read the key value in and store it in the variable   */
        char string_to_add[MAX_KEY_ADD_LEN];
        errno_t rc = memset_s(string_to_add, MAX_KEY_ADD_LEN, 0, MAX_KEY_ADD_LEN);
        securec_check_c(rc, "\0", "\0");
        size_t total_in = 0;
        if (string_args.at(i) == NULL) {
            continue;
        }
        const char *key = string_args.at(i)->key;
        const char *value = string_args.at(i)->value;
        const size_t vallen = string_args.at(i)->valsize;
        if (!key || !value) {
            Assert(false);
            continue;
        }
        Assert(vallen < MAX_KEY_ADD_LEN);
        /*   take key and value Constructed as encrypted_value = ' Ciphertext value ' In the form of   */
        check_strncat_s(strncat_s(string_to_add, MAX_KEY_ADD_LEN, key, strlen(key)));
        total_in += strlen(key);
        check_strncat_s(strncat_s(string_to_add, MAX_KEY_ADD_LEN, "=\'", strlen("=\'")));
        total_in += strlen("=\'");
        check_strncat_s(strncat_s(string_to_add, MAX_KEY_ADD_LEN, value, vallen));
        total_in += vallen;
        check_strncat_s(strncat_s(string_to_add, MAX_KEY_ADD_LEN, "\'", strlen("\'")));
        total_in += strlen("\'");
        Assert(total_in < MAX_KEY_ADD_LEN);
        /*  encrypted_value_location Not empty , It means that the user provides EXPECTED_VALUE, Replace the plaintext value with the ciphertext value    */
        if (encrypted_value_location && encrypted_value_size) {
            *query = (char *)libpq_realloc(*query, query_size, query_size + vallen + 1);
            if (*query == NULL) {
                return;
            }
            check_memset_s(memset_s(*query + query_size, vallen + 1, 0, vallen + 1));
            char *replace_dest = *query + encrypted_value_location + strlen("\'");
            char *move_src =
                *query + encrypted_value_location + encrypted_value_size + quote_num + strlen("\'");
            char *move_dest = *query + encrypted_value_location + vallen + strlen("\'");
            check_memmove_s(memmove_s(move_dest,
                query_size - encrypted_value_location - encrypted_value_size - strlen("\'") + 1,
                move_src,
                query_size - encrypted_value_location - encrypted_value_size - strlen("\'")));
            query_size = query_size + vallen - encrypted_value_size;
            check_memcpy_s(memcpy_s(replace_dest, query_size - encrypted_value_location, value, vallen));
        } else { 
/*  EXPECTED_VALUE It's randomly generated , Insert directly into the original statement   */
            check_strcat_s(strcat_s(string_to_add, MAX_KEY_ADD_LEN, ","));
            size_t string_to_add_size = strlen(string_to_add);
            *query = (char *)libpq_realloc(*query, query_size, query_size + string_to_add_size + 1);
            if (*query == NULL) {
                return;
            }
            check_memmove_s(memmove_s(*query + location + string_to_add_size, query_size - location, *query + location,
                query_size - location));
            query_size += string_to_add_size;
            check_memcpy_s(memcpy_s(*query + location, query_size - location, string_to_add, string_to_add_size));
        }
        query[0][query_size] = '\0';
    }
    return;
}

Create encryption table

Next, create the encryption table .

CREATE TABLE creditcard_info (id_number int, 
name text encrypted with (column_encryption_key = cek_1, encryption_type = DETERMINISTIC), gender varchar(10) encrypted with (column_encryption_key = cek_1, encryption_type = DETERMINISTIC), 
salary float4 encrypted with (column_encryption_key = cek_1, encryption_type = DETERMINISTIC),
credit_card varchar(19) encrypted with (column_encryption_key = cek_1, encryption_type = DETERMINISTIC));

To create an encrypted table SQL The statement enters after parsing CreateStmt Function processing logic , stay run_pre_create_statement Function , Yes CreateStmt->tableElts Each of them ListCell Judge , There are still some constraints on the current encrypted table , The code of encryption table column definition and constraint processing function segment is as follows ：

bool createStmtProcessor::run_pre_create_statement(const CreateStmt * const stmt, StatementData *statement_data)
{
    …
  /*   Encryption table column definition and constraint processing   */
    foreach (elements, stmt->tableElts) {
        Node *element = (Node *)lfirst(elements);
        switch (nodeTag(element)) {
            case T_ColumnDef: {
                …
               /*   check distribute by Compliance with specifications   */
                if (column->colname != NULL &&
                    !check_distributeby(stmt->distributeby, column->colname)) {
                    return false;
                }
               /*   Column definition processing , Storage encryption type , Encryption key and other information   */
                if (!process_column_defintion(column, element, &expr_vec, &cached_columns, 
                    &cached_columns_for_defaults, statement_data)) {
                    return false;
                }
                break;
            }
            /*   Handle check, unique  Or other constraints   */
            case T_Constraint: {
                Constraint *constraint = (Constraint*)element;
                if (constraint->keys != NULL) {
                    ListCell *ixcell = NULL;
                    foreach (ixcell, constraint->keys) {
                        char *ikname = strVal(lfirst(ixcell));
                        for (size_t i = 0; i < cached_columns.size(); i++) {
                            if (strcmp((cached_columns.at(i))->get_col_name(), ikname) == 0 && !check_constraint(
                                constraint, cached_columns.at(i)->get_data_type(), ikname, &cached_columns)) {
                                return false;
                            }
                        }
                    }
                } else if (constraint->raw_expr != NULL) {
                    if (!transform_expr(constraint->raw_expr, "", &cached_columns)) {
                        return false;
                    }
                }
                break;
            }
            default:
                break;
        }
    }
     …
    /*   The plaintext data that needs to be encrypted in the encryption constraint   */
    if (!RawValues::get_raw_values_from_consts_vec(&expr_vec, statement_data, 0, &raw_values_list)) {
        return false;
    }
    return ValuesProcessor::process_values(statement_data, &cached_columns_for_defaults, 1,
        &raw_values_list);
}

After sending the query statement for creating the encrypted table to the server , The server creates successfully and returns a message of successful execution . The data encryption driver can transparently encrypt data before it is sent to the database , Data exists in the form of ciphertext during the processing of the whole statement , When returning results , Decrypt the returned data set , So as to ensure that the whole process is transparent to users 、 Imperceptible .
After defining the complete encryption table , The user can insert data into the table in the normal way . For the complete encryption process, see encrypt_data function , The core logic code is as follows ：

int encrypt_data(const unsigned char *plain_text, int plain_text_length, const AeadAesHamcEncKey &column_encryption_key,
    EncryptionType encryption_type, unsigned char *result, ColumnEncryptionAlgorithm column_encryption_algorithm)
{
    ……
    /*   obtain 16 Bit iv value   */
    unsigned char _iv [g_key_size + 1] = {0};
unsigned char iv_truncated[g_iv_size + 1] = {0};
/*   Deterministic encryption , Through hmac_sha256 Generate iv  */
    if (encryption_type == EncryptionType::DETERMINISTIC_TYPE) {
           hmac_sha256(column_encryption_key.get_iv_key(), g_auth_tag_size, plain_text, plain_text_length, _iv);
       ……
} else {
/*   Random encryption , Random generation iv  */
        if (encryption_type != EncryptionType::RANDOMIZED_TYPE) {
            return 0; 
        }
        int res = RAND_priv_bytes(iv_truncated, g_block_size);
        if (res != 1) {
            return 0;
        }
    }
    int cipherStart = g_algo_version_size + g_auth_tag_size + g_iv_size;
    /*   call encrypt Computational ciphertext   */
int cipherTextSize = encrypt(plain_text, plain_text_length, column_encryption_key.get_encyption_key(), iv_truncated,
        result + cipherStart, column_encryption_algorithm);
   ……
    int ivStartIndex = g_auth_tag_size + g_algo_version_size;
    res = memcpy_s(result + ivStartIndex, g_iv_size, iv_truncated, g_iv_size);
    securec_check_c(res, "\0", "\0");
    /*   Calculation  HMAC  */
    int hmacDataSize = g_algo_version_size + g_iv_size + cipherTextSize;
    hmac_sha256(column_encryption_key.get_mac_key(), g_auth_tag_size,
                result + g_auth_tag_size, hmacDataSize, result); 
    return (g_auth_tag_size + hmacDataSize);
}

openGauss The dense database is used for equivalent query , The whole query process is insensitive to users , Although the data stored in the database is in the form of ciphertext , However, the ciphertext data will be decrypted when the data is displayed to the user . Take the equivalent query statement from the encrypted table as an example , The whole statement processing process is shown in Figure 9-42 Shown . Client resolution SELECT Column attribute information in query statement , If the cache already exists, extract the column attribute information from the cache ; If not found in the cache , You need to query this information from the server , And cache . Column encryption key CEK It is stored in the server in the form of ciphertext , The client needs to decrypt CEK. Then use it to encrypt SELECT Condition parameter in query statement . Encrypted SELECT After the query statement is sent to the database server for execution , Returns the encrypted query result set . The client encrypts the key with the decrypted column CEK Decrypt SELECT Query result set , And return the decrypted plaintext result set to the application end .

  chart 9-42  SELECT Statement sequence diagram

Equivalent query processing run_pre_insert_statement function , The core logic code is as follows ：

bool Processor::run_pre_select_statement(const SelectStmt * const select_stmt, const SetOperation &parent_set_operation,
    const bool &parent_all, StatementData *statement_data, ICachedColumns *cached_columns, ICachedColumns *cached_columns_parents)
{
bool select_res = false;
/*   Handle SELECT Statement   */
    if (select_stmt->op != SETOP_NONE) {
        select_res = process_select_set_operation(select_stmt, statement_data, cached_columns);
        RETURN_IF(!select_res);
    }
    /*   Handle WHERE Clause   */
    ExprPartsList where_expr_parts_list;
    select_res = exprProcessor::expand_expr(select_stmt->whereClause, statement_data, &where_expr_parts_list);
    RETURN_IF(!select_res);
……
    /*   from FROM Clause to get the cached encrypted column   */
CachedColumns cached_columns_from(false, true);
    select_res = run_pre_from_list_statement(select_stmt->fromClause, statement_data, &cached_columns_from,
        cached_columns_parents);
……
/*   Put the encrypted column of the query in cached_columns In structure   */
    for (size_t i = 0; i < cached_columns_from.size(); i++) {
        if (find_in_name_map(target_list, cached_columns_from.at(i)->get_col_name())) {
            CachedColumn *target = new (std::nothrow) CachedColumn(cached_columns_from.at(i));
            if (target == NULL) {
                fprintf(stderr, "failed to new CachedColumn object\n");
                return false;
            }
            cached_columns->push(target);
        }
    }
    if (cached_columns_from.is_empty()) {
        return true; 
    }
    /*   Encrypted columns do not support ORDER BY（ Sort ） operation   */
    if (!deal_order_by_statement(select_stmt, cached_columns)) {
        return false;
    }
   
/*   take WHERE Clause to encrypt the value   */
    if (!WhereClauseProcessor::process(&cached_columns_from, &where_expr_parts_list, statement_data)) {
        return false;
}
……
    return true;
}

The complete client ciphertext decryption function code is as follows ： 

int decrypt_data(const unsigned char *cipher_text, int cipher_text_length,
    const AeadAesHamcEncKey &column_encryption_key, unsigned char *decryptedtext,
    ColumnEncryptionAlgorithm column_encryption_algorithm)
{
    if (cipher_text == NULL || cipher_text_length <= 0 || decryptedtext == NULL) {
        return 0;
    }
    /*   Verify ciphertext length   */
    if (cipher_text_length < min_ciph_len_in_bytes_with_authen_tag) {
        printf("ERROR(CLIENT): The length of cipher_text is invalid, cannot decrypt.\n");
        return 0;
    }
    /*   Verify the version number in the ciphertext   */
    if (cipher_text[g_auth_tag_size] != '1') {
        printf("ERROR(CLIENT): Version byte of cipher_text is invalid, cannot decrypt.\n");
        return 0;
    }
    ……
    /*   Calculation MAC label   */
    unsigned char authenticationTag [g_auth_tag_size] = {0};
    int HMAC_length = cipher_text_length - g_auth_tag_size;
    int res = hmac_sha256(column_encryption_key.get_mac_key(), g_auth_tag_size,
        cipher_text + g_auth_tag_size, HMAC_length, authenticationTag);
    if (res != 1) {
        printf("ERROR(CLIENT): Fail to compute a keyed hash of a given text.\n");
        return 0;
    }
    /*   Verify whether the ciphertext has been tampered with   */
    int cmp_result = my_memcmp(authenticationTag, cipher_text, g_auth_tag_size);
    /*   Decrypt data   */
    int decryptedtext_len = decrypt(cipher_text + cipher_start_index, cipher_value_length,
        column_encryption_key.get_encyption_key(), iv, decryptedtext, column_encryption_algorithm);
    if (decryptedtext_len < 0) {
        return 0;
    }
    decryptedtext[decryptedtext_len] = '\0';
    return decryptedtext_len;
}