diff --git a/include/bit_array.h b/include/bit_array.h index 1393f2e..72c6ad7 100755 --- a/include/bit_array.h +++ b/include/bit_array.h @@ -36,11 +36,14 @@ * @param data pointer to array of BIT_ARRAY_TYPE numbers * @param length length of array in BIT_ARRAY_TYPE * @param error_code initialization or runtime error code + * @param _length_in_types internal length of *data array measured in BIT_ARRAY_TYPEs */ typedef struct { BIT_ARRAY_TYPE *data; uint32_t length; uint8_t error_code; + + uint32_t _length_in_types; } bit_array_s; bit_array_s *bit_array_init(uint32_t size_bits); @@ -51,6 +54,8 @@ void bit_array_clear_bit(bit_array_s *bit_array, uint32_t index); bool bit_array_get_bit(bit_array_s *bit_array, uint32_t index); +void bit_array_not(bit_array_s *bit_array); + void bit_array_clear(bit_array_s *bit_array); void bit_array_destroy(bit_array_s *bit_array); diff --git a/include/petal.h b/include/petal.h index 67d79cb..9b7fd96 100755 --- a/include/petal.h +++ b/include/petal.h @@ -60,6 +60,18 @@ typedef struct { uint32_t length; } petal_shape_s; +/** + * @struct petal_params_s + * Stores simple (non-pointers) optional parameters for dropout and normalization + * + * @param dropout ratio of dropped outputs (0 to 1) + * @param center center of normalization for PETAL_TYPE_NORMALIZE_... + * @param deviation deviation of normalization for PETAL_TYPE_NORMALIZE_... + */ +typedef struct { + float dropout, center, deviation; +} petal_params_s; + /** * @struct petal_s * Stores petal's data @@ -69,7 +81,7 @@ typedef struct { * @param weights - pointers to weights_s structs * @param bias_weights - pointers to weights_s structs * @param activation - pointer to activation_s struct - * @param dropout - ratio of dropped outputs (0 to 1) + * @param params - petal_params_s struct (for dropout / normalization) * @param bit_array_s - pointer to bit_array_s struct that stores indices to drop * @param output - petal outputs * @param error_on_input - petal input state during backpropagation @@ -81,7 +93,7 @@ typedef struct { petal_shape_s *input_shape, *output_shape; weights_s *weights, *bias_weights; activation_s *activation; - float dropout, center, deviation; + petal_params_s params; bit_array_s *bit_array; float *output, *error_on_input; @@ -89,8 +101,8 @@ typedef struct { } petal_s; petal_s *petal_init(uint8_t petal_type, bool first, petal_shape_s *input_shape, petal_shape_s *output_shape, - weights_s *weights, weights_s *bias_weights, activation_s *activation, float dropout, float center, - float deviation); + weights_s *weights, weights_s *bias_weights, activation_s *activation, + petal_params_s *petal_params); void petal_forward(petal_s *petal, float *input, bool training); diff --git a/src/bit_array.c b/src/bit_array.c index c594666..23e52ee 100755 --- a/src/bit_array.c +++ b/src/bit_array.c @@ -48,9 +48,11 @@ bit_array_s *bit_array_init(uint32_t size_bits) { // Reset error bit_array->error_code = ERROR_NONE; - // Calculate the number of BIT_ARRAY_TYPE needed to represent size_bits and initialize array with zeros - bit_array->data = - (BIT_ARRAY_TYPE *) calloc((size_bits + (BIT_ARRAY_BITS - 1U)) / BIT_ARRAY_BITS, sizeof(BIT_ARRAY_TYPE)); + // Calculate the number of BIT_ARRAY_TYPE needed to represent size_bits + bit_array->_length_in_types = (size_bits + (BIT_ARRAY_BITS - 1U)) / BIT_ARRAY_BITS; + + // Initialize array with zeros + bit_array->data = (BIT_ARRAY_TYPE *) calloc(bit_array->_length_in_types, sizeof(BIT_ARRAY_TYPE)); if (!bit_array->data) { logger(LOG_E, "bit_array_init", "Error allocating memory for bit_array->data"); bit_array->error_code = ERROR_MALLOC; @@ -121,6 +123,17 @@ bool bit_array_get_bit(bit_array_s *bit_array, uint32_t index) { (BIT_ARRAY_TYPE) 1) != (BIT_ARRAY_TYPE) 0; } +/** + * @brief Inverts bit array (inverts each bit in it) + * + * @param bit_array pointer to bit_array_s struct + */ +void bit_array_not(bit_array_s *bit_array) { + for (uint32_t i = 0; i < bit_array->_length_in_types; ++i) { + bit_array->data[i] = ~bit_array->data[i]; + } +} + /** * @brief Clears bit array entirely (sets all bits to 0) * diff --git a/src/dropout.c b/src/dropout.c index b271267..6f44ba7 100755 --- a/src/dropout.c +++ b/src/dropout.c @@ -36,28 +36,49 @@ * @param dropout_ratio 0 to 1 */ void dropout_generate_indices(bit_array_s *bit_array, float dropout_ratio) { - // Calculate how many indices we need to drop - uint32_t indices_n_to_drop = (float) bit_array->length * dropout_ratio; + uint32_t indices_n_to_drop_or_keep = 0U; + + // Calculate how many indices we need to drop for [0.0, 0.5] interval + if (dropout_ratio >= 0.f && dropout_ratio <= .5f) + indices_n_to_drop_or_keep = (float) bit_array->length * dropout_ratio; + + // Calculate how many indices we need to keep for [0.5, 1.0] interval (because in this case it's easer to keep) + else if (dropout_ratio <= 1.f && dropout_ratio >= .5f) + indices_n_to_drop_or_keep = bit_array->length - (uint32_t) ((float) bit_array->length * dropout_ratio); // Handle out-of-bounds access - if (indices_n_to_drop > bit_array->length) { - logger(LOG_E, "dropout_generate_indices", "Cannot drop %u indices. Out of bounds for bit array with size %u", - indices_n_to_drop, bit_array->length); + if (indices_n_to_drop_or_keep > bit_array->length) { + logger(LOG_E, "dropout_generate_indices", + "Cannot drop or keep %u indices. Out of bounds for bit array with size %u", indices_n_to_drop_or_keep, + bit_array->length); bit_array->error_code = ERROR_BITMAP_ACCESS_OUT_OF_BOUNDS; return; } - // Drop random indexes - uint32_t drop_counter = 0U; - uint32_t index_to_drop; - while (drop_counter < indices_n_to_drop) { - // Generate random index - index_to_drop = rk_random_() % bit_array->length; - - // Ignore if already dropped - if (!bit_array_get_bit(bit_array, index_to_drop)) { - bit_array_set_bit(bit_array, index_to_drop); - drop_counter++; + // Handle 100% keep / drop + if (indices_n_to_drop_or_keep == bit_array->length) + for (uint32_t i = 0; i < bit_array->length; ++i) + bit_array_set_bit(bit_array, i); + + // Set random indexes + else { + uint32_t set_counter = 0U; + uint32_t index_to_set; + while (set_counter < indices_n_to_drop_or_keep) { + // Generate random index + index_to_set = rk_random_() % bit_array->length; + + // Ignore if already set + if (bit_array_get_bit(bit_array, index_to_set)) + continue; + + // Set bit and increment counter + bit_array_set_bit(bit_array, index_to_set); + set_counter++; } } + + // Invert if we use keep mode + if (dropout_ratio <= 1.f && dropout_ratio >= .5f) + bit_array_not(bit_array); } diff --git a/src/forward.c b/src/forward.c index 85e7103..81ddff3 100755 --- a/src/forward.c +++ b/src/forward.c @@ -38,12 +38,12 @@ void petal_forward(petal_s *petal, float *input, bool training) { // Calculate dropout bool dropout_enabled = false; - if (training && petal->dropout > 0.f && petal->bit_array) { + if (training && petal->params.dropout > 0.f && petal->bit_array) { // Clear previous dropout bit_array_clear(petal->bit_array); // Generate new dropout - dropout_generate_indices(petal->bit_array, petal->dropout); + dropout_generate_indices(petal->bit_array, petal->params.dropout); if (petal->bit_array->error_code != ERROR_NONE) { logger(LOG_E, "petal_forward", "Error generating dropout indices: %s", error_to_str[petal->bit_array->error_code]); @@ -82,7 +82,8 @@ void petal_forward(petal_s *petal, float *input, bool training) { petal->output[i] = 0.f; else { petal->output[i] = ((input[i] - min_value) / (max_value - min_value + EPSILON)); - petal->output[i] = petal->output[i] * 2.f * petal->deviation + petal->center - petal->deviation; + petal->output[i] = + petal->output[i] * 2.f * petal->params.deviation + petal->params.center - petal->params.deviation; } } } @@ -113,8 +114,8 @@ void petal_forward(petal_s *petal, float *input, bool training) { petal->output[index] = 0.f; else { petal->output[index] = ((input[index] - min_value) / (max_value - min_value + EPSILON)); - petal->output[index] = - petal->output[index] * 2.f * petal->deviation + petal->center - petal->deviation; + petal->output[index] = petal->output[index] * 2.f * petal->params.deviation + petal->params.center - + petal->params.deviation; } } } @@ -143,8 +144,8 @@ void petal_forward(petal_s *petal, float *input, bool training) { petal->output[index] = 0.f; else { petal->output[index] = ((input[index] - min_value) / (max_value - min_value + EPSILON)); - petal->output[index] = - petal->output[index] * 2.f * petal->deviation + petal->center - petal->deviation; + petal->output[index] = petal->output[index] * 2.f * petal->params.deviation + petal->params.center - + petal->params.deviation; } } } @@ -195,7 +196,7 @@ void petal_forward(petal_s *petal, float *input, bool training) { // Normalize sum after activation if dropout is enabled if (dropout_enabled) { - float dropout_scaling = 1.f / (1.f - petal->dropout + EPSILON); + float dropout_scaling = 1.f / (1.f - petal->params.dropout + EPSILON); for (uint32_t i = 0; i < petal->output_shape->length; ++i) if (petal->output[i] != 0.f) petal->output[i] *= dropout_scaling; diff --git a/src/petal.c b/src/petal.c index 63cda6b..402566b 100755 --- a/src/petal.c +++ b/src/petal.c @@ -67,14 +67,15 @@ * relu_leak - leak amount (for ACTIVATION_RELU only). Default = 0.01, * elu_alpha - the value to which an ELU saturates for negative net inputs (for ACTIVATION_ELU only). Default = 0.01, * swish_beta - beta for turning Swish into E-Swish (for ACTIVATION_SWISH only). Default = 1.0 - * @param dropout ratio of dropped outputs (0 to 1) - * @param center center of normalization for PETAL_TYPE_NORMALIZE_... Default: 0.0 - * @param deviation deviation of normalization for PETAL_TYPE_NORMALIZE_... Default: 1.0 + * @param params - pointer to petal_params_s struct(for dropout / normalization) + * or NULL if it doesn't need / default values are ok: + * dropout - ratio of dropped outputs (0 to 1) (Default: 0.0) + * center - center of normalization for PETAL_TYPE_NORMALIZE_... (Default: 0.0) + * deviation - deviation of normalization for PETAL_TYPE_NORMALIZE_... (Default: 1.0) * @return petal_s* petal's struct */ petal_s *petal_init(uint8_t petal_type, bool first, petal_shape_s *input_shape, petal_shape_s *output_shape, - weights_s *weights, weights_s *bias_weights, activation_s *activation, float dropout, float center, - float deviation) { + weights_s *weights, weights_s *bias_weights, activation_s *activation, petal_params_s *params) { // Log logger(LOG_I, "petal_init", "Initializing petal with type: %u", petal_type); @@ -96,13 +97,24 @@ petal_s *petal_init(uint8_t petal_type, bool first, petal_shape_s *input_shape, petal->weights = weights; petal->bias_weights = bias_weights; petal->activation = activation; - petal->dropout = dropout; - petal->center = center; - petal->deviation = deviation; petal->bit_array = NULL; if (petal->activation) petal->activation->_derivatives_temp = NULL; + // Copy params + if (params) { + petal->params.dropout = params->dropout; + petal->params.center = params->center; + petal->params.deviation = params->deviation; + } + + // Initialize params with default values + else { + petal->params.dropout = 0.f; + petal->params.center = 0.f; + petal->params.deviation = 1.f; + } + // Check petal type if (petal_type > PETAL_TYPE_MAX) { logger(LOG_E, "petal_init", "Wrong petal type: %u", petal_type); @@ -152,7 +164,7 @@ petal_s *petal_init(uint8_t petal_type, bool first, petal_shape_s *input_shape, } // Initialize dropout - if (dropout > 0.f) { + if (petal->params.dropout > 0.f) { petal->bit_array = bit_array_init(output_shape->length); if (petal->bit_array->error_code != ERROR_NONE) { logger(LOG_E, "petal_init", "Dropout bit array initialization error: %s", diff --git a/test/main.c b/test/main.c index c00484e..3536c46 100755 --- a/test/main.c +++ b/test/main.c @@ -398,12 +398,11 @@ uint8_t test_loss_full() { /** * @brief Tests dropout bit array generation * + * @param bit_size array size + * @param target_ratio how many indices to drop in % (ratio 0 to 1) * @return uint8_t number of fails (0 or 1) */ -uint8_t test_dropout() { - // Test data - uint32_t bit_size = 50U; - float target_ratio = 0.2; +uint8_t test_dropout(uint32_t bit_size, float target_ratio) { printf("\nTesting dropout on array with size %u and ratio: %.2f\n", bit_size, target_ratio); // Initialize array of bits @@ -455,9 +454,8 @@ uint8_t test_normalization() { petal_shape_s output_shape = (petal_shape_s){1U, 12U, 1U, 0UL}; printf("\nTesting normalization petals\n"); - // 1D - petal_s *petal = - petal_init(PETAL_TYPE_NORMALIZE_ALL, false, &input_shape, &output_shape, NULL, NULL, NULL, 0.f, 0.f, 1.f); + // 1D (with default petal_params_s) + petal_s *petal = petal_init(PETAL_TYPE_NORMALIZE_ALL, false, &input_shape, &output_shape, NULL, NULL, NULL, NULL); printf("1D (PETAL_TYPE_NORMALIZE_ALL) Input data:\n"); print_array(inputs, input_shape.rows, input_shape.cols, input_shape.depth); printf("Normalized:\n"); @@ -480,13 +478,12 @@ uint8_t test_normalization() { petal_destroy(petal, true, true, true); printf("\n"); - // 2D + // 2D (with default petal_params_s) input_shape.rows = 3U; input_shape.cols = 4U; output_shape.rows = 3U; output_shape.cols = 4U; - petal = - petal_init(PETAL_TYPE_NORMALIZE_IN_ROWS, false, &input_shape, &output_shape, NULL, NULL, NULL, 0.f, 0.f, 1.f); + petal = petal_init(PETAL_TYPE_NORMALIZE_IN_ROWS, false, &input_shape, &output_shape, NULL, NULL, NULL, NULL); printf("2D (PETAL_TYPE_NORMALIZE_IN_ROWS) Input data:\n"); print_array(inputs, input_shape.rows, input_shape.cols, input_shape.depth); printf("Normalized:\n"); @@ -509,15 +506,14 @@ uint8_t test_normalization() { petal_destroy(petal, true, true, true); printf("\n"); - // 3D + // 3D (with default petal_params_s) input_shape.rows = 3U; input_shape.cols = 2U; input_shape.depth = 2U; output_shape.rows = 3U; output_shape.cols = 2U; output_shape.depth = 2U; - petal = petal_init(PETAL_TYPE_NORMALIZE_IN_CHANNELS, false, &input_shape, &output_shape, NULL, NULL, NULL, 0.f, 0.f, - 1.f); + petal = petal_init(PETAL_TYPE_NORMALIZE_IN_CHANNELS, false, &input_shape, &output_shape, NULL, NULL, NULL, NULL); printf("3D (PETAL_TYPE_NORMALIZE_IN_CHANNELS) Input data:\n"); print_array(inputs, input_shape.rows, input_shape.cols, input_shape.depth); printf("Normalized:\n"); @@ -647,17 +643,17 @@ uint8_t test_dense() { petal_init(PETAL_TYPE_DENSE_1D, true, &(petal_shape_s){1U, 2U, 1U, 0UL}, &(petal_shape_s){1U, 2U, 1U, 0UL}, &(weights_s){true, WEIGHTS_INIT_XAVIER_GLOROT_GAUSSIAN, 4U, NULL, NULL, 0.f, 1.f, NULL, NULL, 0U}, &(weights_s){true, WEIGHTS_INIT_CONSTANT, 2U, NULL, NULL, 0.f, 1.f, NULL, NULL, 0U}, - &(activation_s){ACTIVATION_RELU, 1.f, 0.f, 0.0f, 0.00f, 1.f, NULL}, 0.0f, 0.f, 1.f); + &(activation_s){ACTIVATION_RELU, 1.f, 0.f, 0.0f, 0.00f, 1.f, NULL}, NULL); petal_s *petal_hidden2 = petal_init(PETAL_TYPE_DENSE_1D, false, &(petal_shape_s){1U, 2U, 1U, 0UL}, &(petal_shape_s){1U, 2U, 1U, 0UL}, &(weights_s){true, WEIGHTS_INIT_XAVIER_GLOROT_GAUSSIAN, 4U, NULL, NULL, 0.f, 1.f, NULL, NULL, 0U}, &(weights_s){true, WEIGHTS_INIT_CONSTANT, 2U, NULL, NULL, 0.f, 1.f, NULL, NULL, 0U}, - &(activation_s){ACTIVATION_RELU, 1.f, 0.f, 0.0f, 0.00f, 1.f, NULL}, 0.0f, 0.f, 1.f); + &(activation_s){ACTIVATION_RELU, 1.f, 0.f, 0.0f, 0.00f, 1.f, NULL}, NULL); petal_s *petal_output = petal_init(PETAL_TYPE_DENSE_1D, false, &(petal_shape_s){1U, 2U, 1U, 0UL}, &(petal_shape_s){1U, 2U, 1U, 0UL}, &(weights_s){true, WEIGHTS_INIT_XAVIER_GLOROT_GAUSSIAN, 6U, NULL, NULL, 0.f, 1.f, NULL, NULL, 0U}, &(weights_s){true, WEIGHTS_INIT_CONSTANT, 2U, NULL, NULL, 0.f, 1.f, NULL, NULL, 0U}, - &(activation_s){ACTIVATION_SOFTMAX, 1.f, 0.f, 0.0f, 0.01f, 1.f, NULL}, 0.0f, 0.f, 1.f); + &(activation_s){ACTIVATION_SOFTMAX, 1.f, 0.f, 0.0f, 0.01f, 1.f, NULL}, NULL); // Print weights printf("In -> hidden 1 weights:\n"); @@ -810,11 +806,11 @@ uint8_t test_random() { /** * @brief Automatically performs tests of most function * - * @return int 0 in all test passed, -1 otherwise + * @return int 0 if all test passed, number of fails otherwise */ int main() { // Fails counter - uint8_t fails = 0U; + uint16_t fails = 0U; // Set random seed (seed must be 0 for test_random to work and also for results to be consistant) // rk_seed_(time(NULL) & 0xFFFFFFFFUL); @@ -835,7 +831,10 @@ int main() { printf("\n--------------------------------------------------------------------------------\n"); // Test dropout - fails += test_dropout(); + fails += test_dropout(50U, 0.f); + fails += test_dropout(50U, .2f); + fails += test_dropout(50U, .8f); + fails += test_dropout(50U, 1.f); printf("\n--------------------------------------------------------------------------------\n"); // Test normalization @@ -849,9 +848,8 @@ int main() { // Print number of fails during tests printf("\nFails: %u\n", fails); - if (fails == 0U) { + if (fails == 0U) printf("All tests passed successfully!\n"); - return 0; - } - return -1; + + return fails; }