From b7dffc73ee9d83b9d696e1123922ca56ba410ee1 Mon Sep 17 00:00:00 2001 From: Artem Gavrilov Date: Fri, 2 Aug 2024 15:39:00 +0200 Subject: [PATCH 1/3] PG-934 Fix old telemetry files loading on startup (#10) * PG-934 Fix old telemetry files loading on startup * PG-934 Fix build for PG12 * PG-934 Fix codestyle * PG-934 Fix --- percona_pg_telemetry.c | 70 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 61 insertions(+), 9 deletions(-) diff --git a/percona_pg_telemetry.c b/percona_pg_telemetry.c index 84fc644..db5298f 100644 --- a/percona_pg_telemetry.c +++ b/percona_pg_telemetry.c @@ -74,10 +74,16 @@ static shmem_request_hook_type prev_shmem_request_hook = NULL; static BgwHandleStatus setup_background_worker(const char *bgw_function_name, const char *bgw_name, const char *bgw_type, Oid datid, pid_t bgw_notify_pid); static void start_leader(void); static long server_uptime(void); -static void cleaup_telemetry_dir(void); +static void load_telemery_files(void); static char *generate_filename(char *filename); static bool validate_dir(char *folder_path); +#if PG_VERSION_NUM >= 130000 +static int compare_file_names(const ListCell *a, const ListCell *b); +#else +static int compare_file_names(const void *a, const void *b); +#endif + /* Database information collection and writing to file */ static void write_pg_settings(void); static List *get_database_list(void); @@ -198,12 +204,17 @@ telemetry_file_is_valid(void) } /* - * + * Adds a new filename to the next position in the circular buffer. If position already has a filename + * (i.e. we made full circle), then it will try to remove this file from filesystem. + * Returns the previous filename that was in the position. */ static char * telemetry_file_next(char *filename) { - char *curr_oldest = telemetry_curr_filename(); + /* Get current file that will become previous */ + char *previous = telemetry_curr_filename(); + + /* Increment the index. We are using a circular buffer. */ ptss->curr_file_index = (ptss->curr_file_index + 1) % files_to_keep; /* Remove the existing file on this location if valid */ @@ -212,21 +223,26 @@ telemetry_file_next(char *filename) PathNameDeleteTemporaryFile(ptss->telemetry_filenames[ptss->curr_file_index], false); } + /* Add new file to the new current position */ telemetry_add_filename(filename); - return (*curr_oldest) ? curr_oldest : NULL; + /* Return previous file */ + return (*previous) ? previous : NULL; } /* - * + * Load all telemetry files from the telemetry directory. */ static void -cleaup_telemetry_dir(void) +load_telemery_files(void) { DIR *d; struct dirent *de; uint64 system_id = GetSystemIdentifier(); char json_file_id[MAXPGPATH]; + char full_path[MAXPGPATH]; + List *files_list = NIL; + ListCell *lc = NULL; int file_id_len; validate_dir(ptss->telemetry_path); @@ -248,13 +264,49 @@ cleaup_telemetry_dir(void) { if (strncmp(json_file_id, de->d_name, file_id_len) == 0) { - telemetry_file_next(de->d_name); + /* Construct the file full path */ + snprintf(full_path, sizeof(full_path), "%s/%s", ptss->telemetry_path, de->d_name); + + files_list = lappend(files_list, pstrdup(full_path)); } } +#if PG_VERSION_NUM >= 130000 + list_sort(files_list, compare_file_names); +#else + files_list = list_qsort(files_list, compare_file_names); +#endif + + foreach(lc, files_list) + { + char *file_path = lfirst(lc); + telemetry_file_next(file_path); + } + + list_free_deep(files_list); FreeDir(d); } + +#if PG_VERSION_NUM >= 130000 +static int +compare_file_names(const ListCell *a, const ListCell *b) +{ + char *fna = (char *) lfirst(a); + char *fnb = (char *) lfirst(b); + return strcmp(fna, fnb); +} + +#else +static int +compare_file_names(const void *a, const void *b) +{ + char *fna = (char *) lfirst(*(ListCell **) a); + char *fnb = (char *) lfirst(*(ListCell **) b); + return strcmp(fna, fnb); +} +#endif + /* * telemetry_path */ @@ -850,8 +902,8 @@ percona_pg_telemetry_main(Datum main_arg) /* Initialize shmem */ pt_shmem_init(); - /* Cleanup the directory */ - cleaup_telemetry_dir(); + /* Load existing telemetry files */ + load_telemery_files(); /* Set up connection */ BackgroundWorkerInitializeConnectionByOid(InvalidOid, InvalidOid, 0); From 4d06aa391752109a373faed8ecd5af79bff58552 Mon Sep 17 00:00:00 2001 From: Artem Gavrilov Date: Fri, 2 Aug 2024 16:04:06 +0200 Subject: [PATCH 2/3] PG-947 Set 0644 mode for telemetry files (#11) --- percona_pg_telemetry.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/percona_pg_telemetry.c b/percona_pg_telemetry.c index db5298f..67b6b8f 100644 --- a/percona_pg_telemetry.c +++ b/percona_pg_telemetry.c @@ -50,6 +50,7 @@ PG_MODULE_MAGIC; /* General defines */ #define PT_BUILD_VERSION "1.0" #define PT_FILENAME_BASE "percona_pg_telemetry" +#define PT_FILE_MODE (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) /* Init and exported functions */ void _PG_init(void); @@ -1026,6 +1027,9 @@ percona_pg_telemetry_main(Datum main_arg) /* Generate and save the filename */ telemetry_file_next(generate_filename(filename)); + /* Change the file permissions before making it available to the agent. */ + chmod(ptss->dbtemp_filepath, PT_FILE_MODE); + /* Let's rename the temp file so that agent can pick it up. */ if (rename(ptss->dbtemp_filepath, telemetry_curr_filename()) < 0) { From 8a13c5b2b092896be80aeb3055226d93d34768bb Mon Sep 17 00:00:00 2001 From: Artem Gavrilov Date: Mon, 5 Aug 2024 15:36:30 +0200 Subject: [PATCH 3/3] PG-946 fix telemetry filenames (#13) * PG-946 Fix telemetry filenames * PG-946 Update tests --- expected/basic.out | 6 +++--- expected/basic_1.out | 6 +++--- percona_pg_telemetry.c | 14 +++++--------- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/expected/basic.out b/expected/basic.out index 79427bb..bf615f2 100644 --- a/expected/basic.out +++ b/expected/basic.out @@ -29,9 +29,9 @@ SELECT percona_pg_telemetry_version(); SELECT regexp_replace(regexp_replace(latest_output_filename, '\d{11,}', '', 'g'), '\d{6,}', '', 'g') AS latest_output_filename , pt_enabled FROM percona_pg_telemetry_status(); - latest_output_filename | pt_enabled --------------------------------------------------------+------------ - ./percona_pg_telemetry--.json | t + latest_output_filename | pt_enabled +----------------------------------+------------ + ./-.json | t (1 row) DROP EXTENSION percona_pg_telemetry; diff --git a/expected/basic_1.out b/expected/basic_1.out index f013f22..960aeff 100644 --- a/expected/basic_1.out +++ b/expected/basic_1.out @@ -27,9 +27,9 @@ SELECT percona_pg_telemetry_version(); SELECT regexp_replace(regexp_replace(latest_output_filename, '\d{11,}', '', 'g'), '\d{6,}', '', 'g') AS latest_output_filename , pt_enabled FROM percona_pg_telemetry_status(); - latest_output_filename | pt_enabled --------------------------------------------------------------------------------------+------------ - /usr/local/percona/telemetry/pg/percona_pg_telemetry--.json | t + latest_output_filename | pt_enabled +----------------------------------------------------------------+------------ + /usr/local/percona/telemetry/pg/-.json | t (1 row) DROP EXTENSION percona_pg_telemetry; diff --git a/percona_pg_telemetry.c b/percona_pg_telemetry.c index 67b6b8f..cee3b78 100644 --- a/percona_pg_telemetry.c +++ b/percona_pg_telemetry.c @@ -49,7 +49,6 @@ PG_MODULE_MAGIC; /* General defines */ #define PT_BUILD_VERSION "1.0" -#define PT_FILENAME_BASE "percona_pg_telemetry" #define PT_FILE_MODE (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) /* Init and exported functions */ @@ -167,7 +166,7 @@ generate_filename(char *filename) time_t currentTime; time(¤tTime); - pg_snprintf(f_name, MAXPGPATH, "%s-%lu-%ld.json", PT_FILENAME_BASE, system_id, currentTime); + pg_snprintf(f_name, MAXPGPATH, "%ld-%lu.json", currentTime, system_id); join_path_components(filename, ptss->telemetry_path, f_name); @@ -240,11 +239,10 @@ load_telemery_files(void) DIR *d; struct dirent *de; uint64 system_id = GetSystemIdentifier(); - char json_file_id[MAXPGPATH]; + char filename_tail[MAXPGPATH]; char full_path[MAXPGPATH]; List *files_list = NIL; ListCell *lc = NULL; - int file_id_len; validate_dir(ptss->telemetry_path); @@ -258,12 +256,10 @@ load_telemery_files(void) ptss->telemetry_path))); } - pg_snprintf(json_file_id, sizeof(json_file_id), "%s-%lu", PT_FILENAME_BASE, system_id); - file_id_len = strlen(json_file_id); - + pg_snprintf(filename_tail, sizeof(filename_tail), "%lu.json", system_id); while ((de = ReadDir(d, ptss->telemetry_path)) != NULL) { - if (strncmp(json_file_id, de->d_name, file_id_len) == 0) + if (strstr(de->d_name, filename_tail) != NULL) { /* Construct the file full path */ snprintf(full_path, sizeof(full_path), "%s/%s", ptss->telemetry_path, de->d_name); @@ -413,7 +409,7 @@ pt_shmem_init(void) /* Set paths */ strncpy(ptss->telemetry_path, t_folder, MAXPGPATH); - pg_snprintf(ptss->dbtemp_filepath, MAXPGPATH, "%s/%s-%lu.temp", ptss->telemetry_path, PT_FILENAME_BASE, system_id); + pg_snprintf(ptss->dbtemp_filepath, MAXPGPATH, "%s/%lu.temp", ptss->telemetry_path, system_id); /* Let's be optimistic here. No error code and no file currently being written. */ ptss->error_code = PT_SUCCESS;