From 313e61c05c5c5e409c2f66057b072bf741cab944 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Thu, 29 Aug 2024 09:09:19 -0400 Subject: [PATCH 1/3] Converting row counts in JSON to int The previous push had the row counts stored as strings. If we want to do anything automated with this in the future (?), it would be to our benefit to store the row counts as integers as opposed to strings. Updates the record structure and adds an `Atoi` for the row counts. If the `Atoi` fails, we store -1. --- cmd/row_count.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/row_count.go b/cmd/row_count.go index 46eb66d..3f67f94 100644 --- a/cmd/row_count.go +++ b/cmd/row_count.go @@ -23,7 +23,7 @@ var ( type RowCount struct { Table string `json:"Table"` - Rows string `json:"Rows"` + Rows int `json:"Rows"` } type connection struct { @@ -76,7 +76,11 @@ func check_rows_in_db(source_creds vcap.Credentials) { for _, joined_tables := range strings.Split(joined_tables, "\n") { if joined_tables != "" { s := strings.Split(joined_tables, " ") - rows = append(rows, &RowCount{Table: s[0], Rows: s[1]}) + rows_as_int, err := strconv.Atoi(s[1]) + if err != nil { + rows_as_int = -1 + } + rows = append(rows, &RowCount{Table: s[0], Rows: rows_as_int}) } } From f50edf2a7fc0e83e8b1fae1f35d51a35c240b6c2 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Tue, 29 Oct 2024 15:23:51 -0400 Subject: [PATCH 2/3] Removing tables no longer used --- cmd/assets/db_tables.txt | 149 +++++++++------------------------------ 1 file changed, 34 insertions(+), 115 deletions(-) diff --git a/cmd/assets/db_tables.txt b/cmd/assets/db_tables.txt index bed8911..80afbcb 100644 --- a/cmd/assets/db_tables.txt +++ b/cmd/assets/db_tables.txt @@ -1,134 +1,53 @@ -census_duns21 -census_eins18 -census_cfda19 -census_duns18 -census_eins16 -census_cpas21 -census_cpas17 -census_agency16 -census_captext19 -census_captext_formatted21 -census_cfda20 -census_captext20 -census_agency17 -census_captext21 -dissemination_note -census_cfda16 -census_duns19 -census_cpas18 -census_cpas22 -census_agency19 -census_eins17 -census_duns22 -census_cfda21 -census_captext_formatted19 -census_cpas19 -census_duns16 -census_agency20 -census_cfda22 -census_cfda17 -census_notes20 -census_duns17 -census_cfda18 -census_cpas20 -census_duns20 -census_captext_formatted20 -census_agency22 -census_cpas16 -census_findings18 -census_eins19 -census_findings16 -census_findings20 -census_findings22 -census_findings21 -census_eins21 -census_eins22 -census_eins20 -census_findings19 -census_findings17 -census_findingstext_formatted22 -census_findingstext21 -census_gen16 -census_gen18 -census_findingstext_formatted19 -census_gen17 -census_findingstext_formatted20 -census_findingstext_formatted21 -census_findingstext19 -census_findingstext20 -census_gen22 -census_notes21 -census_notes19 -census_passthrough16 -census_passthrough18 -census_gen19 -census_gen21 -census_gen20 -census_passthrough17 -census_notes22 -census_passthrough22 -census_revisions22 -census_ueis22 -census_revisions19 -census_revisions21 -census_revisions20 -census_passthrough20 -census_passthrough19 -census_passthrough21 +audit_access audit_deletedaccess -census_findingstext22 +audit_excelfile +audit_singleauditchecklist +audit_singleauditreportfile +audit_submissionevent +auth_group auth_group_permissions -django_content_type auth_permission -django_migrations -auth_group auth_user -auth_user_user_permissions -django_admin_log auth_user_groups -audit_submissionevent +auth_user_user_permissions authtoken_token -audit_singleauditreportfile census_historical_migration_elecauditfindings -census_historical_migration_eleceins -census_historical_migration_elecfindingstext -census_historical_migration_eleccpas -census_historical_migration_elecnotes census_historical_migration_elecauditheader census_historical_migration_elecaudits census_historical_migration_eleccaptext -dissemination_additionalein -census_historical_migration_reportmigrationstatus -census_historical_migration_migrationerrordetail -census_historical_migration_elecueis +census_historical_migration_eleccpas +census_historical_migration_eleceins +census_historical_migration_elecfindingstext +census_historical_migration_elecnotes census_historical_migration_elecpassthrough +census_historical_migration_elecueis +census_historical_migration_migrationerrordetail +census_historical_migration_reportmigrationstatus +dissemination_additionalein +dissemination_additionaluei +dissemination_captext +dissemination_federalaward dissemination_finding -support_administrative_key_uuids +dissemination_findingtext dissemination_general +dissemination_invalidauditrecord +dissemination_issuedescriptionrecord dissemination_migrationinspectionrecord +dissemination_note +dissemination_onetimeaccess +dissemination_passthrough +dissemination_secondaryauditor +dissemination_tribalapiaccesskeyids +django_admin_log +django_content_type +django_migrations django_session -dissemination_issuedescriptionrecord -dissemination_invalidauditrecord -users_permission support_adminapievent -users_staffuserlog -users_userprofile +support_administrative_key_uuids support_cognizantassignment -users_staffuser support_cognizantbaseline -dissemination_tribalapiaccesskeyids -dissemination_findingtext -dissemination_additionaluei -dissemination_captext +users_permission +users_staffuser +users_staffuserlog users_userpermission -dissemination_onetimeaccess -dissemination_federalaward -dissemination_passthrough -audit_singleauditchecklist -census_captext22 -census_captext_formatted22 -census_agency21 -dissemination_secondaryauditor -census_agency18 -audit_excelfile -audit_access +users_userprofile From 0ca0010cad512cb0a73b959d5bc6469e3b3be3df Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Tue, 29 Oct 2024 15:43:55 -0400 Subject: [PATCH 3/3] Revert "Converting row counts in JSON to int" This reverts commit 313e61c05c5c5e409c2f66057b072bf741cab944. --- cmd/row_count.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/cmd/row_count.go b/cmd/row_count.go index 3f67f94..46eb66d 100644 --- a/cmd/row_count.go +++ b/cmd/row_count.go @@ -23,7 +23,7 @@ var ( type RowCount struct { Table string `json:"Table"` - Rows int `json:"Rows"` + Rows string `json:"Rows"` } type connection struct { @@ -76,11 +76,7 @@ func check_rows_in_db(source_creds vcap.Credentials) { for _, joined_tables := range strings.Split(joined_tables, "\n") { if joined_tables != "" { s := strings.Split(joined_tables, " ") - rows_as_int, err := strconv.Atoi(s[1]) - if err != nil { - rows_as_int = -1 - } - rows = append(rows, &RowCount{Table: s[0], Rows: rows_as_int}) + rows = append(rows, &RowCount{Table: s[0], Rows: s[1]}) } }