-
Notifications
You must be signed in to change notification settings - Fork 4
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
2024 model data update #670
Changes from 167 commits
acb0154
0652b3a
5191e58
c4807fb
6e2a088
ccc67fa
6a9c130
12b2ebf
a22e314
d835273
7bd08b8
ca76942
8ba251d
1cf60f2
d3ba259
e34b8c8
4ae5554
d2a45fa
aace136
3f070de
b50e813
ebbc52e
e8671f7
967114c
ba0b0a1
aa91cf8
639468c
ed4ba5f
0c41137
7cc2b22
03bdf61
09ce316
7291c2d
f0ca25f
205e5c5
33e1993
ce34f6f
6469e29
0e8dc1f
b4351bc
6bc3d82
373ec21
3ba546f
6d15e66
fdae401
fe8a1cc
601292a
35bfcf9
ce87790
40ddfd7
a3131c6
364fafb
28cbeb2
387731c
e70ea40
43937d3
35dafbf
fe29661
1d118f4
3f5f221
f6bdb9a
7dd95cd
97d41bb
4e22dcc
4a1b303
e25112e
a4277e8
56307bb
cdf419c
1a3f32e
3647eed
9cd97ae
ff423a9
e415509
d903517
f177074
c03c320
48603c0
8544f80
f93fefd
53ef3be
bddc0c8
ed414a9
623a801
cc24ca4
ca3ab54
159df1c
5d8ee59
ecb5fe0
15e6c3b
88d62ea
b3d3e5f
fae0635
dadc100
72a244a
b896304
546cfef
7d98745
f333fa1
8daaffd
2dd70d7
64bbf96
40585a7
92a459c
50c8740
42f6815
d24d72a
530e843
d8afea6
cb4a74b
bb84a02
0390507
ce7e290
2b2b6c4
1e92767
bbef26f
1875dea
a54494e
a8375d3
f492114
5b0d5d7
3e32945
d761d88
2b1e903
08b765f
4c3c256
0739010
d4bddfb
15f480e
5922fd1
1cb245b
233d3fb
cf0d082
3bb9e54
26a236b
8c21816
4eeb02d
38f68fe
5e84a15
2762560
d095bfb
11b33fe
253a71d
b54f6f5
0659dcd
ea3b35c
c76ca75
52f5877
3528bdf
459fc8d
166701f
67771aa
c5e7818
4e6f2e6
7784c29
952b4b8
ab52d12
585a0d5
0760d2b
fb81594
009d471
98a4b96
b6bd970
6377dcc
f6fafdb
bf894a7
2716bdb
2e2805a
dbaa887
1661ece
61a4a4e
3c7fb11
c406783
8a15f61
d4f6241
0149fb1
5b1cbcd
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Not sure if this is in the spirit of the PR, but it does seem like it fits the notion of other tables built by ETL processes getting |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,6 +1,6 @@ | ||
{ | ||
"R": { | ||
"Version": "4.4.2", | ||
"Version": "4.4.1", | ||
"Repositories": [ | ||
{ | ||
"Name": "CRAN", | ||
|
@@ -2808,6 +2808,19 @@ | |
], | ||
"Hash": "ad57b543f7c3fca05213ba78ff63df9b" | ||
}, | ||
"sfarrow": { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This was never added to renv despite being used in the airport noise raster script. |
||
"Package": "sfarrow", | ||
"Version": "0.4.1", | ||
"Source": "Repository", | ||
"Repository": "CRAN", | ||
"Requirements": [ | ||
"arrow", | ||
"dplyr", | ||
"jsonlite", | ||
"sf" | ||
], | ||
"Hash": "b320f164b1d7bb7e4582b841e22d15a0" | ||
}, | ||
"sfheaders": { | ||
"Package": "sfheaders", | ||
"Version": "0.4.4", | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -46,6 +46,5 @@ down_up <- function(x) { | |
} | ||
} | ||
|
||
|
||
# Apply function to foreclosure data | ||
walk(files, down_up) |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -20,15 +20,25 @@ options(timeout = max(300, getOption("timeout"))) | |
cta_feed_dates_list <- c( | ||
"2015-10-29", "2016-09-30", "2017-10-22", "2018-10-06", | ||
"2019-10-04", "2020-10-10", "2021-10-09", "2022-10-20", | ||
"2023-10-04" | ||
"2023-10-04", "2024-10-17" | ||
) | ||
|
||
# If missing feed on S3, download and remove .htm file (causes errors) | ||
# then rezip and upload | ||
get_cta_feed <- function(feed_date) { | ||
feed_url <- paste0( | ||
"https://transitfeeds.com/p/chicago-transit-authority/165/", | ||
str_remove_all(feed_date, "-"), "/download" | ||
feed_url <- ifelse( | ||
substr(feed_date, 1, 4) <= "2023", | ||
paste0( | ||
"https://transitfeeds.com/p/chicago-transit-authority/165/", | ||
str_remove_all(feed_date, "-"), "/download" | ||
), | ||
paste0( | ||
"https://files.mobilitydatabase.org/mdb-389/mdb-389-", | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. transitfeeds is dead, so we need to use mobilitydatabase now |
||
str_remove_all(feed_date, "-"), | ||
"0023/mdb-389-", | ||
str_remove_all(feed_date, "-"), | ||
"0023.zip" | ||
) | ||
) | ||
s3_uri <- file.path(output_path, "cta", paste0(feed_date, "-gtfs.zip")) | ||
|
||
|
@@ -55,14 +65,26 @@ walk(cta_feed_dates_list, get_cta_feed) | |
metra_feed_dates_list <- c( | ||
"2015-10-30", "2016-09-30", "2017-10-21", "2018-10-05", | ||
"2019-10-04", "2020-10-10", "2021-10-08", "2022-10-21", | ||
"2023-10-14" | ||
"2023-10-14", "2024-04-22" | ||
) | ||
|
||
get_metra_feed <- function(feed_date) { | ||
feed_url <- paste0( | ||
"https://transitfeeds.com/p/metra/169/", | ||
str_remove_all(feed_date, "-"), "/download" | ||
|
||
feed_url <- ifelse( | ||
substr(feed_date, 1, 4) <= "2023", | ||
paste0( | ||
"https://transitfeeds.com/p/metra/169/", | ||
str_remove_all(feed_date, "-"), "/download" | ||
), | ||
paste0( | ||
"https://files.mobilitydatabase.org/mdb-1187/mdb-1187-", | ||
str_remove_all(feed_date, "-"), | ||
"0016/mdb-1187-", | ||
str_remove_all(feed_date, "-"), | ||
"0016.zip" | ||
) | ||
) | ||
|
||
s3_uri <- file.path(output_path, "metra", paste0(feed_date, "-gtfs.zip")) | ||
|
||
if (!aws.s3::object_exists(s3_uri)) { | ||
|
@@ -80,7 +102,8 @@ walk(metra_feed_dates_list, get_metra_feed) | |
##### Pace ##### | ||
pace_feed_dates_list <- c( | ||
"2015-10-16", "2016-10-15", "2017-10-16", "2018-10-17", | ||
"2019-10-22", "2020-09-23", "2021-03-15", "2023-09-24" | ||
"2019-10-22", "2020-09-23", "2021-03-15", "2023-09-24", | ||
"2024-02-07" | ||
) | ||
|
||
get_pace_feed <- function(feed_date) { | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Allows CI to pass.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
[Question, blocking] Can you link to an example workflow that failed prior to this change? I want to make sure I understand why this is the best path forward before we move ahead with it.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This was a Dan suggestion after I encountered this.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is definitely a weird and ephemeral bug, but basically the R in superlinter will load
renv
due to the presence of the.Rprofile
file in the working directory. The linter then fails because therenv
environment doesn't havelintr
in it. Removing the.Rprofile
file loads the default superlinter R environment.@wrridgeway You should add a comment to this step to explain why it's necessary/provide some context.