fed_pubs/make_file.R at master · BrunaLab/fed_pubs · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150

# SCRAPE DATA FROM SCOPUS
#
# `01a_scopus_API_fed.R`
# `01b_scopus_API_uni.R`

# DOWNLOAD DATA FROM USGS PUBLICATIONS WAREHOUSE
# https://pubs.usgs.gov/
# save as usgs_YYYYMMDD.csv in './data_raw/usgs_warehouse_downloads'

# process the usgs publication file downloaded from their website ---------

# api_code<-""
# usgs_mostrecent<-"usgs_20260624.csv"
# # # Process with `process_usgs.R`
# source("./code/process_usgs.R")
# process_usgs(usgs_mostrecent,api_code)


###########################################################################
###########################################################################

# to fix: 60002746_2017_04-1

# chose data download date and max month  ---------------------------------


date<-"20260101"
PM_max<-12 # using december - lag time of 3 moinths = ~3-5% underestimate
PY_max<-2026
PY_min_dataset<-2020
PY_min_analyses<-2020
author_position<-"first"
# PM_last_yr<-6
# this is the month_id
cutoff<-77
# 84 = 12 2026
# 83 = 11 2026
# 82 = 10 2026
# 81 = 9 2026
# 80 = 8 2026
# 79 = 7 2026
# 78 = 6 2026
# 77 = 5 2026

# author_position<-"anywhere"

# process federal records -------------------------------------------------

# within year: bind csvs for the scopus IDs -------------------------------

cat<-"fed"
source("./code/csv_binder_within_year.R")
csv_binder_within_year(cat,date)

cat<-"uni"
source("./code/csv_binder_within_year.R")
csv_binder_within_year(cat,date)

# across years: bind the annual csvs --------------------------------------

cat<-"fed"
source("./code/csv_binder_across_years.R")
csv_binder_across_years(cat,date)

cat<-"uni"
source("./code/csv_binder_across_years.R")
csv_binder_across_years(cat,date)

# clean up the composite files --------------------------------------------

##### Need to update fix_usgs_affils to get most recent usgs


# Fed Files
source("./code/clean_fed.R")
clean_fed(date)

# Uni Files
source("./code/clean_uni.R")
clean_uni(date)


# prep datasets for analyses and make overall summaries

# NOTE THE FOLLOWING ARE EXCLUDING *ONLY* BOOK CHAPTERS
# IF YOU WANT BOOK CHAPTERS NEED TO UNCOMMENT
# AND ALSO
# NOT EXCLUDING ANY BY FLAG WORDS

source("./code/prep_analysis_datasets_fed.R")
prep_analysis_datasets_fed(date, PM_max,PY_min_dataset,PY_max)

source("./code/prep_analysis_datasets_uni.R")
prep_analysis_datasets_uni(date, PM_max, PY_min_dataset, PY_max)


# figures and summaries of results ----------------------------------------

source("./code/make_figs_fed.R")
make_figs_fed(date,PM_max,PY_min_analyses,PY_max,author_position,cutoff)

source("./code/make_figs_uni.R")
make_figs_uni(date,PM_max,PY_min_analyses,PY_max,author_position,cutoff)


# bootstrapping -----------------------------------------------------------

source("./code/bootstrap_npubs_fed.R")
bootstrap_npubs_fed(date,PY_max,PM_last_yr,author_position)

source("./code/bootstrap_npubs_uni.R")
bootstrap_npubs_uni(date,PY_max,PM_last_yr,author_position)

# render MS ^ Supplementary Information File -------------------------------


# rmarkdown::render('./docs/Bruna_MS.Rmd'),params=list(args = myarg))

# rmarkdown::render('./docs/Bruna_Supporting_Info.Rmd'),params=list(args = myarg))


###########################################################################
###########################################################################


# lag test ----------------------------------------------------------------


# lag test ----------------------------------------------------------------
library(here)
source(here("code","lag_test.R"))
lag_test_output<-lag_test()
lag_data<-as.data.frame(lag_test_output[1])
lag_fig<-lag_test_output[2]
# lag_fig<-lag_fig[[1]]
sum(lag_data$n_lag)
# date1<-"20250901"
# date2<-"20251010"
#
#
# source("./code/lag_test_fed.R")
# lag_test_uni(date1, date2,PM_max)
#
#
# source("./code/lag_test_uni.R")
# lag_test_uni(cat, date,PM_max)