Line data Source code
1 : module WildfireData
2 :
3 : using Scratch
4 : using HTTP
5 : using JSON3
6 : using GeoJSON
7 :
8 : export WFIGS, IRWIN, FPA_FOD, MTBS, FIRMS, LANDFIRE, FEDS, CWFIS, HMS, GWIS, EGP, RxCADRE
9 :
10 : #-----------------------------------------------------------------------------# Data Directory
11 56 : dir(x...) = joinpath(Scratch.@get_scratch!("data"), x...)
12 :
13 : #-----------------------------------------------------------------------------# Abstract Dataset Interface
14 :
15 : """
16 : AbstractDataset
17 :
18 : Abstract type for dataset metadata. Subtypes must have a `name::String` field
19 : and implement `base_query_url(d)` and `base_layer_url(d)`.
20 : """
21 : abstract type AbstractDataset end
22 :
23 : """
24 : base_query_url(d::AbstractDataset) -> String
25 :
26 : Return the base URL for query requests (without parameters).
27 : """
28 : function base_query_url end
29 :
30 : """
31 : base_layer_url(d::AbstractDataset) -> String
32 :
33 : Return the base URL for layer metadata requests.
34 : """
35 : function base_layer_url end
36 :
37 : #-----------------------------------------------------------------------------# ArcGIS Dataset
38 :
39 : """
40 : ArcGISDataset
41 :
42 : Metadata for an ArcGIS FeatureServer dataset.
43 :
44 : # Fields
45 : - `base_url::String`: Base URL for the ArcGIS service
46 : - `service::String`: ArcGIS service name
47 : - `layer::Int`: Layer index (usually 0)
48 : - `name::String`: Human-readable name
49 : - `description::String`: Dataset description
50 : - `category::Symbol`: Category (e.g., :perimeters, :locations, :incidents, :history)
51 : """
52 : struct ArcGISDataset <: AbstractDataset
53 : base_url::String
54 : service::String
55 : layer::Int
56 : name::String
57 : description::String
58 : category::Symbol
59 : end
60 :
61 55 : base_query_url(d::ArcGISDataset) = "$(d.base_url)/$(d.service)/FeatureServer/$(d.layer)/query"
62 3 : base_layer_url(d::ArcGISDataset) = "$(d.base_url)/$(d.service)/FeatureServer/$(d.layer)"
63 :
64 : #-----------------------------------------------------------------------------# Common URL Builder
65 :
66 : """
67 : query_url(d::AbstractDataset; where="1=1", outfields="*", limit=nothing, format="geojson", bbox=nothing)
68 :
69 : Build a query URL for the dataset.
70 :
71 : # Arguments
72 : - `bbox`: Bounding box as `(west, south, east, north)` tuple or "west,south,east,north" string for spatial filtering.
73 : """
74 70 : function query_url(d::AbstractDataset; where::String="1=1", outfields::String="*",
75 : limit::Union{Int,Nothing}=nothing, format::String="geojson",
76 : bbox::Union{NTuple{4,Real},String,Nothing}=nothing)
77 35 : url = base_query_url(d)
78 35 : params = [
79 : "where" => HTTP.escapeuri(where),
80 : "outFields" => outfields,
81 : "f" => format,
82 : "outSR" => "4326",
83 : ]
84 35 : if !isnothing(limit)
85 44 : push!(params, "resultRecordCount" => string(limit))
86 : end
87 35 : if !isnothing(bbox)
88 0 : envelope = bbox isa String ? bbox : join(bbox, ",")
89 0 : push!(params, "geometry" => envelope)
90 0 : push!(params, "geometryType" => "esriGeometryEnvelope")
91 0 : push!(params, "inSR" => "4326")
92 0 : push!(params, "spatialRel" => "esriSpatialRelIntersects")
93 : end
94 35 : return url * "?" * join(["$k=$v" for (k, v) in params], "&")
95 : end
96 :
97 : #-----------------------------------------------------------------------------# Common API Functions
98 :
99 : """
100 : _datasets(all_datasets::Dict{Symbol,<:AbstractDataset}; category=nothing)
101 :
102 : Filter datasets by category. Used internally by submodules.
103 : """
104 14 : function _datasets(all_datasets::Dict{Symbol,<:AbstractDataset}; category::Union{Symbol,Nothing}=nothing)
105 14 : if isnothing(category)
106 7 : return all_datasets
107 : else
108 7 : return Dict(k => v for (k, v) in all_datasets if v.category == category)
109 : end
110 : end
111 :
112 : """
113 : _info(all_datasets::Dict{Symbol,<:AbstractDataset}, dataset::Symbol, module_name::String)
114 :
115 : Print information about a specific dataset. Used internally by submodules.
116 : """
117 6 : function _info(all_datasets::Dict{Symbol,<:AbstractDataset}, dataset::Symbol, module_name::String)
118 12 : if !haskey(all_datasets, dataset)
119 3 : error("Unknown dataset: $dataset. Use `$module_name.datasets()` to list available datasets.")
120 : end
121 3 : d = all_datasets[dataset]
122 3 : println("Dataset: ", d.name)
123 3 : println("Category: ", d.category)
124 3 : println("Description: ", d.description)
125 3 : println("Service: ", d.service)
126 3 : println("Query URL: ", query_url(d))
127 3 : return nothing
128 : end
129 :
130 : """
131 : _download(all_datasets::Dict{Symbol,<:AbstractDataset}, dataset::Symbol, module_name::String; kwargs...)
132 :
133 : Download a dataset and return it as a GeoJSON FeatureCollection. Used internally by submodules.
134 : """
135 44 : function _download(all_datasets::Dict{Symbol,<:AbstractDataset}, dataset::Symbol, module_name::String;
136 : where::String="1=1", fields::String="*", limit::Union{Int,Nothing}=nothing,
137 : bbox::Union{NTuple{4,Real},String,Nothing}=nothing, verbose::Bool=true)
138 44 : if !haskey(all_datasets, dataset)
139 4 : error("Unknown dataset: $dataset. Use `$module_name.datasets()` to list available datasets.")
140 : end
141 :
142 18 : d = all_datasets[dataset]
143 18 : url = query_url(d; where=where, outfields=fields, limit=limit, bbox=bbox)
144 :
145 18 : verbose && println("Downloading: $(d.name)")
146 18 : verbose && println("URL: $url")
147 :
148 18 : response = HTTP.get(url; status_exception=false, connect_timeout=60, readtimeout=60)
149 :
150 18 : if response.status != 200
151 0 : error("Failed to download dataset. HTTP status: $(response.status)\nResponse: $(String(response.body))")
152 : end
153 :
154 18 : verbose && println("Parsing GeoJSON...")
155 18 : body = String(response.body)
156 :
157 : # Check for ArcGIS error response (avoid full double-parse)
158 36 : if contains(body, "\"error\"")
159 0 : json_data = JSON3.read(body)
160 0 : if haskey(json_data, :error)
161 0 : error("ArcGIS API error: $(json_data.error)")
162 : end
163 : end
164 :
165 : # Parse as GeoJSON
166 18 : data = GeoJSON.read(body)
167 :
168 18 : n = length(data)
169 18 : verbose && println("Downloaded $n features")
170 36 : if contains(body, "exceededTransferLimit") && contains(body, "true")
171 18 : verbose && println("⚠ Warning: Transfer limit exceeded. Use `limit` parameter or refine `where` clause to get all data.")
172 : end
173 :
174 18 : return data
175 : end
176 :
177 : """
178 : _download_file(all_datasets, dataset, module_name, data_dir; kwargs...)
179 :
180 : Download a dataset and save it to the local data directory. Used internally by submodules.
181 : """
182 8 : function _download_file(all_datasets::Dict{Symbol,<:AbstractDataset}, dataset::Symbol, module_name::String, data_dir::String;
183 : filename::Union{String,Nothing}=nothing, force::Bool=false, verbose::Bool=true, kwargs...)
184 8 : if !haskey(all_datasets, dataset)
185 0 : error("Unknown dataset: $dataset. Use `$module_name.datasets()` to list available datasets.")
186 : end
187 :
188 4 : mkpath(data_dir)
189 :
190 4 : if isnothing(filename)
191 4 : filename = string(dataset) * ".geojson"
192 : end
193 4 : filepath = joinpath(data_dir, filename)
194 :
195 4 : if isfile(filepath) && !force
196 0 : verbose && println("File already exists: $filepath")
197 0 : verbose && println("Use `force=true` to overwrite.")
198 0 : return filepath
199 : end
200 :
201 4 : data = _download(all_datasets, dataset, module_name; verbose=verbose, kwargs...)
202 :
203 8 : open(filepath, "w") do io
204 4 : JSON3.write(io, data)
205 : end
206 4 : verbose && println("Saved to: $filepath")
207 :
208 4 : return filepath
209 : end
210 :
211 : """
212 : _load_file(data_dir, dataset; filename=nothing)
213 :
214 : Load a previously downloaded dataset from the local data directory. Used internally by submodules.
215 : """
216 16 : function _load_file(data_dir::String, dataset::Symbol; filename::Union{String,Nothing}=nothing)
217 8 : if isnothing(filename)
218 8 : filename = string(dataset) * ".geojson"
219 : end
220 8 : filepath = joinpath(data_dir, filename)
221 :
222 8 : if !isfile(filepath)
223 4 : error("File not found: $filepath. Download the dataset first.")
224 : end
225 :
226 4 : return GeoJSON.read(read(filepath, String))
227 : end
228 :
229 : """
230 : _count(all_datasets::Dict{Symbol,<:AbstractDataset}, dataset::Symbol, module_name::String; where="1=1")
231 :
232 : Get the count of features in a dataset. Used internally by submodules.
233 : """
234 24 : function _count(all_datasets::Dict{Symbol,<:AbstractDataset}, dataset::Symbol, module_name::String; where::String="1=1")
235 24 : if !haskey(all_datasets, dataset)
236 4 : error("Unknown dataset: $dataset. Use `$module_name.datasets()` to list available datasets.")
237 : end
238 :
239 8 : d = all_datasets[dataset]
240 8 : params = [
241 : "where" => HTTP.escapeuri(where),
242 : "returnCountOnly" => "true",
243 : "f" => "json",
244 : ]
245 16 : full_url = base_query_url(d) * "?" * join(["$k=$v" for (k, v) in params], "&")
246 :
247 8 : response = HTTP.get(full_url; status_exception=false, connect_timeout=60, readtimeout=60)
248 8 : if response.status != 200
249 0 : error("Failed to get count. HTTP status: $(response.status)")
250 : end
251 :
252 8 : data = JSON3.read(response.body)
253 8 : return data.count
254 : end
255 :
256 : """
257 : _fields(all_datasets::Dict{Symbol,<:AbstractDataset}, dataset::Symbol, module_name::String)
258 :
259 : Get the field names and types for a dataset. Used internally by submodules.
260 : """
261 9 : function _fields(all_datasets::Dict{Symbol,<:AbstractDataset}, dataset::Symbol, module_name::String)
262 18 : if !haskey(all_datasets, dataset)
263 4 : error("Unknown dataset: $dataset. Use `$module_name.datasets()` to list available datasets.")
264 : end
265 :
266 5 : d = all_datasets[dataset]
267 5 : url = base_layer_url(d) * "?f=json"
268 :
269 5 : response = HTTP.get(url; status_exception=false, connect_timeout=60, readtimeout=60)
270 5 : if response.status != 200
271 0 : error("Failed to get field info. HTTP status: $(response.status)")
272 : end
273 :
274 5 : data = JSON3.read(response.body)
275 :
276 5 : if haskey(data, :fields)
277 5 : return [(name=f.name, type=f.type, alias=get(f, :alias, f.name)) for f in data.fields]
278 : else
279 0 : error("Could not retrieve field information for this dataset.")
280 : end
281 : end
282 :
283 : #-----------------------------------------------------------------------------# Submodules
284 : include("WFIGS.jl")
285 : include("IRWIN.jl")
286 : include("FPA_FOD.jl")
287 : include("MTBS.jl")
288 : include("FIRMS.jl")
289 : include("LANDFIRE.jl")
290 : include("FEDS.jl")
291 : include("CWFIS.jl")
292 : include("HMS.jl")
293 : include("GWIS.jl")
294 : include("EGP.jl")
295 : include("RxCADRE.jl")
296 :
297 : end
|
