Coverage

89.9
388
24262
39

lib/avy.ex

100.0
0
0
0
Line Hits Source
0 defmodule Avy do
1 @moduledoc """
2 Avy keeps the contexts that define your domain
3 and business logic.
4
5 Contexts are also responsible for managing your data, regardless
6 if it comes from the database, an external API or others.
7 """
8 end

lib/avy/api_spec.ex

0.0
9
0
9
Line Hits Source
0 defmodule Avy.ApiSpec do
1 alias OpenApiSpex.Parameter
2 alias OpenApiSpex.RequestBody
3
4 alias OpenApiSpex.{
5 Components,
6 Info,
7 OpenApi,
8 Paths,
9 Server,
10 Response,
11 MediaType,
12 Schema,
13 RequestBody,
14 Example,
15 Reference
16 }
17
18 alias AvyWeb.{Endpoint, Router}
19 @behaviour OpenApi
20
21 # References are not supported for RequestBody. Let's define it here
22 # https://github.com/open-api-spex/open_api_spex/issues/692
23 @request_body_spec %RequestBody{
24 description: """
25 all parameters must be submitted as part of the POST body. The quality,
26 minimumlength and longestonly parameters should be specified as key=value pairs on separate lines
27 and the simple-time and channel constraints parameters repeated as many times as necessary following
28 this pattern:
29 ````
30 quality=M
31 NET STA LOC CHA STARTTIME ENDTIME
32 NET STA LOC CHA STARTTIME ENDTIME
33 NET STA LOC CHA STARTTIME ENDTIME
34 ````
35 All rules for parameters apply equally whether specified using the GET or POST methods with the
36 exception of blank location IDs, which must be specified as “--“ in the POST body due to spaces being
37 used as the field separato
38 """,
39 required: true,
40 content: %{
41 "plain/text" => %MediaType{
42 schema: %Schema{type: :string},
43 example: """
44 quality=B
45 FR CIEL 00 HHZ 2025-11-20T12:00:00 2025-11-20T12:01:00
46 RA UNIO 00 HNZ 2025-11-20T12:00:00 2025-11-20T12:01:00
47 """
48 }
49 }
50 }
51 @responses_spec [
52 ok: %Reference{"$ref": "#/components/responses/success"},
53 no_content: %Reference{"$ref": "#/components/responses/nodata"},
54 not_found: %Reference{"$ref": "#/components/responses/nodata"},
55 bad_request: %Reference{"$ref": "#/components/responses/bad_request"},
56 request_entity_too_large: %Reference{"$ref": "#/components/responses/too_much_data"}
57 ]
58 @parameters_spec [
59 %Reference{"$ref": "#/components/parameters/network"},
60 %Reference{"$ref": "#/components/parameters/station"},
61 %Reference{"$ref": "#/components/parameters/location"},
62 %Reference{"$ref": "#/components/parameters/channel"},
63 %Reference{"$ref": "#/components/parameters/starttime"},
64 %Reference{"$ref": "#/components/parameters/endtime"},
65 %Reference{"$ref": "#/components/parameters/nodata"},
66 %Reference{"$ref": "#/components/parameters/format"},
67 %Reference{"$ref": "#/components/parameters/quality"},
68 %Reference{"$ref": "#/components/parameters/merge"},
69 %Reference{"$ref": "#/components/parameters/orderby"},
70 %Reference{"$ref": "#/components/parameters/includerestricted"}
71 ]
72 0 def request_body_spec, do: @request_body_spec
73 0 def responses_spec, do: @responses_spec
74 0 def parameters_spec, do: @parameters_spec
75
76 def query_parameters_spec,
77 do:
78 0 @parameters_spec ++
79 [
80 %Reference{"$ref": "#/components/parameters/mergegaps"},
81 %Reference{"$ref": "#/components/parameters/show"},
82 %Reference{"$ref": "#/components/parameters/showgaps"}
83 ]
84
85 @impl OpenApi
86 def spec do
87 %OpenApi{
88 servers: [
89 openapi_server_from_endpoint(Endpoint)
90 ],
91 info: %Info{
92 title: "FDSNWS availability - OpenAPI 3.0",
93 version: "1.0"
94 },
95 # Populate the paths from a phoenix router
96 paths: Paths.from_router(Router),
97 components: %Components{
98 responses: %{
99 success: %Response{
100 description: "successful request",
101 content: %{
102 "text/plain" => %MediaType{schema: %Schema{type: :string}},
103 "application/json" => %MediaType{schema: %Schema{type: :string}},
104 "text/csv" => %MediaType{schema: %Schema{type: :string}}
105 }
106 },
107 nodata: %Response{
108 description: "no data"
109 },
110 bad_request: %Response{
111 description: "Invalid parameters in query string."
112 },
113 too_much_data: %Response{
114 description:
115 "Too much data requested. Try to split your request with narrower criteria"
116 }
117 },
118 requestBodies: %{query: @request_body_spec},
119 parameters: %{
120 network: %Parameter{
121 name: "network",
122 in: :query,
123 required: false,
124 explode: false,
125 description: """
126 Select one or more network codes. Can be SEED network codes or data center defined codes.
127 Multiple codes are comma-separated. Shortcut `net` can also be used.
128 Wildcards are supported (`_` means any character, once; `%` means any character 0 or several times.)
129 """,
130 schema: %Schema{
131 type: :array,
132 items: %Schema{type: :string, example: "FR"}
133 },
134 examples: %{
135 FR: %Example{value: "FR", description: "Select the FR network code"},
136 "FR,RA": %Example{
137 value: ["FR", "RA"],
138 description: "Select the FR and RA network codes"
139 },
140 "X%": %Example{
141 value: "X%",
142 description: "Select all network codes starting with X"
143 }
144 }
145 },
146 station: %Parameter{
147 name: "station",
148 in: :query,
149 required: false,
150 explode: false,
151 description: """
152 Select one or more SEED station codes.
153 Multiple codes are comma-separated.
154 Shortcut `sta` can also be used.
155 Wildcards are supported (`_` means any character, once; `%` means any character 0 or several times.)
156 """,
157 schema: %Schema{
158 type: :array,
159 items: %Schema{type: :string, example: "CIEL"}
160 },
161 examples: %{
162 CIEL: %Example{value: "CIEL", description: "Select the CIEL sation code"},
163 "CIEL,ILLK": %Example{
164 value: ["CIEL", "ILLK"],
165 description: "Select two station codes"
166 },
167 "Z%": %Example{
168 value: "Z%",
169 description: "Select all station codes starting with Z"
170 }
171 }
172 },
173 location: %Parameter{
174 name: "location",
175 in: :query,
176 required: false,
177 explode: false,
178 description: """
179 Select one or more SEED location codes.
180 Empty location code can be specified using `--`.
181 Multiple codes are comma-separated.
182 Shortcut `loc` can also be used.
183 Wildcards are supported (`_` means any character, once; `%` means any character 0 or several times.)
184 """,
185 schema: %Schema{
186 type: :array,
187 items: %Schema{type: :string, example: "00"}
188 },
189 examples: %{
190 "00": %Example{value: "00", description: "Select the 00 location code"},
191 "00,--": %Example{
192 value: ["00", "--"],
193 description: "Select location codes 00 and empty"
194 }
195 }
196 },
197 channel: %Parameter{
198 name: "channel",
199 in: :query,
200 required: false,
201 explode: false,
202 description: """
203 Select one or more SEED channel codes.
204 Multiple codes are comma-separated.
205 Shortcut `cha` can also be used.
206 Wildcards are supported (`_` means any character, once; `%` means any character 0 or several times.)
207 """,
208 schema: %Schema{
209 type: :array,
210 items: %Schema{type: :string, example: "HHZ"}
211 },
212 examples: %{
213 HHZ: %Example{value: "HHZ", description: "Select the HNE channel code"},
214 "HHZ,BHZ": %Example{value: ["HHZ", "BHZ"], description: "Select two channel codes"},
215 "%Z": %Example{description: "Select all channel ending with Z"}
216 }
217 },
218 starttime: %Parameter{
219 name: "starttime",
220 in: :query,
221 required: false,
222 explode: false,
223 description: """
224 Limit results to time series samples on or after the specified start time. If not specified, timezone is assumed as UTC. Shortcut `start`
225 """,
226 schema: %Schema{
227 type: :string,
228 format: "date-time"
229 },
230 examples: %{
231 "2025-11-20T12:38:04.000000Z": %Example{
232 value: "2025-11-20T12:38:04.000000Z",
233 description: "microseconds and timezone"
234 },
235 "2025-11-20T12:38:04Z": %Example{
236 value: "2025-11-20T12:38:04Z",
237 description: "seconds and timezone"
238 },
239 "2025-11-20T12:38:04": %Example{
240 value: "2025-11-20T12:38:04",
241 description: "seconds without timezone (UTC is assumed)"
242 },
243 "2025-11-20": %Example{
244 value: "2025-11-20",
245 description: "Without time, 00:00:00.000000Z is assumed"
246 }
247 }
248 },
249 endtime: %Parameter{
250 name: "endtime",
251 in: :query,
252 required: false,
253 explode: false,
254 description:
255 "Limit results to time series samples on or after the specified end time. If not specified, timezone is assumed as UTC. Shortcut `end`",
256 schema: %Schema{
257 type: :string,
258 format: "date-time"
259 },
260 examples: %{
261 "2025-11-20T12:38:14.000000Z": %Example{
262 value: "2025-11-20T12:38:14.000000Z",
263 description: "microseconds and timezone"
264 },
265 "2025-11-20T12:38:14Z": %Example{
266 value: "2025-11-20T12:38:14Z",
267 description: "seconds and timezone"
268 },
269 "2025-11-20T12:38:14": %Example{
270 value: "2025-11-20T12:38:14",
271 description: "seconds without timezone (UTC is assumed)"
272 },
273 "2025-11-21": %Example{
274 value: "2025-11-21",
275 description: "Without time, 00:00:00.000000Z is assumed"
276 }
277 }
278 },
279 format: %Parameter{
280 name: "format",
281 in: :query,
282 required: false,
283 description: "Specify format of result, the default value is text.",
284 schema: %Schema{
285 type: :string,
286 default: "text",
287 enum: ["text", "geocsv", "json", "request"]
288 }
289 },
290 nodata: %Parameter{
291 name: "nodata",
292 in: :query,
293 required: false,
294 description: "Specify the return code for no data. Valid values are 204 or 404",
295 schema: %Schema{
296 type: :integer,
297 default: 204,
298 enum: [204, 404]
299 }
300 },
301 quality: %Parameter{
302 name: "quality",
303 in: :query,
304 required: false,
305 explode: false,
306 description: """
307 Select a specific SEED quality indicator.
308 * R -> Raw
309 * Q -> Quality controlled
310 * M -> Modified
311 * D -> Data, unspecified
312 """,
313 schema: %Schema{
314 type: :array,
315 items: %Schema{type: :string, example: "M", enum: ["R", "Q", "M", "D"]},
316 default: "M"
317 }
318 },
319 merge: %Parameter{
320 name: "merge",
321 in: :query,
322 required: false,
323 explode: false,
324 description: """
325 If set to one or more of the following values, time spans are merged as described. Multiple values may be specified as a
326 comma-delimited list, e.g. ``merge=samplerate,quality``.
327 * `samplerate`: time spans from data with differing sample rates will be grouped together. If specified this field will be
328 omitted from the result.
329 * `quality`: time spans from data with differing quality codes will be grouped together. If specified this field will be omitted
330 from the result.
331 * `overlap`: time spans from data that overlap will be merged together. This option does not apply to the extent method
332 """,
333 schema: %Schema{
334 type: :array,
335 items: %Schema{
336 type: :string,
337 example: "samplerate",
338 enum: ["quality", "samplerate", "overlap"]
339 },
340 default: ""
341 }
342 },
343 includerestricted: %Parameter{
344 name: "includerestricted",
345 in: :query,
346 required: false,
347 description:
348 "If true, all data are reported. If false, only data that can be openly accessed.",
349 schema: %Schema{
350 type: :string,
351 default: "TRUE",
352 enum: ["TRUE", "FALSE"]
353 }
354 },
355 orderby: %Parameter{
356 name: "orderby",
357 in: :query,
358 required: false,
359 description: """
360 Sort results by one of the following values in the order specified:
361 * `nslc_time_quality_samplerate`: network, station, location, channel, time-range, quality, sample-rate (default)
362 * `latestupdate`: update-date (past to present), network, station, location, channel, time-range, quality, sample-rate
363 * `latestupdate_desc`: update-date (present to past), network, station, location, channel, time-range, quality, sample-rate
364 * `timespancount`: number of timespans (small to large), network, station, location, channel, time-range, quality, sample-rate
365 * `timespancount_desc`: number of timespans (large to small), network, station, location, channel, time-range, quality, sample-rate
366 """,
367 schema: %Schema{
368 type: :strung,
369 example: "latestupdate_desc",
370 enum: [
371 "nslc_time_quality_samplerate",
372 "latestupdate",
373 "latestupdate_desc",
374 "timaspancount",
375 "timespancount_desc"
376 ],
377 default: "latestupdate"
378 }
379 },
380 mergegaps: %Parameter{
381 name: "mergegaps",
382 in: :query,
383 required: false,
384 description:
385 "Merge time spans that are separated by the specified tolerance in seconds. Only positive values are allowed.",
386 schema: %Schema{
387 type: :number,
388 minimum: 0,
389 example: 10.5,
390 default: 0.0
391 }
392 },
393 show: %Parameter{
394 name: "show",
395 in: :query,
396 required: false,
397 description: """
398 If set to `latestupdate`, the latest times at which data contributing to the returned time spans were loaded into the repository are included in the result.
399 This option applies to all formats except `request`.
400 """,
401 schema: %Schema{
402 type: :string,
403 example: "latestupdate",
404 enum: ["latestupdate", ""],
405 default: ""
406 }
407 },
408 showgaps: %Parameter{
409 name: "showgaps",
410 in: :query,
411 required: false,
412 description: """
413 If present, instead of showing timespans of available data, the result shows timespans of missing data.
414 This parameter is not in the FDSN specification and has been added for convenience.
415 """,
416 schema: %Schema{
417 type: :boolean,
418 nullable: true
419 }
420 }
421 }
422 }
423 }
424 # Discover request/response schemas from path specs
425 0 |> OpenApiSpex.resolve_schema_modules()
426 end
427
428 defp openapi_server_from_endpoint(endpoint) do
429 0 uri = endpoint.struct_url()
430 0 path = Application.get_env(:avy, :url_prefix)
431 0 uri = %{uri | path: path}
432
433 0 %Server{
434 url: URI.to_string(uri)
435 }
436 end
437 end

lib/avy/application.ex

71.4
7
5
2
Line Hits Source
0 defmodule Avy.Application do
1 # See https://hexdocs.pm/elixir/Application.html
2 # for more information on OTP Applications
3 @moduledoc false
4
5 use Application
6 # require OpentelemetryEcto
7 # require OpentelemetryBandit
8 # require OpentelemetryPhoenix
9
10 @impl true
11 def start(_type, _args) do
12 1 :logger.add_handler(:my_sentry_handler, Sentry.LoggerHandler, %{
13 config: %{metadata: [:file, :line]}
14 })
15
16 # Set up LiveView trace propagation (must be BEFORE OpentelemetryPhoenix)
17 # Sentry.OpenTelemetry.LiveViewPropagator.setup()
18
19 1 children =
20 [
21 AvyWeb.Telemetry,
22 Avy.Repo,
23 1 {DNSCluster, query: Application.get_env(:avy, :dns_cluster_query) || :ignore},
24 {Phoenix.PubSub, name: Avy.PubSub},
25
26 # Start the Finch HTTP client for sending emails
27 {Finch, name: Avy.Finch},
28 # Start a worker by calling: Avy.Worker.start_link(arg)
29 # {Avy.Worker, arg},
30 # Start to serve requests, typically the last entry
31 AvyWeb.Endpoint
32 ] ++ FdsnPlugs.JwkConfig.child_specs()
33
34 # See https://hexdocs.pm/elixir/Supervisor.html
35 # for other strategies and supported options
36 1 opts = [strategy: :one_for_one, name: Avy.Supervisor]
37 1 Supervisor.start_link(children, opts)
38 end
39
40 # Tell Phoenix to update the endpoint configuration
41 # whenever the application is updated.
42 @impl true
43 0 def config_change(changed, _new, removed) do
44 0 AvyWeb.Endpoint.config_change(changed, removed)
45 :ok
46 end
47 end

lib/avy/data/content.ex

100.0
1
2960
0
Line Hits Source
0 defmodule Avy.Data.Content do
1 use Ecto.Schema
2
3 2960 schema "inventory_content" do
4 field :start, :utc_datetime, source: :start
5 field :end, :utc_datetime, source: :end
6 field :quality, :integer
7 field :time_ordered, :boolean, default: false
8 field :time_spans, {:array, :map}, source: :time_spans
9 field :modified, :utc_datetime, source: :modified
10
11 many_to_many :epochs, Avy.Metadata.Epoch,
12 join_through: "inventory_content_epochs",
13 preload_order: [asc: :starttime]
14 end
15 end

lib/avy/data/content_epoch.ex

0.0
1
0
1
Line Hits Source
0 defmodule Avy.Data.ContentEpoch do
1 use Ecto.Schema
2 alias Avy.Data.Content
3 alias Avy.Metadata.Epoch
4
5 # Important: Set primary_key to false for junction tables
6 @primary_key false
7 0 schema "inventory_content_epochs" do
8 belongs_to :content, Content, primary_key: true
9 belongs_to :epoch, Epoch, primary_key: true
10 end
11 end

lib/avy/datasource.ex

100.0
10
1293
0
Line Hits Source
0 defmodule Avy.Datasource do
1 @moduledoc """
2 A Datasource is the structure that correspond to the output of the webservice.
3 It is generated by a request in Avy.Repo and manipulated after that
4 in order to produce the expected output
5 """
6 require Logger
7
8 @derive {Jason.Encoder,
9 only: [
10 :network,
11 :station,
12 :location,
13 :channel,
14 :quality,
15 :samplerate,
16 :earliest,
17 :latest,
18 :timespans,
19 :restriction,
20 :timespancount,
21 :updated
22 ]}
23 defstruct network: "",
24 station: "",
25 location: "",
26 channel: "",
27 quality: "",
28 samplerate: "",
29 earliest: DateTime.from_unix!(1),
30 latest: DateTime.from_unix!(5_682_956_400),
31 timespans: [],
32 restriction: "O",
33 timespancount: 0,
34 updated: DateTime.from_unix!(1)
35
36 @type t() :: %Avy.Datasource{
37 network: String.t(),
38 station: String.t(),
39 location: String.t(),
40 channel: String.t(),
41 quality: String.t(),
42 samplerate: float(),
43 earliest: DateTime.t(),
44 latest: DateTime.t(),
45 timespans: list(map),
46 restriction: String.t(),
47 timespancount: integer(),
48 updated: DateTime.t()
49 }
50
51 @doc """
52 For a given content, remove timespans outside of scope and trim the one that intersect
53 """
54 @spec trim_timespans(t()) :: t()
55 def trim_timespans(ds) do
56 104 Logger.debug("Trimming timespans for ds #{inspect(ds, pretty: true)}")
57 104 earliest_ns = DateTime.to_unix(ds.earliest, :nanosecond)
58 104 latest_ns = DateTime.to_unix(ds.latest, :nanosecond)
59
60 %{
61 ds
62 104 | timespans:
63 104 ds.timespans
64 156 |> Enum.map(&trim_timespan(&1, earliest_ns, latest_ns))
65 156 |> Enum.reject(&is_nil(&1))
66 }
67 end
68
69 # If the timespan intersects the earliest and latest boundary
70 # Trim the timespan to the boundaries
71 # Else return nil
72 defp trim_timespan(ts, earliest, latest) do
73 156 Logger.debug("Trimming timespan #{inspect(ts)} over #{earliest} -> #{latest}")
74
75 156 if earliest <= ts["end"] and latest >= ts["start"] do
76 149 %{
77 "samplerate" => ts["samplerate"],
78 "end" => min(ts["end"], latest),
79 "start" => max(ts["start"], earliest)
80 }
81 else
82 nil
83 end
84 end
85 end

lib/avy/filter.ex

75.0
12
351
3
Line Hits Source
0 defmodule Avy.Filter do
1 alias FdsnPlugs.SourceIdentifier
2 require Logger
3
4 @moduledoc """
5 A filter is a structure containing all filter parameters given in an FDSN webservice request.
6
7 """
8 defstruct net: "*",
9 sta: "*",
10 loc: "*",
11 cha: "*",
12 start: DateTime.from_unix!(1),
13 end: DateTime.from_unix!(5_682_956_400),
14 quality: [:R, :D, :M, :Q],
15 includerestricted: false,
16 epochids: []
17
18 @type t() :: %Avy.Filter{
19 net: String.t(),
20 sta: String.t(),
21 loc: String.t(),
22 cha: String.t(),
23 start: DateTime.t(),
24 end: DateTime.t(),
25 quality: String.t(),
26 includerestricted: boolean,
27 epochids: list
28 }
29
30 @spec split_channel(t()) :: map()
31 def split_channel(f) do
32 39 case SourceIdentifier.split_channel(%SourceIdentifier{cha: f.cha}) do
33 {:ok, [b, i, o]} ->
34 39 %{band_code: b, instrument_code: i, orientation_code: o}
35
36 {:error, m} ->
37 0 Logger.error(inspect(m))
38 0 %{band_code: "*", instrument_code: "*", orientation_code: "*"}
39
40 _ ->
41 0 %{}
42 end
43 end
44
45 @spec from_source_identifier(SourceIdentifier.t(), boolean, list) :: t()
46 def from_source_identifier(sid, includerestricted \\ false, quality \\ [:R, :D, :M, :Q]) do
47 39 %Avy.Filter{
48 39 net: sid.net,
49 39 sta: sid.sta,
50 39 loc: sid.loc,
51 39 cha: sid.cha,
52 39 start: sid.start,
53 39 end: sid.end,
54 includerestricted: includerestricted,
55 quality: quality
56 }
57 end
58 end

lib/avy/mailer.ex

100.0
0
0
0
Line Hits Source
0 defmodule Avy.Mailer do
1 use Swoosh.Mailer, otp_app: :avy
2 end

lib/avy/metadata/epoch.ex

100.0
3
5148
0
Line Hits Source
0 defmodule Avy.Metadata.Epoch do
1 use Ecto.Schema
2
3 5146 schema "inventory_epoch" do
4 field :band_code, :string
5 field :instrument_code, :string
6 field :orientation_code, :string
7 field :location_code, :string, source: :location_code
8 field :start_date, :utc_datetime, source: :start_date
9 field :end_date, :utc_datetime, source: :end_date
10 field :policy, :string, default: "O"
11 field :sample_rate, :decimal, source: :sample_rate
12 field :modified, :utc_datetime, source: :modified
13 belongs_to :station, Avy.Metadata.Station
14
15 many_to_many :contents, Avy.Data.Content,
16 join_through: "inventory_content_epochs",
17 preload_order: [asc: :starttime]
18 end
19
20 def source_identifier(c) do
21 1 c = Avy.Repo.preload(c, station: [:network])
22
23 1 "#{c.station.network.code}_#{c.station.code}_#{c.location_code}_#{c.band_code}_#{c.instrument_code}_#{c.orientation_code}"
24 end
25 end

lib/avy/metadata/network.ex

100.0
1
3267
0
Line Hits Source
0 defmodule Avy.Metadata.Network do
1 use Ecto.Schema
2
3 3267 schema "inventory_network" do
4 field :code, :string
5 field :start_date, :utc_datetime, source: :start_date
6 field :end_date, :utc_datetime, source: :end_date
7 has_many :stations, Avy.Metadata.Station
8 end
9 end

lib/avy/metadata/station.ex

100.0
1
3308
0
Line Hits Source
0 defmodule Avy.Metadata.Station do
1 use Ecto.Schema
2
3 3308 schema "inventory_station" do
4 field :code, :string
5 field :start_date, :utc_datetime, source: :start_date
6 field :end_date, :utc_datetime, source: :end_date
7 belongs_to :network, Avy.Metadata.Network
8
9 has_many :epochs, Avy.Metadata.Epoch,
10 preload_order: [:band_code, :instrument_code, :orientation_cde, :starttime]
11 end
12 end

lib/avy/repo.ex

97.2
36
1564
1
Line Hits Source
0 defmodule Avy.Repo do
1 use Ecto.Repo,
2 otp_app: :avy,
3 adapter: Ecto.Adapters.Postgres
4
5 import Ecto.Query
6
7 alias FdsnPlugs.PublicationVersion
8 alias Avy.Metadata.{Epoch, Station, Network}
9 alias Avy.Data.Content
10 alias Avy.Datasource
11
12 # Create a map of all keyword parameters.
13 # Especially, splits the channel code from filter into instrument, band, orientation codes.
14 @spec make_keywords_params(Avy.Filter.t()) :: map()
15 defp make_keywords_params(filter) do
16 Avy.Filter.split_channel(filter)
17 39 |> Map.merge(%{
18 39 net: filter.net,
19 39 sta: filter.sta,
20 39 loc: filter.loc,
21 39 includerestricted: filter.includerestricted
22 })
23 end
24
25 # NOTE: the call to split_channels seems to bother dialyzer. I was not able to fix this.
26 @doc """
27 Get all contents from database related to the Avy.Filter
28 """
29 @spec get_contents(Avy.Filter.t()) :: [Datasource.t()]
30 def get_contents(filter) do
31 39 query =
32 from(c in Content)
33 39 |> join(:inner, [c], ec in "inventory_content_epochs",
34 on: c.id == ec.content_id,
35 as: :content_epochs
36 )
37 39 |> join(:inner, [c, ec], e in Epoch, on: ec.epoch_id == e.id, as: :epochs)
38 39 |> join(:inner, [c, ec, e], s in Station, on: e.station_id == s.id, as: :stations)
39 39 |> join(:inner, [c, ec, e, s], n in Network, on: s.network_id == n.id, as: :networks)
40 |> where(^conditional_where_clause(make_keywords_params(filter)))
41 |> where(
42 [c, ec, e, s, n],
43 39 c.quality in ^PublicationVersion.qualities_to_pubversions(filter.quality)
44 )
45 39 |> where([c, ec, e, s, n], c.start <= ^filter.end)
46 39 |> where([c, ec, e, s, n], c.end >= ^filter.start)
47 39 |> where([c, ec, e, s], e.start_date <= ^filter.end)
48 39 |> where([c, ec, e, s], is_nil(e.end_date) or e.end_date >= ^filter.start)
49
50 39 query =
51 39 from [c, ec, e, s, n] in query,
52 order_by: [
53 asc: e.band_code,
54 asc: e.instrument_code,
55 asc: e.orientation_code,
56 asc: e.start_date,
57 asc: c.start
58 ],
59 select: %Datasource{
60 network: n.code,
61 station: s.code,
62 location: e.location_code,
63 channel: fragment("?||?||?", e.band_code, e.instrument_code, e.orientation_code),
64 quality: c.quality,
65 samplerate: e.sample_rate,
66 earliest:
67 39 fragment("GREATEST(?, ?, ?) AT TIME ZONE 'utc'", e.start_date, c.start, ^filter.start),
68 39 latest: fragment("LEAST(?,?, ?) AT TIME ZONE 'utc'", e.end_date, c.end, ^filter.end),
69 timespans: c.time_spans,
70 restriction: e.policy,
71 updated: c.modified
72 }
73
74 Avy.Repo.all(query)
75 104 |> Enum.reject(fn t -> is_nil(t) end)
76 39 |> Enum.map(&naive_to_utc(&1))
77 end
78
79 # Define all dynamic where clauses.
80 # If a parameter is a wildcard, we do not need to add the where clause.
81 @spec conditional_where_clause(map()) :: any()
82 def conditional_where_clause(keyword_params) do
83 39 Enum.reduce(keyword_params, dynamic(true), fn
84 {_k, "*"}, dynamic ->
85 # User wants all, no where clause
86 178 dynamic
87
88 {:includerestricted, true}, dynamic ->
89 2 dynamic
90
91 {:includerestricted, false}, dynamic ->
92 37 dynamic([epochs: e], ^dynamic and e.policy == "O")
93
94 {:instrument_code, value}, dynamic ->
95 4 dynamic([epochs: e], ^dynamic and like(e.instrument_code, ^fdsn_wildcard_to_sql(value)))
96
97 {:band_code, value}, dynamic ->
98 4 dynamic([epochs: e], ^dynamic and like(e.band_code, ^fdsn_wildcard_to_sql(value)))
99
100 {:orientation_code, value}, dynamic ->
101 4 dynamic([epochs: e], ^dynamic and like(e.orientation_code, ^fdsn_wildcard_to_sql(value)))
102
103 {:loc, value}, dynamic ->
104 1 dynamic([epochs: e], ^dynamic and like(e.location_code, ^fdsn_wildcard_to_sql(value)))
105
106 {:sta, value}, dynamic ->
107 37 dynamic([stations: s], ^dynamic and like(s.code, ^fdsn_wildcard_to_sql(value)))
108
109 {:net, value}, dynamic ->
110 6 dynamic([networks: n], ^dynamic and like(n.code, ^fdsn_wildcard_to_sql(value)))
111 end)
112 end
113
114 #
115 # From FDSN wildcard spec to SQL wildcards
116 @spec fdsn_wildcard_to_sql(String.t()) :: String.t()
117 defp fdsn_wildcard_to_sql(s) do
118 String.replace(s, "*", "%")
119 56 |> String.replace("?", "_")
120 end
121
122 @spec naive_to_utc(map) :: map
123 defp naive_to_utc(q) do
124 104 if Map.has_key?(q.earliest, :time_zone) do
125 0 q
126 else
127 %{
128 q
129 104 | earliest: DateTime.from_naive!(q.earliest, "Etc/UTC"),
130 104 latest: DateTime.from_naive!(q.latest, "Etc/UTC")
131 }
132 end
133 end
134 end

lib/avy_web.ex

50.0
6
24
3
Line Hits Source
0 defmodule AvyWeb do
1 @moduledoc """
2 The entrypoint for defining your web interface, such
3 as controllers, components, channels, and so on.
4
5 This can be used in your application as:
6
7 use AvyWeb, :controller
8 use AvyWeb, :html
9
10 The definitions below will be executed for every controller,
11 component, etc, so keep them short and clean, focused
12 on imports, uses and aliases.
13
14 Do NOT define functions inside the quoted expressions
15 below. Instead, define additional modules and import
16 those modules here.
17 """
18
19 8 def static_paths,
20 do: ~w(assets fonts images favicon.ico robots.txt application.wadl)
21
22 def router do
23 0 quote do
24 use Phoenix.Router, helpers: false
25
26 # Import common connection and controller functions to use in pipelines
27 import Plug.Conn
28 import Phoenix.Controller
29 end
30 end
31
32 def channel do
33 0 quote do
34 use Phoenix.Channel
35 end
36 end
37
38 def controller do
39 0 quote do
40 use Phoenix.Controller,
41 formats: [:text, :json, :geocsv, :request, :html],
42 layouts: [html: AvyWeb.Layouts]
43
44 import Plug.Conn
45
46 unquote(verified_routes())
47 end
48 end
49
50 def verified_routes do
51 8 quote do
52 use Phoenix.VerifiedRoutes,
53 endpoint: AvyWeb.Endpoint,
54 router: AvyWeb.Router,
55 statics: AvyWeb.static_paths()
56 end
57 end
58
59 @doc """
60 When used, dispatch to the appropriate controller/live_view/etc.
61 """
62 defmacro __using__(which) when is_atom(which) do
63 8 apply(__MODULE__, which, [])
64 end
65 end

lib/avy_web/avy_geoscv.ex

97.6
43
640
1
Line Hits Source
0 defmodule AvyWeb.AvyGEOCSV do
1 require Logger
2
3 @extent_header "#dataset: GeoCSV 2.0
4 #delimiter: |
5 #field_unit: unitless|unitless|unitless|unitless|unitless|hertz|ISO_8601|ISO_8601|ISO_8601|unitless|unitless
6 #field_type: string|string|string|string|string|float|datetime|datetime|datetime|integer|string
7 Network|Station|Location|Channel|Quality|SampleRate|Earliest|Latest|Updated|TimeSpans|Restriction"
8
9 0 def extent_post(assigns), do: extent(assigns)
10
11 def extent(assigns) do
12 4 Logger.debug(assigns.availabilities, pretty: true)
13
14 [
15 format_header(@extent_header, assigns)
16 4 | Enum.map(assigns.availabilities, fn d ->
17 10 format_datasource(d, assigns.fdsn_parameters, :extent)
18 end)
19 ]
20 4 |> Enum.join("\n")
21 end
22
23 @query_header "#dataset: GeoCSV 2.0
24 #delimiter: |
25 #field_unit: unitless|unitless|unitless|unitless|unitless|hertz|ISO_8601|ISO_8601|ISO_8601
26 #field_type: string|string|string|string|string|float|datetime|datetime|datetime
27 Network|Station|Location|Channel|Quality|SampleRate|Earliest|Latest|Updated"
28 def query(assigns) do
29 [
30 format_header(@query_header, assigns)
31 4 | Enum.map(assigns.availabilities, fn d ->
32 6 Enum.map(d.timespans, fn ts ->
33 22 %{d | timespans: [ts], earliest: List.first(ts), latest: List.last(ts)}
34 end)
35 6 |> Enum.map(fn splitd ->
36 22 format_datasource(
37 splitd,
38 22 assigns.fdsn_parameters,
39 :query
40 )
41 end)
42 end)
43 ]
44 |> List.flatten()
45 4 |> Enum.join("\n")
46 end
47
48 defp format_header(headers, %{fdsn_parameters: fdsn_params}) do
49 8 headers =
50 8 if :samplerate in fdsn_params.merge do
51 String.replace(headers, "hertz|", "")
52 |> String.replace("float|", "")
53 3 |> String.replace("SampleRate|", "")
54 else
55 5 headers
56 end
57
58 8 headers =
59 8 if :quality in fdsn_params.merge do
60 String.replace(headers, "unitless|", "", global: false)
61 |> String.replace("string|", "", global: false)
62 2 |> String.replace("Quality|", "")
63 else
64 6 headers
65 end
66
67 8 if :latestupdate in fdsn_params.show do
68 4 headers
69 else
70 String.replace(headers, "|ISO_8601", "", global: false)
71 |> String.replace("|datetime", "", global: false)
72 4 |> String.replace("|Updated", "")
73 end
74 end
75
76 # Network|Station|Location|Channel|Quality|SampleRate|Earliest|Latest|Updated|TimeSpans|Restriction
77 defp format_datasource(d, fdsn_params, method) do
78 32 attr_list =
79 32 if method == :extent do
80 10 [d.timespancount, d.restriction]
81 else
82 []
83 end
84
85 32 attr_list =
86 32 if :latestupdate in fdsn_params.show do
87 [
88 10 DateTime.to_iso8601(d.earliest),
89 10 DateTime.to_iso8601(d.latest),
90 10 DateTime.to_iso8601(d.updated) | attr_list
91 ]
92 else
93 [
94 22 DateTime.to_iso8601(d.earliest),
95 22 DateTime.to_iso8601(d.latest)
96 | attr_list
97 ]
98 end
99
100 32 attr_list =
101 32 if :samplerate in fdsn_params.merge do
102 14 attr_list
103 else
104 18 [d.samplerate | attr_list]
105 end
106
107 32 attr_list =
108 32 if :quality in fdsn_params.merge do
109 12 attr_list
110 else
111 20 [d.quality | attr_list]
112 end
113
114 32 [d.network, d.station, d.location, d.channel | attr_list]
115 32 |> Enum.join("|")
116 end
117 end

lib/avy_web/avy_json.ex

89.2
28
121
3
Line Hits Source
0 defmodule AvyWeb.AvyJSON do
1 require Logger
2
3 def version(_) do
4 0 %{
5 application: Application.spec(:avy)[:app],
6 version: Application.spec(:avy)[:vsn],
7 commit: System.get_env("SENTRY_RELEASE", "unspecified")
8 }
9 end
10
11 def extent(assigns) do
12 5 %{
13 created: DateTime.utc_now(),
14 version: 1.0,
15 5 datasources: assigns.availabilities
16 }
17 end
18
19 def query(assigns) do
20 7 %{
21 created: DateTime.utc_now(),
22 version: 1.0,
23 7 datasources: assigns.availabilities
24 }
25 end
26
27 @doc """
28 This view is meant to output a json format for grafana "state timeline" graphs.
29 It is very different from the FDSN output and therefore needs a lot of manipulations.
30
31 The final structure is a list of maps with the key :timestamp as a date,
32 and each FDSN source identifiers + quality code in the response as keys, with default value "".
33
34 At each timestamp, we want to put for each FDSN source identifier a quality code or
35 an empty string if there is no data.
36
37 [
38 %{ timeline: ~U[2026-01-01 10:00:00Z],
39 "FR_CIEL_00_HNZ/D" => "D"
40 }
41 }
42 """
43 def statetimeline(assigns) do
44 # First get all FDSN identifiers in the availabilities and build the default map of identifiers.
45 1 fdsn_identifiers =
46 1 assigns.availabilities
47 3 |> Enum.reduce([], fn a, acc ->
48 [availability_to_fdsn_source_identifier(a) | acc]
49 end)
50 |> Enum.sort()
51 |> Enum.dedup()
52 |> Map.from_keys("")
53
54 # Then build a list of timestamps
55 # All elements in the timespans list must result in a new entry in the list
56 # In the same time, we add the correct FDSN identifier and quality in the map.
57 # The list needs to be sorted by DateTime
58 1 timeline_list =
59 1 assigns.availabilities
60 |> Enum.reduce([], fn a, acc ->
61 [
62 3 elem(
63 Enum.reduce(
64 3 a.timespans,
65 3 {availability_to_fdsn_source_identifier(a), a.quality, []},
66 fn [ts_start, ts_end], {fdsnid, qual, acc} ->
67 8 {fdsnid, qual,
68 [
69 %{fdsnid => "", "Timestamps" => ts_end},
70 %{fdsnid => qual, "Timestamps" => ts_start}
71 ] ++ acc}
72 end
73 ),
74 2
75 )
76 3 ] ++ acc
77 end)
78 |> List.flatten()
79 16 |> Enum.sort_by(&Map.fetch!(&1, "Timestamps"), DateTime)
80
81 1 Logger.debug(inspect(timeline_list, pretty: true))
82
83 # Then we need to fill the struct with all other missing fdsn identifiers.
84 # For each timeline, the quality value must be the same as in the previous
85 # timeline. Therefore, we need to know which it was.
86 1 initial_tl = Map.merge(fdsn_identifiers, List.first(timeline_list))
87
88 Enum.reduce(timeline_list, {initial_tl, [initial_tl]}, fn tl, {prev_tl, acc} ->
89 16 full_tl = Map.merge(prev_tl, tl)
90
91 16 if Map.fetch!(tl, "Timestamps") == Map.fetch!(prev_tl, "Timestamps") do
92 {full_tl, List.replace_at(acc, 0, full_tl)}
93 else
94 {full_tl, [full_tl | acc]}
95 end
96 end)
97 1 |> elem(1)
98 1 |> Enum.reverse()
99 end
100
101 defp availability_to_fdsn_source_identifier(a) do
102 6 cha = a.channel |> String.split("", trim: true) |> Enum.join("_")
103 6 "#{a.network}_#{a.station}_#{a.location}_#{cha} #{quality_to_pubversion(a.quality)}"
104 end
105
106 0 defp quality_to_pubversion("R"), do: 2
107 4 defp quality_to_pubversion("Q"), do: 3
108 0 defp quality_to_pubversion("M"), do: 4
109 2 defp quality_to_pubversion(_q), do: 1
110 end

lib/avy_web/avy_request.ex

100.0
13
53
0
Line Hits Source
0 defmodule AvyWeb.AvyREQUEST do
1 @doc """
2 Request output format
3 """
4
5 def extent(assigns) do
6 1 Enum.map_join(assigns.availabilities, "\n", fn d ->
7 2 format_datasource(d)
8 end)
9 end
10
11 def query(assigns) do
12 1 Enum.map(assigns.availabilities, fn d ->
13 2 Enum.map(d.timespans, fn ts ->
14 4 %{d | earliest: List.first(ts), latest: List.last(ts)}
15 end)
16 end)
17 |> List.flatten()
18 1 |> Enum.map_join("\n", fn d -> format_datasource(d) end)
19 end
20
21 defp format_datasource(d) do
22 [
23 6 d.network,
24 6 d.station,
25 6 d.location,
26 6 d.channel,
27 6 DateTime.to_iso8601(d.earliest),
28 6 DateTime.to_iso8601(d.latest)
29 ]
30 6 |> Enum.join(" ")
31 end
32 end

lib/avy_web/avy_text.ex

100.0
49
708
0
Line Hits Source
0 defmodule AvyWeb.AvyTEXT do
1 @moduledoc """
2 Text format renderer
3 """
4
5 @extent_header [
6 "Network",
7 "Station",
8 "Location",
9 "Channel",
10 "Quality",
11 "SampleRate",
12 String.pad_trailing("Earliest", 27),
13 String.pad_trailing("Latest", 27),
14 String.pad_trailing("Updated", 20),
15 "TimeSpans",
16 "Restriction"
17 ]
18
19 @doc """
20 Text output
21 """
22
23 def version(_) do
24 2 "#{Application.spec(:avy)[:vsn]}"
25 end
26
27 def extent(assigns) do
28 8 headers = format_header(@extent_header, assigns)
29
30 [
31 headers
32 8 | Enum.map(assigns.availabilities, fn d ->
33 13 format_datasource(d, assigns.fdsn_parameters, :extent)
34 end)
35 ]
36 8 |> Enum.join("\n")
37 end
38
39 @query_header [
40 "Network",
41 "Station",
42 "Location",
43 "Channel",
44 "Quality",
45 "SampleRate",
46 String.pad_trailing("Earliest", 27),
47 String.pad_trailing("Latest", 27),
48 "Updated"
49 ]
50 def query(assigns) do
51 5 headers = format_header(@query_header, assigns)
52
53 [
54 headers
55 5 | Enum.map(assigns.availabilities, fn d ->
56 9 Enum.map(d.timespans, fn ts ->
57 13 %{d | timespans: [ts], earliest: List.first(ts), latest: List.last(ts)}
58 end)
59 9 |> Enum.map(fn splitd ->
60 13 format_datasource(
61 splitd,
62 13 assigns.fdsn_parameters,
63 :query
64 )
65 end)
66 end)
67 ]
68 |> List.flatten()
69 5 |> Enum.join("\n")
70 end
71
72 defp format_header(headers, %{fdsn_parameters: fdsn_params} = _) do
73 13 headers =
74 13 if :samplerate in fdsn_params.merge do
75 2 Enum.reject(headers, &String.starts_with?(&1, "SampleRate"))
76 else
77 11 headers
78 end
79
80 13 headers =
81 13 if :quality in fdsn_params.merge do
82 2 Enum.reject(headers, &String.starts_with?(&1, "Quality"))
83 else
84 11 headers
85 end
86
87 13 headers =
88 13 if :latestupdate in fdsn_params.show do
89 12 headers
90 else
91 1 Enum.reject(headers, &String.starts_with?(&1, "Updated"))
92 end
93
94 13 "#" <> Enum.join(headers, " ")
95 end
96
97 defp format_datasource(d, fdsn_params, method) do
98 26 attr_list =
99 13 if method == :extent do
100 13 [d.timespancount |> Integer.to_string() |> String.pad_trailing(9), d.restriction]
101 else
102 []
103 end
104
105 26 attr_list =
106 26 if :latestupdate in fdsn_params.show do
107 [
108 25 DateTime.to_iso8601(d.earliest),
109 25 DateTime.to_iso8601(d.latest),
110 25 DateTime.to_iso8601(d.updated) | attr_list
111 ]
112 else
113 [
114 1 DateTime.to_iso8601(d.earliest),
115 1 DateTime.to_iso8601(d.latest)
116 | attr_list
117 ]
118 end
119
120 26 attr_list =
121 26 if :samplerate in fdsn_params.merge do
122 3 attr_list
123 else
124 [
125 23 d.samplerate
126 |> Decimal.to_string(:xsd)
127 |> String.pad_trailing(10)
128 | attr_list
129 ]
130 end
131
132 26 attr_list =
133 26 if :quality in fdsn_params.merge do
134 2 attr_list
135 else
136 24 [String.pad_trailing(d.quality, 7) | attr_list]
137 end
138
139 [
140 26 String.pad_trailing(d.network, 8),
141 26 String.pad_trailing(d.station, 7),
142 26 String.pad_trailing(d.location, 8),
143 26 String.pad_trailing(d.channel, 7) | attr_list
144 ]
145 26 |> Enum.join(" ")
146 end
147 end

lib/avy_web/controllers/avy_controller.ex

91.3
138
4116
12
Line Hits Source
0 defmodule AvyWeb.AvyController do
1 @moduledoc """
2 Manages a user request.
3 """
4 alias Avy.Repo
5 require Logger
6
7 use AvyWeb, :controller
8 alias FdsnPlugs.PublicationVersion
9 alias Avy.Datasource
10 use OpenApiSpex.ControllerSpecs
11
12 plug :forbidden_params_extent, ["mergegaps"] when action in [:extent]
13 plug :forbidden_params_extent, ["show"] when action in [:extent]
14 plug :format
15 plug AvyWeb.Plugs.Filters
16
17 ### Define the OpenApi spec
18 0 operation :version,
19 summary: "Get the version number of the service"
20
21 0 operation :extent,
22 summary: "Query the datacenter for available waveform data.",
23 description: "Returns only the earliest and latest data.",
24 parameters: Avy.ApiSpec.parameters_spec(),
25 responses: Avy.ApiSpec.responses_spec()
26
27 0 operation :extent_post,
28 summary: "Query the datacenter for available waveform data.",
29 description: "Returns only the earliest and latest data.",
30 request_body: Avy.ApiSpec.request_body_spec(),
31 responses: Avy.ApiSpec.responses_spec()
32
33 0 operation :query,
34 summary: "Query the datacenter for available waveform data.",
35 description: "Returns details of all segments of data",
36 parameters: Avy.ApiSpec.query_parameters_spec(),
37 responses: Avy.ApiSpec.responses_spec()
38
39 0 operation :query_post,
40 summary: "Query the datacenter for available waveform data",
41 request_body: Avy.ApiSpec.request_body_spec(),
42 responses: Avy.ApiSpec.responses_spec()
43
44 ### End of OpenApi definition
45
46 def version(conn, _params) do
47 2 render(conn, :version)
48 end
49
50 # It seems that I need to define this placeholder function in order to be able to define a specific
51 # operation for POST extent in OpenApi
52 1 def extent_post(%Plug.Conn{method: "POST"} = conn, params), do: extent(conn, params)
53
54 def extent(conn, _params) do
55 20 case manage_request(conn, :extent) do
56 {:ok, []} ->
57 conn
58 2 |> send_resp(conn.assigns.fdsn_parameters.nodata, "")
59 2 |> halt
60
61 {:ok, ds} ->
62 18 datasources = Enum.map(ds, &Map.drop(&1, [:timespans]))
63
64 conn
65 |> assign(:availabilities, datasources |> Enum.map(&Map.from_struct(&1)))
66 18 |> render(:extent)
67
68 {:too_much_data, msg} ->
69 0 conn |> send_resp(413, msg) |> halt()
70 end
71 end
72
73 @doc """
74 This controller exists to reformat the output suitable for a grafana state timeline
75
76 All the formatting logic takes place in the render function AvyJson.statetimeline/1
77 """
78 def statetimeline(conn, opts) do
79 conn
80 |> put_format("json")
81 |> put_resp_content_type("application/json")
82 1 |> query(opts)
83 end
84
85 def query(conn, _params) do
86 19 case manage_request(conn, :query) do
87 {:ok, []} ->
88 conn
89 1 |> send_resp(conn.assigns.fdsn_parameters.nodata, "")
90 1 |> halt
91
92 {:ok, ds} ->
93 18 datasources = Enum.map(ds, &Map.drop(&1, [:timespancount]))
94
95 18 Logger.info("Fetched #{length(datasources)} traces")
96
97 conn
98 |> assign(:availabilities, datasources |> Enum.map(&Map.from_struct(&1)))
99 18 |> render()
100
101 {:too_much_data, msg} ->
102 0 conn |> send_resp(413, msg) |> halt()
103 end
104 end
105
106 @doc """
107 This function is called on a /extent request.
108 It will fetch all the contents corresponding to the filters that were set up by
109 AvyWeb.Plugs.Filters, in the assigns of the connection plug (`conn.assigns.filters`).
110 For each filter, the database request is done asynchronously in parallel.
111 Desactivate the timeout, as the DB pool has it's own already, and some requests can get quite long.
112
113 For each entry, we call the post_process function in order to build a list of datasources and merge the contents of the same properties.
114
115 Finally, the datasources are put in the conn.assigns in order for ther renderer to access them in the `:availabilities` keyword.
116 """
117 @spec manage_request(Plug.Conn.t(), atom) :: {:ok, list} | {:too_much_data, String.t()}
118 def manage_request(conn, method) do
119 39 datasources =
120 39 Task.async_stream(conn.assigns.filters, Repo, :get_contents, [], timeout: :infinity)
121 39 |> Enum.flat_map(fn {:ok, f} -> f end)
122
123 # TODO Improve evaluation of too much data
124 39 if length(datasources) > Application.get_env(:avy, :max_sids) do
125 {:too_much_data,
126 0 "Request has #{length(datasources)} epochs. Try to restrict the source identifiers selection and/or select smaller timespans."}
127 else
128 39 datasources =
129 datasources
130 |> post_process(
131 method,
132 39 conn.assigns.fdsn_parameters.merge,
133 39 conn.assigns.fdsn_parameters.mergegaps * 10 ** 9
134 )
135 64 |> Enum.sort_by(&{&1.network, &1.station, &1.location, &1.channel, &1.earliest})
136 39 |> orderby(conn.assigns.fdsn_parameters.orderby, method)
137 39 |> to_gaps(Map.has_key?(conn.params, "showgaps"))
138
139 {:ok, datasources}
140 end
141 end
142
143 @doc """
144
145 This function will convert a list of ordered datasources to
146 show gaps.
147
148 First, group all datasources by the full keys.
149 Then for each list:
150 - keep the previous end date. (Start first iteration at second element)
151 - on the current element:
152 - set the start time at previous end date
153 - set then end time at current start.
154 - put the result at the head of the list we are building
155 - we obtain a reversed list, so at the end we need to reverse it.
156 """
157
158 @spec to_gaps(list(), boolean) :: list()
159 39 def to_gaps(datasources, false), do: datasources
160
161 def to_gaps(datasources, true) do
162 1 Logger.info("Computing gaps")
163
164 datasources
165 |> Enum.group_by(group_by_keys([], :query))
166 |> Enum.map(fn {_, contents} ->
167 1 Enum.map(contents, fn content ->
168 2 Logger.debug(inspect(content, pretty: true))
169 2 [[_, init_end] | tail] = content.timespans
170
171 2 %{prev_end: _, list: gaps} =
172 Enum.reduce(tail, %{prev_end: init_end, list: []}, fn [tss, tse], acc ->
173 2 %{
174 prev_end: tse,
175 2 list: [[acc.prev_end, tss] | acc.list]
176 }
177 end)
178
179 2 Logger.debug(inspect(gaps))
180 2 %{content | timespans: Enum.reverse(gaps)}
181 end)
182 end)
183 |> List.flatten()
184 1 |> Enum.reject(fn a -> a.timespans == [] end)
185 end
186
187 #### Private functions for query requests
188
189 # post_process/4 contains all the logic to handle the contents retrieved in the Repo.
190 # Convert the structure returned by the Repo in a usable list of datasources, fit for the views
191 # Post process the structure returned by the Repo call in order to:
192 # - Group all elements by their discriminating properties (including restriction status)
193 # - For each group,
194 # - clean the unwanted timespans in contents
195 # - merge consecutive timespans either by using the mergegap provided in the request, or by the samplerate
196 # With the result, prepare a structure suitable for the views:
197 # - compute the number of timespans
198 # - convert publication version to quality code
199 #
200 @spec post_process(list, atom, list, integer) :: list
201 defp post_process(lst, method, merge_properties, mergegap) do
202 lst
203 # Group all the contents by a key, depending on the user request
204 |> Enum.group_by(group_by_keys(merge_properties, method))
205 # With the grouped contents, do the post processing.
206 |> Task.async_stream(fn {k, contents} ->
207 67 Logger.info("Processing #{length(contents)} contents for #{inspect(k)}")
208 67 Logger.debug(inspect(contents, pretty: true))
209 67 contents = filter_outofbound_timespans(contents)
210
211 67 if contents != [] do
212 64 ref_content = List.first(contents)
213
214 64 ref_samplerate =
215 64 Enum.min_by(contents, & &1.samplerate) |> Map.fetch!(:samplerate) |> Decimal.to_float()
216
217 64 Logger.debug("Min samplerate: #{ref_samplerate}")
218
219 # When we merge contiguous timespans based on the samplerate
220 64 mergegap =
221 if mergegap <= 0 do
222 # When no mergegaps are indicated
223 50 10 ** 9 / ref_samplerate
224 else
225 14 mergegap
226 end
227
228 64 Logger.debug("Merging on gaps lower than #{mergegap}")
229
230 64 Logger.debug(
231 0 "Timespans: #{inspect(Enum.flat_map(contents, & &1.timespans), pretty: true)}"
232 )
233
234 64 merged_timespans =
235 101 Enum.flat_map(contents, & &1.timespans)
236 |> merge_contiguous_timespans(mergegap)
237 137 |> Enum.map(fn x ->
238 [
239 Map.fetch!(x, "start") |> DateTime.from_unix!(:nanosecond),
240 Map.fetch!(x, "end") |> DateTime.from_unix!(:nanosecond)
241 ]
242 end)
243
244 # |> snip_timespans(ref_content.earliest, List.last(contents).latest)
245
246 64 %Datasource{
247 64 network: ref_content.network,
248 64 station: ref_content.station,
249 64 location: ref_content.location,
250 64 channel: ref_content.channel,
251 64 quality: PublicationVersion.pubversion_to_quality!(ref_content.quality),
252 64 samplerate: ref_content.samplerate,
253 64 earliest: contents |> List.first() |> Map.fetch!(:earliest),
254 64 latest: contents |> List.last() |> Map.fetch!(:latest),
255 64 updated: Enum.max_by(contents, & &1.updated, Date) |> Map.fetch!(:updated),
256 restriction:
257 64 case ref_content.restriction do
258 62 "O" -> "OPEN"
259 2 _ -> "RESTRICTED"
260 end,
261 101 timespancount: Enum.reduce(contents, 0, fn c, acc -> acc + length(c.timespans) end),
262 timespans: merged_timespans
263 }
264 else
265 nil
266 end
267 end)
268 67 |> Enum.into([], fn {:ok, res} -> res end)
269 39 |> Enum.reject(&is_nil(&1))
270 end
271
272 #
273 # After grouping all contents by epochs,
274 # In the timespans, there can be some contents that are outside of the request boundaries.
275 # This function will remove contents that are completely out of bound
276 # And if a content is left empty, we remove it from the list.
277 defp filter_outofbound_timespans(contents) do
278 contents
279 |> Enum.map(&Datasource.trim_timespans(&1))
280 67 |> Enum.reject(&(&1.timespans == []))
281 end
282
283 defp merge_contiguous_timespans(timespans, mergegap) do
284 64 Logger.debug("Timespans to reduce: #{inspect(timespans, pretty: true)}")
285
286 Enum.reduce(
287 timespans,
288 Enum.take(timespans, 1),
289 fn timespan, acc ->
290 149 [previous | tail] = acc
291
292 149 if timespan["start"] - previous["end"] <= mergegap do
293 76 Logger.debug(
294 0 "Merging timespan #{previous["start"] |> DateTime.from_unix!(:nanosecond)} -> #{previous["end"] |> DateTime.from_unix!(:nanosecond)}"
295 )
296
297 76 Logger.debug(
298 0 "and #{timespan["start"] |> DateTime.from_unix!(:nanosecond)} -> #{timespan["end"] |> DateTime.from_unix!(:nanosecond)}"
299 )
300
301 76 Logger.debug("with gap #{timespan["start"] - previous["end"]} <= #{mergegap}")
302
303 [
304 %{
305 "start" => previous["start"],
306 "end" => timespan["end"],
307 "sample_rate" => previous["sample_rate"]
308 }
309 | tail
310 ]
311 else
312 [timespan | acc]
313 end
314 end
315 )
316 64 |> Enum.reverse()
317 end
318
319 # Plug function: In /extent action, mergegaps and show are not allowed
320 defp forbidden_params_extent(conn, [opt]) do
321 41 Logger.debug(
322 0 "Check for forbidden params for query whith #{inspect(conn.params, pretty: true)}"
323 )
324
325 41 case Map.fetch(conn.params, opt) do
326 {:ok, _} ->
327 2 send_resp(conn, 400, "Options #{opt} is only supported in the /query method.")
328 2 |> halt
329
330 :error ->
331 39 conn
332 end
333 end
334
335 @spec group_by_keys(list(atom), :query | :extent) :: function
336 defp group_by_keys(merge_properties, :extent) do
337 20 cond do
338 20 :quality in merge_properties and :samplerate in merge_properties ->
339 2 fn a ->
340 6 {a.network, a.station, a.location, a.channel, a.restriction}
341 end
342
343 18 :samplerate in merge_properties ->
344 2 fn a ->
345 6 {a.network, a.station, a.location, a.channel, a.quality, a.restriction}
346 end
347
348 16 :quality in merge_properties ->
349 4 fn a ->
350 15 {a.network, a.station, a.location, a.channel, a.samplerate, a.restriction}
351 end
352
353 # Default behaviour
354 12 true ->
355 12 fn a ->
356 29 {a.network, a.station, a.location, a.channel, a.quality, a.samplerate, a.restriction}
357 end
358 end
359 end
360
361 defp group_by_keys(merge_properties, :query) do
362 20 cond do
363 20 :quality in merge_properties and :samplerate in merge_properties ->
364 1 fn a ->
365 3 {a.network, a.station, a.location, a.channel}
366 end
367
368 19 :samplerate in merge_properties ->
369 1 fn a ->
370 3 {a.network, a.station, a.location, a.channel, a.quality}
371 end
372
373 18 :quality in merge_properties ->
374 1 fn a ->
375 3 {a.network, a.station, a.location, a.channel, a.samplerate}
376 end
377
378 # Default behaviour
379 17 true ->
380 17 fn a ->
381 41 {a.network, a.station, a.location, a.channel, a.quality, a.samplerate}
382 end
383 end
384 end
385
386 #  The API uses the "format" parameter on order to set the output format.
387 # Translate this to the _format param for Phoenix magic to take place.
388 # Also force the response content type header to text
389 defp format(conn, _) do
390 41 with {:ok, _} <- Map.fetch(conn.assigns, :fdsn_parameters),
391 39 {:ok, format} <- Map.fetch(conn.assigns.fdsn_parameters, :format) do
392 39 case format do
393 :json ->
394 conn
395 |> Phoenix.Controller.put_format("json")
396 13 |> Plug.Conn.put_resp_content_type("application/json")
397
398 :geocsv ->
399 conn
400 |> Phoenix.Controller.put_format("geocsv")
401 8 |> Plug.Conn.put_resp_content_type("text/csv")
402
403 :request ->
404 conn
405 |> Phoenix.Controller.put_format("request")
406 2 |> Plug.Conn.put_resp_content_type("text/plain")
407
408 _ ->
409 conn
410 |> Phoenix.Controller.put_format("text")
411 16 |> Plug.Conn.put_resp_content_type("text/plain")
412 end
413 else
414 _ ->
415 conn
416 |> Phoenix.Controller.put_format("text")
417 2 |> Plug.Conn.put_resp_content_type("text/plain")
418 end
419 end
420
421 defp orderby(ds, orderby, :extent) do
422 20 Logger.debug("Reorder datasources by #{orderby}")
423
424 20 case orderby do
425 1 :latestupdate -> Enum.sort_by(ds, & &1.updated, {:asc, DateTime})
426 1 :latestupdate_desc -> Enum.sort_by(ds, & &1.updated, {:desc, DateTime})
427 1 :timespancount -> Enum.sort_by(ds, & &1.timespancount, :asc)
428 1 :timespancount_desc -> Enum.sort_by(ds, & &1.timespancount, :desc)
429 16 _ -> ds
430 end
431 end
432
433 defp orderby(ds, orderby, :query) do
434 19 Logger.debug("Reorder datasources by #{orderby}")
435
436 19 case orderby do
437 2 :latestupdate -> Enum.sort_by(ds, & &1.updated, {:asc, DateTime})
438 2 :latestupdate_desc -> Enum.sort_by(ds, & &1.updated, {:desc, DateTime})
439 15 _ -> ds
440 end
441 end
442 end

lib/avy_web/controllers/error_json.ex

100.0
1
2
0
Line Hits Source
0 defmodule AvyWeb.ErrorJSON do
1 @moduledoc """
2 This module is invoked by your endpoint in case of errors on JSON requests.
3
4 See config/config.exs.
5 """
6
7 # If you want to customize a particular status code,
8 # you may add your own clauses, such as:
9 #
10 # def render("500.json", _assigns) do
11 # %{errors: %{detail: "Internal Server Error"}}
12 # end
13
14 # By default, Phoenix returns the status message from
15 # the template name. For example, "404.json" becomes
16 # "Not Found".
17 def render(template, _assigns) do
18 2 %{errors: %{detail: Phoenix.Controller.status_message_from_template(template)}}
19 end
20 end

lib/avy_web/endpoint.ex

66.6
3
106
1
Line Hits Source
0 defmodule AvyWeb.Endpoint do
1 use Sentry.PlugCapture
2 use Phoenix.Endpoint, otp_app: :avy
3
4 # The session will be stored in the cookie and signed,
5 # this means its contents can be read but not tampered with.
6 # Set :encryption_salt if you would also like to encrypt it.
7 @session_options [
8 store: :cookie,
9 key: "_avy_key",
10 signing_salt: "xyXumY7h",
11 same_site: "Lax"
12 ]
13
14 socket "/live", Phoenix.LiveView.Socket,
15 websocket: [connect_info: [session: @session_options]],
16 longpoll: [connect_info: [session: @session_options]]
17
18 # Serve at "/" the static files from "priv/static" directory.
19 #
20 # You should set gzip to true if you are running phx.digest
21 # when deploying your static files in production.
22 plug Plug.Static,
23 at: "/",
24 from: :avy,
25 gzip: true,
26 content_types: %{"application.wadl" => "text/xml"},
27 only: AvyWeb.static_paths()
28
29 # Answer to /_health request
30 plug AvyWeb.Plug.HealthCheck
31
32 # Code reloading can be explicitly enabled under the
33 # :code_reloader configuration of your endpoint.
34 if code_reloading? do
35 plug Phoenix.CodeReloader
36 plug Phoenix.Ecto.CheckRepoStatus, otp_app: :avy
37 end
38
39 plug Phoenix.LiveDashboard.RequestLogger,
40 param_key: "request_logger",
41 cookie_key: "request_logger"
42
43 plug Plug.RequestId
44 plug Plug.Telemetry, event_prefix: [:phoenix, :endpoint], log: {__MODULE__, :log_level, []}
45
46 plug Plug.Parsers,
47 parsers: [:urlencoded, :multipart, :json],
48 pass: ["*/*"],
49 json_decoder: Phoenix.json_library()
50
51 plug Sentry.PlugContext
52 plug Plug.MethodOverride
53 plug Plug.Head
54 plug Plug.Session, @session_options
55
56 plug AvyWeb.Plug.TrafficDrain
57 plug AvyWeb.Router
58 #
59 # Return `false` for the __health endpoint
60 0 def log_level(%{path_info: ["__health" | _]}), do: false
61 2 def log_level(%{path_info: ["__traffic" | _]}), do: false
62 # Default to the normal loglevel
63 104 def log_level(_), do: :info
64 end

lib/avy_web/plugs/filters.ex

100.0
7
279
0
Line Hits Source
0 defmodule AvyWeb.Plugs.Filters do
1 @moduledoc """
2 A plug to add a Filter struct to the connection assigns.
3 The filter is made of the source identifier + includerestricted + quality
4 NOTE: this is maybe redundant with FdsnPlugs.RequestParams but is more suited to filtering by Avy.Repo
5 Refactoring may be.
6 """
7 import Plug.Conn
8
9 41 def init(opts), do: opts
10
11 def call(conn, _opts) do
12 conn
13 41 |> assign(
14 :filters,
15 41 if Map.has_key?(conn.assigns, :fdsn_parameters) do
16 39 Enum.map(conn.assigns.fdsn_parameters.source_identifiers, fn s ->
17 39 Avy.Filter.from_source_identifier(
18 s,
19 39 conn.assigns.fdsn_parameters.includerestricted,
20 39 conn.assigns.fdsn_parameters.quality
21 )
22 end)
23 else
24 []
25 end
26 )
27 end
28 end

lib/avy_web/plugs/health_check.ex

100.0
3
108
0
Line Hits Source
0 defmodule AvyWeb.Plug.HealthCheck do
1 import Plug.Conn
2
3 54 def init(opts), do: opts
4
5 # If the request path matches "/_health", we return a 200 response.
6 def call(%Plug.Conn{request_path: "/__health"} = conn, _opts) do
7 conn
8 |> send_resp(
9 200,
10 ""
11 )
12 1 |> halt()
13 end
14
15 # If the request path is anything else, we pass the connection along.
16 53 def call(conn, _opts), do: conn
17 end

lib/avy_web/plugs/traffic_drain.ex

80.0
5
107
1
Line Hits Source
0 defmodule AvyWeb.Plug.TrafficDrain do
1 @moduledoc """
2 Plug for handling Kubernetes readinessProbe.
3
4 Plug starts responding with 503 - Service Unavailable from `/__traffic`, when traffic is being drained.
5 Otherwise we respond with 200 - OK.
6 """
7
8 import Plug.Conn
9
10 @behaviour Plug
11
12 @impl true
13 53 def init(opts), do: opts
14
15 @impl true
16 def call(%Plug.Conn{path_info: ["__traffic"]} = conn, _opts) do
17 1 case GracefulStop.get_status() do
18 :stopping ->
19 conn
20 |> put_resp_content_type("text/plain")
21 |> send_resp(:service_unavailable, "Draining")
22 0 |> halt()
23
24 :running ->
25 conn
26 |> put_resp_content_type("text/plain")
27 |> send_resp(:ok, "Serving")
28 1 |> halt()
29 end
30 end
31
32 @impl true
33 def call(conn, _opts) do
34 52 conn
35 end
36 end

lib/avy_web/router.ex

81.8
11
102
2
Line Hits Source
0 defmodule AvyWeb.Router do
1 use AvyWeb, :router
2
3 49 pipeline :fdsn do
4 plug :accepts, ["json", "text"]
5
6 plug FdsnAvailabilityPlugs
7 end
8
9 0 pipeline :openapi do
10 plug OpenApiSpex.Plug.PutApiSpec, module: Avy.ApiSpec
11 end
12
13 1 pipeline :browser do
14 plug :accepts, ["html"]
15 plug :fetch_session
16 # plug :fetch_live_flash
17 plug :put_root_layout, html: {AvyWeb.Layouts, :root}
18 plug :protect_from_forgery
19 plug :put_secure_browser_headers
20 end
21
22 2 get "/version", AvyWeb.AvyController, :version
23
24 scope "/", AvyWeb do
25 pipe_through :fdsn
26 26 get "/extent", AvyController, :extent
27 4 post "/extent", AvyController, :extent_post
28 16 get "/query", AvyController, :query
29 2 post "/query", AvyController, :query
30 1 get "/statetimeline", AvyController, :statetimeline
31 end
32
33 scope "/" do
34 pipe_through :browser
35
36 # To serve the UI, a static path has to be given at compile time.
37 # I declare here the official FDSN location it should work in
38 # most cases.
39 1 get "/", OpenApiSpex.Plug.SwaggerUI, path: Application.compile_env(:avy, :open_api_path)
40 end
41
42 scope "/" do
43 pipe_through :openapi
44 0 get "/openapi", OpenApiSpex.Plug.RenderSpec, []
45 end
46
47 # Enable LiveDashboard and Swoosh mailbox preview in development
48 if Application.compile_env(:avy, :dev_routes) do
49 # If you want to use the LiveDashboard in production, you should put
50 # it behind authentication and allow only admins to access it.
51 # If your application does not have an admins-only section yet,
52 # you can use Plug.BasicAuth to set up some basic authentication
53 # as long as you are also using SSL (which you should anyway).
54 import Phoenix.LiveDashboard.Router
55
56 scope "/dev" do
57 pipe_through [:fetch_session, :protect_from_forgery]
58
59 live_dashboard "/dashboard",
60 metrics: AvyWeb.Telemetry,
61 additional_pages: [
62 flame_on: FlameOn.DashboardPage
63 ]
64
65 forward "/mailbox", Plug.Swoosh.MailboxPreview
66 end
67 end
68 end