Coverage

90.0
391
23556
39

lib/avy.ex

100.0
0
0
0
Line Hits Source
0 defmodule Avy do
1 @moduledoc """
2 Avy keeps the contexts that define your domain
3 and business logic.
4
5 Contexts are also responsible for managing your data, regardless
6 if it comes from the database, an external API or others.
7 """
8 end

lib/avy/api_spec.ex

0.0
9
0
9
Line Hits Source
0 defmodule Avy.ApiSpec do
1 alias OpenApiSpex.Parameter
2 alias OpenApiSpex.RequestBody
3
4 alias OpenApiSpex.{
5 Components,
6 Info,
7 OpenApi,
8 Paths,
9 Server,
10 Response,
11 MediaType,
12 Schema,
13 RequestBody,
14 Example,
15 Reference
16 }
17
18 alias AvyWeb.{Endpoint, Router}
19 @behaviour OpenApi
20
21 # References are not supported for RequestBody. Let's define it here
22 # https://github.com/open-api-spex/open_api_spex/issues/692
23 @request_body_spec %RequestBody{
24 description: """
25 all parameters must be submitted as part of the POST body. The quality,
26 minimumlength and longestonly parameters should be specified as key=value pairs on separate lines
27 and the simple-time and channel constraints parameters repeated as many times as necessary following
28 this pattern:
29 ````
30 quality=M
31 NET STA LOC CHA STARTTIME ENDTIME
32 NET STA LOC CHA STARTTIME ENDTIME
33 NET STA LOC CHA STARTTIME ENDTIME
34 ````
35 All rules for parameters apply equally whether specified using the GET or POST methods with the
36 exception of blank location IDs, which must be specified as “--“ in the POST body due to spaces being
37 used as the field separato
38 """,
39 required: true,
40 content: %{
41 "plain/text" => %MediaType{
42 schema: %Schema{type: :string},
43 example: """
44 quality=B
45 FR CIEL 00 HHZ 2025-11-20T12:00:00 2025-11-20T12:01:00
46 RA UNIO 00 HNZ 2025-11-20T12:00:00 2025-11-20T12:01:00
47 """
48 }
49 }
50 }
51 @responses_spec [
52 ok: %Reference{"$ref": "#/components/responses/success"},
53 no_content: %Reference{"$ref": "#/components/responses/nodata"},
54 not_found: %Reference{"$ref": "#/components/responses/nodata"},
55 bad_request: %Reference{"$ref": "#/components/responses/bad_request"},
56 request_entity_too_large: %Reference{"$ref": "#/components/responses/too_much_data"}
57 ]
58 @parameters_spec [
59 %Reference{"$ref": "#/components/parameters/network"},
60 %Reference{"$ref": "#/components/parameters/station"},
61 %Reference{"$ref": "#/components/parameters/location"},
62 %Reference{"$ref": "#/components/parameters/channel"},
63 %Reference{"$ref": "#/components/parameters/starttime"},
64 %Reference{"$ref": "#/components/parameters/endtime"},
65 %Reference{"$ref": "#/components/parameters/nodata"},
66 %Reference{"$ref": "#/components/parameters/format"},
67 %Reference{"$ref": "#/components/parameters/quality"},
68 %Reference{"$ref": "#/components/parameters/merge"},
69 %Reference{"$ref": "#/components/parameters/orderby"},
70 %Reference{"$ref": "#/components/parameters/includerestricted"}
71 ]
72 0 def request_body_spec, do: @request_body_spec
73 0 def responses_spec, do: @responses_spec
74 0 def parameters_spec, do: @parameters_spec
75
76 def query_parameters_spec,
77 do:
78 0 @parameters_spec ++
79 [
80 %Reference{"$ref": "#/components/parameters/mergegaps"},
81 %Reference{"$ref": "#/components/parameters/show"},
82 %Reference{"$ref": "#/components/parameters/showgaps"}
83 ]
84
85 @impl OpenApi
86 def spec do
87 %OpenApi{
88 servers: [
89 openapi_server_from_endpoint(Endpoint)
90 ],
91 info: %Info{
92 title: "FDSNWS availability - OpenAPI 3.0",
93 version: "1.0"
94 },
95 # Populate the paths from a phoenix router
96 paths: Paths.from_router(Router),
97 components: %Components{
98 responses: %{
99 success: %Response{
100 description: "successful request",
101 content: %{
102 "text/plain" => %MediaType{schema: %Schema{type: :string}},
103 "application/json" => %MediaType{schema: %Schema{type: :string}},
104 "text/csv" => %MediaType{schema: %Schema{type: :string}}
105 }
106 },
107 nodata: %Response{
108 description: "no data"
109 },
110 bad_request: %Response{
111 description: "Invalid parameters in query string."
112 },
113 too_much_data: %Response{
114 description:
115 "Too much data requested. Try to split your request with narrower criteria"
116 }
117 },
118 requestBodies: %{query: @request_body_spec},
119 parameters: %{
120 network: %Parameter{
121 name: "network",
122 in: :query,
123 required: false,
124 explode: false,
125 description: """
126 Select one or more network codes. Can be SEED network codes or data center defined codes.
127 Multiple codes are comma-separated. Shortcut `net` can also be used.
128 Wildcards are supported (`_` means any character, once; `%` means any character 0 or several times.)
129 """,
130 schema: %Schema{
131 type: :array,
132 items: %Schema{type: :string, example: "FR"}
133 },
134 examples: %{
135 FR: %Example{value: "FR", description: "Select the FR network code"},
136 "FR,RA": %Example{
137 value: ["FR", "RA"],
138 description: "Select the FR and RA network codes"
139 },
140 "X%": %Example{
141 value: "X%",
142 description: "Select all network codes starting with X"
143 }
144 }
145 },
146 station: %Parameter{
147 name: "station",
148 in: :query,
149 required: false,
150 explode: false,
151 description: """
152 Select one or more SEED station codes.
153 Multiple codes are comma-separated.
154 Shortcut `sta` can also be used.
155 Wildcards are supported (`_` means any character, once; `%` means any character 0 or several times.)
156 """,
157 schema: %Schema{
158 type: :array,
159 items: %Schema{type: :string, example: "CIEL"}
160 },
161 examples: %{
162 CIEL: %Example{value: "CIEL", description: "Select the CIEL sation code"},
163 "CIEL,ILLK": %Example{
164 value: ["CIEL", "ILLK"],
165 description: "Select two station codes"
166 },
167 "Z%": %Example{
168 value: "Z%",
169 description: "Select all station codes starting with Z"
170 }
171 }
172 },
173 location: %Parameter{
174 name: "location",
175 in: :query,
176 required: false,
177 explode: false,
178 description: """
179 Select one or more SEED location codes.
180 Empty location code can be specified using `--`.
181 Multiple codes are comma-separated.
182 Shortcut `loc` can also be used.
183 Wildcards are supported (`_` means any character, once; `%` means any character 0 or several times.)
184 """,
185 schema: %Schema{
186 type: :array,
187 items: %Schema{type: :string, example: "00"}
188 },
189 examples: %{
190 "00": %Example{value: "00", description: "Select the 00 location code"},
191 "00,--": %Example{
192 value: ["00", "--"],
193 description: "Select location codes 00 and empty"
194 }
195 }
196 },
197 channel: %Parameter{
198 name: "channel",
199 in: :query,
200 required: false,
201 explode: false,
202 description: """
203 Select one or more SEED channel codes.
204 Multiple codes are comma-separated.
205 Shortcut `cha` can also be used.
206 Wildcards are supported (`_` means any character, once; `%` means any character 0 or several times.)
207 """,
208 schema: %Schema{
209 type: :array,
210 items: %Schema{type: :string, example: "HHZ"}
211 },
212 examples: %{
213 HHZ: %Example{value: "HHZ", description: "Select the HNE channel code"},
214 "HHZ,BHZ": %Example{value: ["HHZ", "BHZ"], description: "Select two channel codes"},
215 "%Z": %Example{description: "Select all channel ending with Z"}
216 }
217 },
218 starttime: %Parameter{
219 name: "starttime",
220 in: :query,
221 required: false,
222 explode: false,
223 description: """
224 Limit results to time series samples on or after the specified start time. If not specified, timezone is assumed as UTC. Shortcut `start`
225 """,
226 schema: %Schema{
227 type: :string,
228 format: "date-time"
229 },
230 examples: %{
231 "2025-11-20T12:38:04.000000Z": %Example{
232 value: "2025-11-20T12:38:04.000000Z",
233 description: "microseconds and timezone"
234 },
235 "2025-11-20T12:38:04Z": %Example{
236 value: "2025-11-20T12:38:04Z",
237 description: "seconds and timezone"
238 },
239 "2025-11-20T12:38:04": %Example{
240 value: "2025-11-20T12:38:04",
241 description: "seconds without timezone (UTC is assumed)"
242 },
243 "2025-11-20": %Example{
244 value: "2025-11-20",
245 description: "Without time, 00:00:00.000000Z is assumed"
246 }
247 }
248 },
249 endtime: %Parameter{
250 name: "endtime",
251 in: :query,
252 required: false,
253 explode: false,
254 description:
255 "Limit results to time series samples on or after the specified end time. If not specified, timezone is assumed as UTC. Shortcut `end`",
256 schema: %Schema{
257 type: :string,
258 format: "date-time"
259 },
260 examples: %{
261 "2025-11-20T12:38:14.000000Z": %Example{
262 value: "2025-11-20T12:38:14.000000Z",
263 description: "microseconds and timezone"
264 },
265 "2025-11-20T12:38:14Z": %Example{
266 value: "2025-11-20T12:38:14Z",
267 description: "seconds and timezone"
268 },
269 "2025-11-20T12:38:14": %Example{
270 value: "2025-11-20T12:38:14",
271 description: "seconds without timezone (UTC is assumed)"
272 },
273 "2025-11-21": %Example{
274 value: "2025-11-21",
275 description: "Without time, 00:00:00.000000Z is assumed"
276 }
277 }
278 },
279 format: %Parameter{
280 name: "format",
281 in: :query,
282 required: false,
283 description: "Specify format of result, the default value is text.",
284 schema: %Schema{
285 type: :string,
286 default: "text",
287 enum: ["text", "geocsv", "json", "request"]
288 }
289 },
290 nodata: %Parameter{
291 name: "nodata",
292 in: :query,
293 required: false,
294 description: "Specify the return code for no data. Valid values are 204 or 404",
295 schema: %Schema{
296 type: :integer,
297 default: 204,
298 enum: [204, 404]
299 }
300 },
301 quality: %Parameter{
302 name: "quality",
303 in: :query,
304 required: false,
305 explode: false,
306 description: """
307 Select a specific SEED quality indicator.
308 * R -> Raw
309 * Q -> Quality controlled
310 * M -> Modified
311 * D -> Data, unspecified
312 """,
313 schema: %Schema{
314 type: :array,
315 items: %Schema{type: :string, example: "M", enum: ["R", "Q", "M", "D"]},
316 default: "M"
317 }
318 },
319 merge: %Parameter{
320 name: "merge",
321 in: :query,
322 required: false,
323 explode: false,
324 description: """
325 If set to one or more of the following values, time spans are merged as described. Multiple values may be specified as a
326 comma-delimited list, e.g. ``merge=samplerate,quality``.
327 * `samplerate`: time spans from data with differing sample rates will be grouped together. If specified this field will be
328 omitted from the result.
329 * `quality`: time spans from data with differing quality codes will be grouped together. If specified this field will be omitted
330 from the result.
331 * `overlap`: time spans from data that overlap will be merged together. This option does not apply to the extent method
332 """,
333 schema: %Schema{
334 type: :array,
335 items: %Schema{
336 type: :string,
337 example: "samplerate",
338 enum: ["quality", "samplerate", "overlap"]
339 },
340 default: ""
341 }
342 },
343 includerestricted: %Parameter{
344 name: "includerestricted",
345 in: :query,
346 required: false,
347 description:
348 "If true, all data are reported. If false, only data that can be openly accessed.",
349 schema: %Schema{
350 type: :string,
351 default: "TRUE",
352 enum: ["TRUE", "FALSE"]
353 }
354 },
355 orderby: %Parameter{
356 name: "orderby",
357 in: :query,
358 required: false,
359 description: """
360 Sort results by one of the following values in the order specified:
361 * `nslc_time_quality_samplerate`: network, station, location, channel, time-range, quality, sample-rate (default)
362 * `latestupdate`: update-date (past to present), network, station, location, channel, time-range, quality, sample-rate
363 * `latestupdate_desc`: update-date (present to past), network, station, location, channel, time-range, quality, sample-rate
364 * `timespancount`: number of timespans (small to large), network, station, location, channel, time-range, quality, sample-rate
365 * `timespancount_desc`: number of timespans (large to small), network, station, location, channel, time-range, quality, sample-rate
366 """,
367 schema: %Schema{
368 type: :strung,
369 example: "latestupdate_desc",
370 enum: [
371 "nslc_time_quality_samplerate",
372 "latestupdate",
373 "latestupdate_desc",
374 "timaspancount",
375 "timespancount_desc"
376 ],
377 default: "latestupdate"
378 }
379 },
380 mergegaps: %Parameter{
381 name: "mergegaps",
382 in: :query,
383 required: false,
384 description:
385 "Merge time spans that are separated by the specified tolerance in seconds. Only positive values are allowed.",
386 schema: %Schema{
387 type: :number,
388 minimum: 0,
389 example: 10.5,
390 default: 0.0
391 }
392 },
393 show: %Parameter{
394 name: "show",
395 in: :query,
396 required: false,
397 description: """
398 If set to `latestupdate`, the latest times at which data contributing to the returned time spans were loaded into the repository are included in the result.
399 This option applies to all formats except `request`.
400 """,
401 schema: %Schema{
402 type: :string,
403 example: "latestupdate",
404 enum: ["latestupdate", ""],
405 default: ""
406 }
407 },
408 showgaps: %Parameter{
409 name: "showgaps",
410 in: :query,
411 required: false,
412 description: """
413 If present, instead of showing timespans of available data, the result shows timespans of missing data.
414 This parameter is not in the FDSN specification and has been added for convenience.
415 """,
416 schema: %Schema{
417 type: :boolean,
418 nullable: true
419 }
420 }
421 }
422 }
423 }
424 # Discover request/response schemas from path specs
425 0 |> OpenApiSpex.resolve_schema_modules()
426 end
427
428 defp openapi_server_from_endpoint(endpoint) do
429 0 uri = endpoint.struct_url()
430 0 path = Application.get_env(:avy, :url_prefix)
431 0 uri = %{uri | path: path}
432
433 0 %Server{
434 url: URI.to_string(uri)
435 }
436 end
437 end

lib/avy/application.ex

81.8
11
9
2
Line Hits Source
0 defmodule Avy.Application do
1 # See https://hexdocs.pm/elixir/Application.html
2 # for more information on OTP Applications
3 @moduledoc false
4
5 use Application
6 # require OpentelemetryEcto
7 # require OpentelemetryBandit
8 # require OpentelemetryPhoenix
9
10 @impl true
11 def start(_type, _args) do
12 1 :logger.add_handler(:my_sentry_handler, Sentry.LoggerHandler, %{
13 config: %{metadata: [:file, :line]}
14 })
15
16 # Set up LiveView trace propagation (must be BEFORE OpentelemetryPhoenix)
17 1 Sentry.OpenTelemetry.LiveViewPropagator.setup()
18 # Set up OpenTelemetry instrumentation
19 1 OpentelemetryBandit.setup()
20 1 OpentelemetryPhoenix.setup(adapter: :bandit)
21 1 OpentelemetryEcto.setup([:avy, :repo], db_statement: :enabled)
22
23 1 children =
24 [
25 AvyWeb.Telemetry,
26 Avy.Repo,
27 1 {DNSCluster, query: Application.get_env(:avy, :dns_cluster_query) || :ignore},
28 {Phoenix.PubSub, name: Avy.PubSub},
29
30 # Start the Finch HTTP client for sending emails
31 {Finch, name: Avy.Finch},
32 # Start a worker by calling: Avy.Worker.start_link(arg)
33 # {Avy.Worker, arg},
34 # Start to serve requests, typically the last entry
35 AvyWeb.Endpoint
36 ] ++ FdsnPlugs.JwkConfig.child_specs()
37
38 # See https://hexdocs.pm/elixir/Supervisor.html
39 # for other strategies and supported options
40 1 opts = [strategy: :one_for_one, name: Avy.Supervisor]
41 1 Supervisor.start_link(children, opts)
42 end
43
44 # Tell Phoenix to update the endpoint configuration
45 # whenever the application is updated.
46 @impl true
47 0 def config_change(changed, _new, removed) do
48 0 AvyWeb.Endpoint.config_change(changed, removed)
49 :ok
50 end
51 end

lib/avy/data/content.ex

100.0
1
2933
0
Line Hits Source
0 defmodule Avy.Data.Content do
1 use Ecto.Schema
2
3 2933 schema "inventory_content" do
4 field :start, :utc_datetime, source: :start
5 field :end, :utc_datetime, source: :end
6 field :quality, :integer
7 field :time_ordered, :boolean, default: false
8 field :time_spans, {:array, :map}, source: :time_spans
9 field :modified, :utc_datetime, source: :modified
10
11 many_to_many :epochs, Avy.Metadata.Epoch,
12 join_through: "inventory_content_epochs",
13 preload_order: [asc: :starttime]
14 end
15 end

lib/avy/data/content_epoch.ex

0.0
1
0
1
Line Hits Source
0 defmodule Avy.Data.ContentEpoch do
1 use Ecto.Schema
2 alias Avy.Data.Content
3 alias Avy.Metadata.Epoch
4
5 # Important: Set primary_key to false for junction tables
6 @primary_key false
7 0 schema "inventory_content_epochs" do
8 belongs_to :content, Content, primary_key: true
9 belongs_to :epoch, Epoch, primary_key: true
10 end
11 end

lib/avy/filter.ex

75.0
12
342
3
Line Hits Source
0 defmodule Avy.Filter do
1 alias FdsnPlugs.SourceIdentifier
2 require Logger
3
4 @moduledoc """
5 A filter is a structure containing all filter parameters given in an FDSN webservice request.
6
7 """
8 defstruct net: "*",
9 sta: "*",
10 loc: "*",
11 cha: "*",
12 start: DateTime.from_unix!(1),
13 end: DateTime.from_unix!(5_682_956_400),
14 quality: [:R, :D, :M, :Q],
15 includerestricted: false,
16 epochids: []
17
18 @type t() :: %Avy.Filter{
19 net: String.t(),
20 sta: String.t(),
21 loc: String.t(),
22 cha: String.t(),
23 start: DateTime.t(),
24 end: DateTime.t(),
25 quality: String.t(),
26 includerestricted: boolean,
27 epochids: list
28 }
29
30 @spec split_channel(t()) :: map()
31 def split_channel(f) do
32 38 case SourceIdentifier.split_channel(%SourceIdentifier{cha: f.cha}) do
33 {:ok, [b, i, o]} ->
34 38 %{band_code: b, instrument_code: i, orientation_code: o}
35
36 {:error, m} ->
37 0 Logger.error(inspect(m))
38 0 %{band_code: "*", instrument_code: "*", orientation_code: "*"}
39
40 _ ->
41 0 %{}
42 end
43 end
44
45 @spec from_source_identifier(SourceIdentifier.t(), boolean, list) :: t()
46 def from_source_identifier(sid, includerestricted \\ false, quality \\ [:R, :D, :M, :Q]) do
47 38 %Avy.Filter{
48 38 net: sid.net,
49 38 sta: sid.sta,
50 38 loc: sid.loc,
51 38 cha: sid.cha,
52 38 start: sid.start,
53 38 end: sid.end,
54 includerestricted: includerestricted,
55 quality: quality
56 }
57 end
58 end

lib/avy/mailer.ex

100.0
0
0
0
Line Hits Source
0 defmodule Avy.Mailer do
1 use Swoosh.Mailer, otp_app: :avy
2 end

lib/avy/metadata/epoch.ex

100.0
3
4936
0
Line Hits Source
0 defmodule Avy.Metadata.Epoch do
1 use Ecto.Schema
2
3 4934 schema "inventory_epoch" do
4 field :band_code, :string
5 field :instrument_code, :string
6 field :orientation_code, :string
7 field :location_code, :string, source: :location_code
8 field :start_date, :utc_datetime, source: :start_date
9 field :end_date, :utc_datetime, source: :end_date
10 field :policy, :string, default: "O"
11 field :sample_rate, :decimal, source: :sample_rate
12 field :modified, :utc_datetime, source: :modified
13 belongs_to :station, Avy.Metadata.Station
14
15 many_to_many :contents, Avy.Data.Content,
16 join_through: "inventory_content_epochs",
17 preload_order: [asc: :starttime]
18 end
19
20 def source_identifier(c) do
21 1 c = Avy.Repo.preload(c, station: [:network])
22
23 1 "#{c.station.network.code}_#{c.station.code}_#{c.location_code}_#{c.band_code}_#{c.instrument_code}_#{c.orientation_code}"
24 end
25 end

lib/avy/metadata/network.ex

100.0
1
3201
0
Line Hits Source
0 defmodule Avy.Metadata.Network do
1 use Ecto.Schema
2
3 3201 schema "inventory_network" do
4 field :code, :string
5 field :start_date, :utc_datetime, source: :start_date
6 field :end_date, :utc_datetime, source: :end_date
7 has_many :stations, Avy.Metadata.Station
8 end
9 end

lib/avy/metadata/station.ex

100.0
1
3235
0
Line Hits Source
0 defmodule Avy.Metadata.Station do
1 use Ecto.Schema
2
3 3235 schema "inventory_station" do
4 field :code, :string
5 field :start_date, :utc_datetime, source: :start_date
6 field :end_date, :utc_datetime, source: :end_date
7 belongs_to :network, Avy.Metadata.Network
8
9 has_many :epochs, Avy.Metadata.Epoch,
10 preload_order: [:band_code, :instrument_code, :orientation_cde, :starttime]
11 end
12 end

lib/avy/repo.ex

97.0
34
1451
1
Line Hits Source
0 defmodule Avy.Repo do
1 use Ecto.Repo,
2 otp_app: :avy,
3 adapter: Ecto.Adapters.Postgres
4
5 import Ecto.Query
6
7 alias FdsnPlugs.PublicationVersion
8 alias Avy.Metadata.{Epoch, Station, Network}
9 alias Avy.Data.Content
10
11 # Create a map of all keyword parameters.
12 # Especially, splits the channel code from filter into instrument, band, orientation codes.
13 @spec make_keywords_params(Avy.Filter.t()) :: map()
14 defp make_keywords_params(filter) do
15 Avy.Filter.split_channel(filter)
16 38 |> Map.merge(%{
17 38 net: filter.net,
18 38 sta: filter.sta,
19 38 loc: filter.loc,
20 38 includerestricted: filter.includerestricted
21 })
22 end
23
24 # NOTE: the call to split_channels seems to bother dialyzer. I was not able to fix this.
25 @doc """
26 Get all contents from database related to the Avy.Filter
27 """
28 @spec get_contents(Avy.Filter.t()) :: [map()]
29 def get_contents(filter) do
30 38 query =
31 from(c in Content)
32 38 |> join(:inner, [c], ec in "inventory_content_epochs",
33 on: c.id == ec.content_id,
34 as: :content_epochs
35 )
36 38 |> join(:inner, [c, ec], e in Epoch, on: ec.epoch_id == e.id, as: :epochs)
37 38 |> join(:inner, [c, ec, e], s in Station, on: e.station_id == s.id, as: :stations)
38 38 |> join(:inner, [c, ec, e, s], n in Network, on: s.network_id == n.id, as: :networks)
39 |> where(^conditional_where_clause(make_keywords_params(filter)))
40 |> where(
41 [c, ec, e, s, n],
42 38 c.quality in ^PublicationVersion.qualities_to_pubversions(filter.quality)
43 )
44 38 |> where([c, ec, e, s, n], c.start <= ^filter.end)
45 38 |> where([c, ec, e, s, n], c.end >= ^filter.start)
46
47 38 query =
48 38 from [c, ec, e, s, n] in query,
49 order_by: [
50 asc: e.band_code,
51 asc: e.instrument_code,
52 asc: e.orientation_code,
53 asc: e.start_date,
54 asc: c.start
55 ],
56 select: %{
57 network: n.code,
58 station: s.code,
59 location: e.location_code,
60 channel: fragment("?||?||?", e.band_code, e.instrument_code, e.orientation_code),
61 quality: c.quality,
62 samplerate: e.sample_rate,
63 38 earliest: fragment("GREATEST(?, ?) AT TIME ZONE 'utc'", c.start, ^filter.start),
64 38 latest: fragment("LEAST(?,?) AT TIME ZONE 'utc'", c.end, ^filter.end),
65 timespans: c.time_spans,
66 restriction: e.policy,
67 updated: c.modified
68 }
69
70 Avy.Repo.all(query)
71 102 |> Enum.reject(fn t -> is_nil(t) end)
72 38 |> Enum.map(&naive_to_utc(&1))
73 end
74
75 # Define all dynamic where clauses.
76 # If a parameter is a wildcard, we do not need to add the where clause.
77 @spec conditional_where_clause(map()) :: any()
78 def conditional_where_clause(keyword_params) do
79 38 Enum.reduce(keyword_params, dynamic(true), fn
80 {_k, "*"}, dynamic ->
81 # User wants all, no where clause
82 173 dynamic
83
84 {:includerestricted, true}, dynamic ->
85 2 dynamic
86
87 {:includerestricted, false}, dynamic ->
88 36 dynamic([epochs: e], ^dynamic and e.policy == "O")
89
90 {:instrument_code, value}, dynamic ->
91 4 dynamic([epochs: e], ^dynamic and like(e.instrument_code, ^fdsn_wildcard_to_sql(value)))
92
93 {:band_code, value}, dynamic ->
94 4 dynamic([epochs: e], ^dynamic and like(e.band_code, ^fdsn_wildcard_to_sql(value)))
95
96 {:orientation_code, value}, dynamic ->
97 4 dynamic([epochs: e], ^dynamic and like(e.orientation_code, ^fdsn_wildcard_to_sql(value)))
98
99 {:loc, value}, dynamic ->
100 1 dynamic([epochs: e], ^dynamic and like(e.location_code, ^fdsn_wildcard_to_sql(value)))
101
102 {:sta, value}, dynamic ->
103 36 dynamic([stations: s], ^dynamic and like(s.code, ^fdsn_wildcard_to_sql(value)))
104
105 {:net, value}, dynamic ->
106 6 dynamic([networks: n], ^dynamic and like(n.code, ^fdsn_wildcard_to_sql(value)))
107 end)
108 end
109
110 #
111 # From FDSN wildcard spec to SQL wildcards
112 @spec fdsn_wildcard_to_sql(String.t()) :: String.t()
113 defp fdsn_wildcard_to_sql(s) do
114 String.replace(s, "*", "%")
115 55 |> String.replace("?", "_")
116 end
117
118 @spec naive_to_utc(map) :: map
119 defp naive_to_utc(q) do
120 102 if Map.has_key?(q.earliest, :time_zone) do
121 0 q
122 else
123 %{
124 q
125 102 | earliest: DateTime.from_naive!(q.earliest, "Etc/UTC"),
126 102 latest: DateTime.from_naive!(q.latest, "Etc/UTC")
127 }
128 end
129 end
130 end

lib/avy_web.ex

50.0
6
21
3
Line Hits Source
0 defmodule AvyWeb do
1 @moduledoc """
2 The entrypoint for defining your web interface, such
3 as controllers, components, channels, and so on.
4
5 This can be used in your application as:
6
7 use AvyWeb, :controller
8 use AvyWeb, :html
9
10 The definitions below will be executed for every controller,
11 component, etc, so keep them short and clean, focused
12 on imports, uses and aliases.
13
14 Do NOT define functions inside the quoted expressions
15 below. Instead, define additional modules and import
16 those modules here.
17 """
18
19 7 def static_paths,
20 do: ~w(assets fonts images favicon.ico robots.txt application.wadl)
21
22 def router do
23 0 quote do
24 use Phoenix.Router, helpers: false
25
26 # Import common connection and controller functions to use in pipelines
27 import Plug.Conn
28 import Phoenix.Controller
29 end
30 end
31
32 def channel do
33 0 quote do
34 use Phoenix.Channel
35 end
36 end
37
38 def controller do
39 0 quote do
40 use Phoenix.Controller,
41 formats: [:text, :json, :geocsv, :request, :html],
42 layouts: [html: AvyWeb.Layouts]
43
44 import Plug.Conn
45
46 unquote(verified_routes())
47 end
48 end
49
50 def verified_routes do
51 7 quote do
52 use Phoenix.VerifiedRoutes,
53 endpoint: AvyWeb.Endpoint,
54 router: AvyWeb.Router,
55 statics: AvyWeb.static_paths()
56 end
57 end
58
59 @doc """
60 When used, dispatch to the appropriate controller/live_view/etc.
61 """
62 defmacro __using__(which) when is_atom(which) do
63 7 apply(__MODULE__, which, [])
64 end
65 end

lib/avy_web/avy_geoscv.ex

97.6
43
674
1
Line Hits Source
0 defmodule AvyWeb.AvyGEOCSV do
1 require Logger
2
3 @extent_header "#dataset: GeoCSV 2.0
4 #delimiter: |
5 #field_unit: unitless|unitless|unitless|unitless|unitless|hertz|ISO_8601|ISO_8601|ISO_8601|unitless|unitless
6 #field_type: string|string|string|string|string|float|datetime|datetime|datetime|integer|string
7 Network|Station|Location|Channel|Quality|SampleRate|Earliest|Latest|Updated|TimeSpans|Restriction"
8
9 0 def extent_post(assigns), do: extent(assigns)
10
11 def extent(assigns) do
12 4 Logger.debug(assigns.availabilities, pretty: true)
13
14 [
15 format_header(@extent_header, assigns)
16 4 | Enum.map(assigns.availabilities, fn d ->
17 12 format_datasource(d, assigns.fdsn_parameters, :extent)
18 end)
19 ]
20 4 |> Enum.join("\n")
21 end
22
23 @query_header "#dataset: GeoCSV 2.0
24 #delimiter: |
25 #field_unit: unitless|unitless|unitless|unitless|unitless|hertz|ISO_8601|ISO_8601|ISO_8601
26 #field_type: string|string|string|string|string|float|datetime|datetime|datetime
27 Network|Station|Location|Channel|Quality|SampleRate|Earliest|Latest|Updated"
28 def query(assigns) do
29 [
30 format_header(@query_header, assigns)
31 4 | Enum.map(assigns.availabilities, fn d ->
32 6 Enum.map(d.timespans, fn ts ->
33 22 %{d | timespans: [ts], earliest: List.first(ts), latest: List.last(ts)}
34 end)
35 6 |> Enum.map(fn splitd ->
36 22 format_datasource(
37 splitd,
38 22 assigns.fdsn_parameters,
39 :query
40 )
41 end)
42 end)
43 ]
44 |> List.flatten()
45 4 |> Enum.join("\n")
46 end
47
48 defp format_header(headers, %{fdsn_parameters: fdsn_params}) do
49 8 headers =
50 8 if :samplerate in fdsn_params.merge do
51 String.replace(headers, "hertz|", "")
52 |> String.replace("float|", "")
53 3 |> String.replace("SampleRate|", "")
54 else
55 5 headers
56 end
57
58 8 headers =
59 8 if :quality in fdsn_params.merge do
60 String.replace(headers, "unitless|", "", global: false)
61 |> String.replace("string|", "", global: false)
62 2 |> String.replace("Quality|", "")
63 else
64 6 headers
65 end
66
67 8 if :latestupdate in fdsn_params.show do
68 4 headers
69 else
70 String.replace(headers, "|ISO_8601", "", global: false)
71 |> String.replace("|datetime", "", global: false)
72 4 |> String.replace("|Updated", "")
73 end
74 end
75
76 # Network|Station|Location|Channel|Quality|SampleRate|Earliest|Latest|Updated|TimeSpans|Restriction
77 defp format_datasource(d, fdsn_params, method) do
78 34 attr_list =
79 34 if method == :extent do
80 12 [d.timespancount, d.restriction]
81 else
82 []
83 end
84
85 34 attr_list =
86 34 if :latestupdate in fdsn_params.show do
87 [
88 12 DateTime.to_iso8601(d.earliest),
89 12 DateTime.to_iso8601(d.latest),
90 12 DateTime.to_iso8601(d.updated) | attr_list
91 ]
92 else
93 [
94 22 DateTime.to_iso8601(d.earliest),
95 22 DateTime.to_iso8601(d.latest)
96 | attr_list
97 ]
98 end
99
100 34 attr_list =
101 34 if :samplerate in fdsn_params.merge do
102 14 attr_list
103 else
104 20 [d.samplerate | attr_list]
105 end
106
107 34 attr_list =
108 34 if :quality in fdsn_params.merge do
109 12 attr_list
110 else
111 22 [d.quality | attr_list]
112 end
113
114 34 [d.network, d.station, d.location, d.channel | attr_list]
115 34 |> Enum.join("|")
116 end
117 end

lib/avy_web/avy_json.ex

95.6
23
128
1
Line Hits Source
0 defmodule AvyWeb.AvyJSON do
1 require Logger
2
3 def version(_) do
4 0 %{
5 application: Application.spec(:avy)[:app],
6 version: Application.spec(:avy)[:vsn],
7 commit: System.get_env("SENTRY_RELEASE", "unspecified")
8 }
9 end
10
11 def extent(assigns) do
12 5 %{
13 created: DateTime.utc_now(),
14 version: 1.0,
15 5 datasources: assigns.availabilities
16 }
17 end
18
19 def query(assigns) do
20 6 %{
21 created: DateTime.utc_now(),
22 version: 1.0,
23 6 datasources: assigns.availabilities
24 }
25 end
26
27 @doc """
28 This view is meant to output a json format for grafana "state timeline" graphs.
29 It is very different from the FDSN output and therefore needs a lot of manipulations.
30
31 The final structure is a list of maps with the key :timestamp as a date,
32 and all FDSN source identifiers in the response as keys, with default value "".
33
34 At each timestamp, we want to put for each FDSN source identifier a quality code or
35 an empty string if there is no data.
36
37 [
38 %{ timeline: ~U[2026-01-01 10:00:00Z],
39 "FR_CIEL_00_HNZ" => "D"
40 }
41 }
42 """
43 def statetimeline(assigns) do
44 # First get all FDSN identifiers in the availabilities and build the default map of identifiers.
45 1 fdsn_identifiers =
46 1 assigns.availabilities
47 4 |> Enum.reduce([], fn a, acc ->
48 [availability_to_fdsn_source_identifier(a) | acc]
49 end)
50 |> Enum.sort()
51 |> Enum.dedup()
52 |> Map.from_keys("")
53
54 # Then build a list of timestamps
55 # All elements in the timespans list must result in a new entry in the list
56 # In the same time, we add the correct FDSN identifier and quality in the map.
57 # The list needs to be sorted by DateTime
58 1 timeline_list =
59 1 assigns.availabilities
60 |> Enum.reduce([], fn a, acc ->
61 [
62 4 elem(
63 Enum.reduce(
64 4 a.timespans,
65 4 {availability_to_fdsn_source_identifier(a), a.quality, []},
66 fn [ts_start, ts_end], {fdsnid, qual, acc} ->
67 10 {fdsnid, qual,
68 [
69 %{fdsnid => "", "Timestamps" => ts_end},
70 %{fdsnid => qual, "Timestamps" => ts_start}
71 ] ++ acc}
72 end
73 ),
74 2
75 )
76 4 ] ++ acc
77 end)
78 |> List.flatten()
79 20 |> Enum.sort_by(&Map.fetch!(&1, "Timestamps"), DateTime)
80
81 1 Logger.debug(inspect(timeline_list, pretty: true))
82
83 # Then we need to fill the struct with all other missing fdsn identifiers.
84 # For each timeline, the quality value must be the same as in the previous
85 # timeline. Therefore, we need to know which it was.
86 1 initial_tl = Map.merge(fdsn_identifiers, List.first(timeline_list))
87
88 Enum.reduce(timeline_list, {initial_tl, [initial_tl]}, fn tl, {prev_tl, acc} ->
89 20 full_tl = Map.merge(prev_tl, tl)
90
91 20 if Map.fetch!(tl, "Timestamps") == Map.fetch!(prev_tl, "Timestamps") do
92 {full_tl, List.replace_at(acc, 0, full_tl)}
93 else
94 {full_tl, [full_tl | acc]}
95 end
96 end)
97 1 |> elem(1)
98 1 |> Enum.reverse()
99 end
100
101 defp availability_to_fdsn_source_identifier(a) do
102 8 "#{a.network}_#{a.station}_#{a.location}_#{a.channel}"
103 end
104 end

lib/avy_web/avy_request.ex

100.0
13
53
0
Line Hits Source
0 defmodule AvyWeb.AvyREQUEST do
1 @doc """
2 Request output format
3 """
4
5 def extent(assigns) do
6 1 Enum.map_join(assigns.availabilities, "\n", fn d ->
7 2 format_datasource(d)
8 end)
9 end
10
11 def query(assigns) do
12 1 Enum.map(assigns.availabilities, fn d ->
13 2 Enum.map(d.timespans, fn ts ->
14 4 %{d | earliest: List.first(ts), latest: List.last(ts)}
15 end)
16 end)
17 |> List.flatten()
18 1 |> Enum.map_join("\n", fn d -> format_datasource(d) end)
19 end
20
21 defp format_datasource(d) do
22 [
23 6 d.network,
24 6 d.station,
25 6 d.location,
26 6 d.channel,
27 6 DateTime.to_iso8601(d.earliest),
28 6 DateTime.to_iso8601(d.latest)
29 ]
30 6 |> Enum.join(" ")
31 end
32 end

lib/avy_web/avy_text.ex

100.0
49
708
0
Line Hits Source
0 defmodule AvyWeb.AvyTEXT do
1 @moduledoc """
2 Text format renderer
3 """
4
5 @extent_header [
6 "Network",
7 "Station",
8 "Location",
9 "Channel",
10 "Quality",
11 "SampleRate",
12 String.pad_trailing("Earliest", 27),
13 String.pad_trailing("Latest", 27),
14 String.pad_trailing("Updated", 20),
15 "TimeSpans",
16 "Restriction"
17 ]
18
19 @doc """
20 Text output
21 """
22
23 def version(_) do
24 2 "#{Application.spec(:avy)[:vsn]}"
25 end
26
27 def extent(assigns) do
28 8 headers = format_header(@extent_header, assigns)
29
30 [
31 headers
32 8 | Enum.map(assigns.availabilities, fn d ->
33 13 format_datasource(d, assigns.fdsn_parameters, :extent)
34 end)
35 ]
36 8 |> Enum.join("\n")
37 end
38
39 @query_header [
40 "Network",
41 "Station",
42 "Location",
43 "Channel",
44 "Quality",
45 "SampleRate",
46 String.pad_trailing("Earliest", 27),
47 String.pad_trailing("Latest", 27),
48 "Updated"
49 ]
50 def query(assigns) do
51 5 headers = format_header(@query_header, assigns)
52
53 [
54 headers
55 5 | Enum.map(assigns.availabilities, fn d ->
56 9 Enum.map(d.timespans, fn ts ->
57 13 %{d | timespans: [ts], earliest: List.first(ts), latest: List.last(ts)}
58 end)
59 9 |> Enum.map(fn splitd ->
60 13 format_datasource(
61 splitd,
62 13 assigns.fdsn_parameters,
63 :query
64 )
65 end)
66 end)
67 ]
68 |> List.flatten()
69 5 |> Enum.join("\n")
70 end
71
72 defp format_header(headers, %{fdsn_parameters: fdsn_params} = _) do
73 13 headers =
74 13 if :samplerate in fdsn_params.merge do
75 2 Enum.reject(headers, &String.starts_with?(&1, "SampleRate"))
76 else
77 11 headers
78 end
79
80 13 headers =
81 13 if :quality in fdsn_params.merge do
82 2 Enum.reject(headers, &String.starts_with?(&1, "Quality"))
83 else
84 11 headers
85 end
86
87 13 headers =
88 13 if :latestupdate in fdsn_params.show do
89 12 headers
90 else
91 1 Enum.reject(headers, &String.starts_with?(&1, "Updated"))
92 end
93
94 13 "#" <> Enum.join(headers, " ")
95 end
96
97 defp format_datasource(d, fdsn_params, method) do
98 26 attr_list =
99 13 if method == :extent do
100 13 [d.timespancount |> Integer.to_string() |> String.pad_trailing(9), d.restriction]
101 else
102 []
103 end
104
105 26 attr_list =
106 26 if :latestupdate in fdsn_params.show do
107 [
108 25 DateTime.to_iso8601(d.earliest),
109 25 DateTime.to_iso8601(d.latest),
110 25 DateTime.to_iso8601(d.updated) | attr_list
111 ]
112 else
113 [
114 1 DateTime.to_iso8601(d.earliest),
115 1 DateTime.to_iso8601(d.latest)
116 | attr_list
117 ]
118 end
119
120 26 attr_list =
121 26 if :samplerate in fdsn_params.merge do
122 3 attr_list
123 else
124 [
125 23 d.samplerate
126 |> Decimal.to_string(:xsd)
127 |> String.pad_trailing(10)
128 | attr_list
129 ]
130 end
131
132 26 attr_list =
133 26 if :quality in fdsn_params.merge do
134 2 attr_list
135 else
136 24 [String.pad_trailing(d.quality, 7) | attr_list]
137 end
138
139 [
140 26 String.pad_trailing(d.network, 8),
141 26 String.pad_trailing(d.station, 7),
142 26 String.pad_trailing(d.location, 8),
143 26 String.pad_trailing(d.channel, 7) | attr_list
144 ]
145 26 |> Enum.join(" ")
146 end
147 end

lib/avy_web/controllers/avy_controller.ex

90.9
154
5176
14
Line Hits Source
0 defmodule AvyWeb.AvyController do
1 @moduledoc """
2 Manages a user request.
3 """
4 alias Avy.Repo
5 require Logger
6
7 use AvyWeb, :controller
8 alias FdsnPlugs.PublicationVersion
9 use OpenApiSpex.ControllerSpecs
10
11 plug :forbidden_params_extent, ["mergegaps"] when action in [:extent]
12 plug :forbidden_params_extent, ["show"] when action in [:extent]
13 plug :format
14 plug AvyWeb.Plugs.Filters
15
16 ### Define the OpenApi spec
17 0 operation :version,
18 summary: "Get the version number of the service"
19
20 0 operation :extent,
21 summary: "Query the datacenter for available waveform data.",
22 description: "Returns only the earliest and latest data.",
23 parameters: Avy.ApiSpec.parameters_spec(),
24 responses: Avy.ApiSpec.responses_spec()
25
26 0 operation :extent_post,
27 summary: "Query the datacenter for available waveform data.",
28 description: "Returns only the earliest and latest data.",
29 request_body: Avy.ApiSpec.request_body_spec(),
30 responses: Avy.ApiSpec.responses_spec()
31
32 0 operation :query,
33 summary: "Query the datacenter for available waveform data.",
34 description: "Returns details of all segments of data",
35 parameters: Avy.ApiSpec.query_parameters_spec(),
36 responses: Avy.ApiSpec.responses_spec()
37
38 0 operation :query_post,
39 summary: "Query the datacenter for available waveform data",
40 request_body: Avy.ApiSpec.request_body_spec(),
41 responses: Avy.ApiSpec.responses_spec()
42
43 ### End of OpenApi definition
44
45 def version(conn, _params) do
46 2 render(conn, :version)
47 end
48
49 # It seems that I need to define this placeholder function in order to be able to define a specific
50 # operation for POST extent in OpenApi
51 1 def extent_post(%Plug.Conn{method: "POST"} = conn, params), do: extent(conn, params)
52
53 def extent(conn, _params) do
54 20 case manage_request(conn, :extent) do
55 {:ok, []} ->
56 conn
57 2 |> send_resp(conn.assigns.fdsn_parameters.nodata, "")
58 2 |> halt
59
60 {:ok, ds} ->
61 18 datasources = Enum.map(ds, &Map.drop(&1, [:timespans]))
62
63 conn
64 |> assign(:availabilities, datasources)
65 18 |> render(:extent)
66
67 {:too_much_data, msg} ->
68 0 conn |> send_resp(413, msg) |> halt()
69 end
70 end
71
72 @doc """
73 This controller exists to reformat the output suitable for a grafana state timeline
74
75 All the formatting logic takes place in the render function AvyJson.statetimeline/1
76 """
77 def statetimeline(conn, opts) do
78 conn
79 |> put_format("json")
80 |> put_resp_content_type("application/json")
81 1 |> query(opts)
82 end
83
84 def query(conn, _params) do
85 18 case manage_request(conn, :query) do
86 {:ok, []} ->
87 conn
88 1 |> send_resp(conn.assigns.fdsn_parameters.nodata, "")
89 1 |> halt
90
91 {:ok, ds} ->
92 17 datasources = Enum.map(ds, &Map.drop(&1, [:timespancount]))
93
94 17 Logger.info("Fetched #{length(datasources)} traces")
95
96 conn
97 |> assign(:availabilities, datasources)
98 17 |> render()
99
100 {:too_much_data, msg} ->
101 0 conn |> send_resp(413, msg) |> halt()
102 end
103 end
104
105 @doc """
106 This function is called on a /extent request.
107 It will fetch all the contents corresponding to the filters that were set up by
108 AvyWeb.Plugs.Filters, in the assigns of the connection plug (`conn.assigns.filters`).
109 For each filter, the database request is done asynchronously in parallel.
110 Desactivate the timeout, as the DB pool has it's own already, and some requests can get quite long.
111
112 For each entry, we call the post_process function in order to build a list of datasources and merge the contents of the same properties.
113
114 Finally, the datasources are put in the conn.assigns in order for ther renderer to access them in the `:availabilities` keyword.
115 """
116 @spec manage_request(Plug.Conn.t(), atom) :: {:ok, list} | {:too_much_data, String.t()}
117 def manage_request(conn, method) do
118 38 datasources =
119 38 Task.async_stream(conn.assigns.filters, Repo, :get_contents, [], timeout: :infinity)
120 38 |> Enum.flat_map(fn {:ok, f} -> f end)
121
122 # TODO Improve evaluation of too much data
123 38 if length(datasources) > Application.get_env(:avy, :max_sids) do
124 {:too_much_data,
125 0 "Request has #{length(datasources)} epochs. Try to restrict the source identifiers selection and/or select smaller timespans."}
126 else
127 38 datasources =
128 datasources
129 |> post_process(
130 method,
131 38 conn.assigns.fdsn_parameters.merge,
132 38 conn.assigns.fdsn_parameters.mergegaps * 10 ** 9
133 )
134 66 |> Enum.sort_by(&{&1.network, &1.station, &1.location, &1.channel, &1.earliest})
135 38 |> orderby(conn.assigns.fdsn_parameters.orderby, method)
136 38 |> to_gaps(Map.has_key?(conn.params, "showgaps"))
137
138 {:ok, datasources}
139 end
140 end
141
142 @doc """
143
144 This function will convert a list of ordered datasources to
145 show gaps.
146
147 First, group all datasources by the full keys.
148 Then for each list:
149 - keep the previous end date. (Start first iteration at second element)
150 - on the current element:
151 - set the start time at previous end date
152 - set then end time at current start.
153 - put the result at the head of the list we are building
154 - we obtain a reversed list, so at the end we need to reverse it.
155 """
156
157 @spec to_gaps(list(), boolean) :: list()
158 38 def to_gaps(datasources, false), do: datasources
159
160 def to_gaps(datasources, true) do
161 1 Logger.info("Computing gaps")
162
163 datasources
164 |> Enum.group_by(group_by_keys([], :query))
165 |> Enum.map(fn {_, contents} ->
166 1 Enum.map(contents, fn content ->
167 2 Logger.debug(inspect(content, pretty: true))
168 2 [[_, init_end] | tail] = content.timespans
169
170 2 %{prev_end: _, list: gaps} =
171 Enum.reduce(tail, %{prev_end: init_end, list: []}, fn [tss, tse], acc ->
172 2 %{
173 prev_end: tse,
174 2 list: [[acc.prev_end, tss] | acc.list]
175 }
176 end)
177
178 2 Logger.debug(inspect(gaps))
179 2 %{content | timespans: Enum.reverse(gaps)}
180 end)
181 end)
182 |> List.flatten()
183 1 |> Enum.reject(fn a -> a.timespans == [] end)
184 end
185
186 #### Private functions for query requests
187
188 # post_process/4 contains all the logic to handle the contents retrieved in the Repo.
189 # Convert the structure returned by the Repo in a usable list of datasources, fit for the views
190 # Post process the structure returned by the Repo call in order to:
191 # - Group all elements by their discriminating properties (including restriction status)
192 # - For each group,
193 # - clean the unwanted timespans in contents
194 # - merge consecutive timespans either by using the mergegap provided in the request, or by the samplerate
195 # With the result, prepare a structure suitable for the views:
196 # - compute the number of timespans
197 # - convert publication version to quality code
198 #
199 @spec post_process(list, atom, list, integer) :: list
200 defp post_process(lst, method, merge_properties, mergegap) do
201 lst
202 # Group all the contents by a key, depending on the user request
203 |> Enum.group_by(group_by_keys(merge_properties, method))
204 # With the grouped contents, do the post processing.
205 |> Task.async_stream(fn {k, contents} ->
206 66 Logger.info("Processing #{length(contents)} contents for #{inspect(k)}")
207 66 contents = filter_outofbound_timespans(contents, List.first(contents).earliest)
208
209 66 if length(contents) > 0 do
210 66 ref_content = List.first(contents)
211
212 66 ref_samplerate =
213 66 Enum.min_by(contents, & &1.samplerate) |> Map.fetch!(:samplerate) |> Decimal.to_float()
214
215 66 Logger.debug("Min samplerate: #{ref_samplerate}")
216
217 # When we merge contiguous timespans based on the samplerate
218 66 mergegap =
219 if mergegap <= 0 do
220 # When no mergegaps are indicated
221 52 10 ** 9 / ref_samplerate
222 else
223 14 mergegap
224 end
225
226 66 Logger.debug("Merging on gaps lower than #{mergegap}")
227
228 66 merged_timespans =
229 102 Enum.flat_map(contents, & &1.timespans)
230 |> merge_contiguous_timespans(mergegap)
231 142 |> Enum.map(fn x ->
232 [
233 Map.fetch!(x, "start") |> DateTime.from_unix!(:nanosecond),
234 Map.fetch!(x, "end") |> DateTime.from_unix!(:nanosecond)
235 ]
236 end)
237 66 |> snip_timespans(ref_content.earliest, List.last(contents).latest)
238
239 66 %{
240 66 network: ref_content.network,
241 66 station: ref_content.station,
242 66 location: ref_content.location,
243 66 channel: ref_content.channel,
244 66 quality: PublicationVersion.pubversion_to_quality!(ref_content.quality),
245 66 samplerate: ref_content.samplerate,
246 66 earliest: contents |> List.first() |> Map.fetch!(:earliest),
247 66 latest: contents |> List.last() |> Map.fetch!(:latest),
248 66 updated: Enum.max_by(contents, & &1.updated, Date) |> Map.fetch!(:updated),
249 restriction:
250 66 case ref_content.restriction do
251 64 "O" -> "OPEN"
252 2 _ -> "RESTRICTED"
253 end,
254 102 timespancount: Enum.reduce(contents, 0, fn c, acc -> acc + length(c.timespans) end),
255 timespans: merged_timespans
256 }
257 else
258 nil
259 end
260 end)
261 66 |> Enum.into([], fn {:ok, res} -> res end)
262 38 |> Enum.reject(&is_nil(&1))
263 end
264
265 #
266 # After grouping all contents by epochs,
267 # In the timespans, there can be some contents that are outside of the request boundaries.
268 # This function will remove contents that are completely out of bound
269 # And if a content is left empty, we remove it from the list.
270 defp filter_outofbound_timespans(contents, lowerbound) do
271 contents
272 |> Enum.map(fn content ->
273 102 lower_bound = DateTime.to_unix(lowerbound, :nanosecond)
274
275 %{
276 content
277 102 | timespans:
278 102 Enum.reject(content.timespans, fn ts ->
279 154 if ts["end"] <= lower_bound do
280 0 Logger.debug(
281 0 "Rejecting #{DateTime.from_unix!(ts["end"], :nanosecond)} < #{content.earliest}"
282 )
283
284 true
285 end
286 end)
287 }
288 end)
289 66 |> Enum.reject(fn content ->
290 102 if content.timespans == [] do
291 0 Logger.debug("Rejecting empty timespans in #{inspect(content)}")
292 true
293 end
294 end)
295 end
296
297 defp snip_timespans(timespans, earliest, latest) do
298 66 Logger.debug("Snip #{inspect(timespans, pretty: true)}")
299 66 Logger.debug("Over #{earliest} and #{latest}")
300
301 66 [[ts_start, ts_end] | tail] = timespans
302
303 66 snipped_timespans_at_start =
304 1 if DateTime.before?(ts_start, earliest) do
305 # TODO Faire un test qui arrive ici
306 [[earliest, ts_end] | tail]
307 else
308 65 timespans
309 end
310
311 66 [[ts_start, ts_end] | tail] =
312 Enum.reverse(snipped_timespans_at_start)
313 67 |> Enum.drop_while(fn [ts, _] -> DateTime.after?(ts, latest) end)
314
315 66 if DateTime.after?(ts_end, latest) do
316 4 [[ts_start, latest] | tail] |> Enum.reverse()
317 else
318 62 snipped_timespans_at_start
319 end
320 end
321
322 defp merge_contiguous_timespans(timespans, mergegap) do
323 Enum.reduce(
324 timespans,
325 Enum.take(timespans, 1),
326 fn timespan, acc ->
327 154 [previous | tail] = acc
328
329 154 if timespan["start"] - previous["end"] <= mergegap do
330 78 Logger.debug(
331 0 "Merging timespan #{previous["start"] |> DateTime.from_unix!(:nanosecond)} -> #{previous["end"] |> DateTime.from_unix!(:nanosecond)}"
332 )
333
334 78 Logger.debug(
335 0 "and #{timespan["start"] |> DateTime.from_unix!(:nanosecond)} -> #{timespan["end"] |> DateTime.from_unix!(:nanosecond)}"
336 )
337
338 78 Logger.debug("with gap #{timespan["start"] - previous["end"]} <= #{mergegap}")
339
340 [
341 %{
342 "start" => previous["start"],
343 "end" => timespan["end"],
344 "sample_rate" => previous["sample_rate"]
345 }
346 | tail
347 ]
348 else
349 [timespan | acc]
350 end
351 end
352 )
353 66 |> Enum.reverse()
354 end
355
356 # Plug function: In /extent action, mergegaps and show are not allowed
357 defp forbidden_params_extent(conn, [opt]) do
358 41 Logger.debug(
359 0 "Check for forbidden params for query whith #{inspect(conn.params, pretty: true)}"
360 )
361
362 41 case Map.fetch(conn.params, opt) do
363 {:ok, _} ->
364 2 send_resp(conn, 400, "Options #{opt} is only supported in the /query method.")
365 2 |> halt
366
367 :error ->
368 39 conn
369 end
370 end
371
372 @spec group_by_keys(list(atom), :query | :extent) :: function
373 defp group_by_keys(merge_properties, :extent) do
374 20 cond do
375 20 :quality in merge_properties and :samplerate in merge_properties ->
376 2 fn a ->
377 6 {a.network, a.station, a.location, a.channel, a.restriction}
378 end
379
380 18 :samplerate in merge_properties ->
381 2 fn a ->
382 6 {a.network, a.station, a.location, a.channel, a.quality, a.restriction}
383 end
384
385 16 :quality in merge_properties ->
386 4 fn a ->
387 15 {a.network, a.station, a.location, a.channel, a.samplerate, a.restriction}
388 end
389
390 # Default behaviour
391 12 true ->
392 12 fn a ->
393 29 {a.network, a.station, a.location, a.channel, a.quality, a.samplerate, a.restriction}
394 end
395 end
396 end
397
398 defp group_by_keys(merge_properties, :query) do
399 19 cond do
400 19 :quality in merge_properties and :samplerate in merge_properties ->
401 1 fn a ->
402 3 {a.network, a.station, a.location, a.channel}
403 end
404
405 18 :samplerate in merge_properties ->
406 1 fn a ->
407 3 {a.network, a.station, a.location, a.channel, a.quality}
408 end
409
410 17 :quality in merge_properties ->
411 1 fn a ->
412 3 {a.network, a.station, a.location, a.channel, a.samplerate}
413 end
414
415 # Default behaviour
416 16 true ->
417 16 fn a ->
418 39 {a.network, a.station, a.location, a.channel, a.quality, a.samplerate}
419 end
420 end
421 end
422
423 #  The API uses the "format" parameter on order to set the output format.
424 # Translate this to the _format param for Phoenix magic to take place.
425 # Also force the response content type header to text
426 defp format(conn, _) do
427 40 with {:ok, _} <- Map.fetch(conn.assigns, :fdsn_parameters),
428 38 {:ok, format} <- Map.fetch(conn.assigns.fdsn_parameters, :format) do
429 38 case format do
430 :json ->
431 conn
432 |> Phoenix.Controller.put_format("json")
433 12 |> Plug.Conn.put_resp_content_type("application/json")
434
435 :geocsv ->
436 conn
437 |> Phoenix.Controller.put_format("geocsv")
438 8 |> Plug.Conn.put_resp_content_type("text/csv")
439
440 :request ->
441 conn
442 |> Phoenix.Controller.put_format("request")
443 2 |> Plug.Conn.put_resp_content_type("text/plain")
444
445 _ ->
446 conn
447 |> Phoenix.Controller.put_format("text")
448 16 |> Plug.Conn.put_resp_content_type("text/plain")
449 end
450 else
451 _ ->
452 conn
453 |> Phoenix.Controller.put_format("text")
454 2 |> Plug.Conn.put_resp_content_type("text/plain")
455 end
456 end
457
458 defp orderby(ds, orderby, :extent) do
459 20 Logger.debug("Reorder datasources by #{orderby}")
460
461 20 case orderby do
462 1 :latestupdate -> Enum.sort_by(ds, & &1.updated, {:asc, DateTime})
463 1 :latestupdate_desc -> Enum.sort_by(ds, & &1.updated, {:desc, DateTime})
464 1 :timespancount -> Enum.sort_by(ds, & &1.timespancount, :asc)
465 1 :timespancount_desc -> Enum.sort_by(ds, & &1.timespancount, :desc)
466 16 _ -> ds
467 end
468 end
469
470 defp orderby(ds, orderby, :query) do
471 18 Logger.debug("Reorder datasources by #{orderby}")
472
473 18 case orderby do
474 2 :latestupdate -> Enum.sort_by(ds, & &1.updated, {:asc, DateTime})
475 2 :latestupdate_desc -> Enum.sort_by(ds, & &1.updated, {:desc, DateTime})
476 14 _ -> ds
477 end
478 end
479 end

lib/avy_web/controllers/error_json.ex

100.0
1
2
0
Line Hits Source
0 defmodule AvyWeb.ErrorJSON do
1 @moduledoc """
2 This module is invoked by your endpoint in case of errors on JSON requests.
3
4 See config/config.exs.
5 """
6
7 # If you want to customize a particular status code,
8 # you may add your own clauses, such as:
9 #
10 # def render("500.json", _assigns) do
11 # %{errors: %{detail: "Internal Server Error"}}
12 # end
13
14 # By default, Phoenix returns the status message from
15 # the template name. For example, "404.json" becomes
16 # "Not Found".
17 def render(template, _assigns) do
18 2 %{errors: %{detail: Phoenix.Controller.status_message_from_template(template)}}
19 end
20 end

lib/avy_web/endpoint.ex

66.6
3
104
1
Line Hits Source
0 defmodule AvyWeb.Endpoint do
1 use Sentry.PlugCapture
2 use Phoenix.Endpoint, otp_app: :avy
3
4 # The session will be stored in the cookie and signed,
5 # this means its contents can be read but not tampered with.
6 # Set :encryption_salt if you would also like to encrypt it.
7 @session_options [
8 store: :cookie,
9 key: "_avy_key",
10 signing_salt: "xyXumY7h",
11 same_site: "Lax"
12 ]
13
14 socket "/live", Phoenix.LiveView.Socket,
15 websocket: [connect_info: [session: @session_options]],
16 longpoll: [connect_info: [session: @session_options]]
17
18 # Serve at "/" the static files from "priv/static" directory.
19 #
20 # You should set gzip to true if you are running phx.digest
21 # when deploying your static files in production.
22 plug Plug.Static,
23 at: "/",
24 from: :avy,
25 gzip: true,
26 content_types: %{"application.wadl" => "text/xml"},
27 only: AvyWeb.static_paths()
28
29 # Answer to /_health request
30 plug AvyWeb.Plug.HealthCheck
31
32 # Code reloading can be explicitly enabled under the
33 # :code_reloader configuration of your endpoint.
34 if code_reloading? do
35 plug Phoenix.CodeReloader
36 plug Phoenix.Ecto.CheckRepoStatus, otp_app: :avy
37 end
38
39 plug Phoenix.LiveDashboard.RequestLogger,
40 param_key: "request_logger",
41 cookie_key: "request_logger"
42
43 plug Plug.RequestId
44 plug Plug.Telemetry, event_prefix: [:phoenix, :endpoint], log: {__MODULE__, :log_level, []}
45
46 plug Plug.Parsers,
47 parsers: [:urlencoded, :multipart, :json],
48 pass: ["*/*"],
49 json_decoder: Phoenix.json_library()
50
51 plug Sentry.PlugContext
52 plug Plug.MethodOverride
53 plug Plug.Head
54 plug Plug.Session, @session_options
55
56 plug AvyWeb.Plug.TrafficDrain
57 plug AvyWeb.Router
58 #
59 # Return `false` for the __health endpoint
60 0 def log_level(%{path_info: ["__health" | _]}), do: false
61 2 def log_level(%{path_info: ["__traffic" | _]}), do: false
62 # Default to the normal loglevel
63 102 def log_level(_), do: :info
64 end

lib/avy_web/plugs/filters.ex

100.0
7
272
0
Line Hits Source
0 defmodule AvyWeb.Plugs.Filters do
1 @moduledoc """
2 A plug to add a Filter struct to the connection assigns.
3 The filter is made of the source identifier + includerestricted + quality
4 NOTE: this is maybe redundant with FdsnPlugs.RequestParams but is more suited to filtering by Avy.Repo
5 Refactoring may be.
6 """
7 import Plug.Conn
8
9 40 def init(opts), do: opts
10
11 def call(conn, _opts) do
12 conn
13 40 |> assign(
14 :filters,
15 40 if Map.has_key?(conn.assigns, :fdsn_parameters) do
16 38 Enum.map(conn.assigns.fdsn_parameters.source_identifiers, fn s ->
17 38 Avy.Filter.from_source_identifier(
18 s,
19 38 conn.assigns.fdsn_parameters.includerestricted,
20 38 conn.assigns.fdsn_parameters.quality
21 )
22 end)
23 else
24 []
25 end
26 )
27 end
28 end

lib/avy_web/plugs/health_check.ex

100.0
3
106
0
Line Hits Source
0 defmodule AvyWeb.Plug.HealthCheck do
1 import Plug.Conn
2
3 53 def init(opts), do: opts
4
5 # If the request path matches "/_health", we return a 200 response.
6 def call(%Plug.Conn{request_path: "/__health"} = conn, _opts) do
7 conn
8 |> send_resp(
9 200,
10 ""
11 )
12 1 |> halt()
13 end
14
15 # If the request path is anything else, we pass the connection along.
16 52 def call(conn, _opts), do: conn
17 end

lib/avy_web/plugs/traffic_drain.ex

80.0
5
105
1
Line Hits Source
0 defmodule AvyWeb.Plug.TrafficDrain do
1 @moduledoc """
2 Plug for handling Kubernetes readinessProbe.
3
4 Plug starts responding with 503 - Service Unavailable from `/__traffic`, when traffic is being drained.
5 Otherwise we respond with 200 - OK.
6 """
7
8 import Plug.Conn
9
10 @behaviour Plug
11
12 @impl true
13 52 def init(opts), do: opts
14
15 @impl true
16 def call(%Plug.Conn{path_info: ["__traffic"]} = conn, _opts) do
17 1 case GracefulStop.get_status() do
18 :stopping ->
19 conn
20 |> put_resp_content_type("text/plain")
21 |> send_resp(:service_unavailable, "Draining")
22 0 |> halt()
23
24 :running ->
25 conn
26 |> put_resp_content_type("text/plain")
27 |> send_resp(:ok, "Serving")
28 1 |> halt()
29 end
30 end
31
32 @impl true
33 def call(conn, _opts) do
34 51 conn
35 end
36 end

lib/avy_web/router.ex

81.8
11
100
2
Line Hits Source
0 defmodule AvyWeb.Router do
1 use AvyWeb, :router
2
3 48 pipeline :fdsn do
4 plug :accepts, ["json", "text"]
5
6 plug FdsnAvailabilityPlugs
7 end
8
9 0 pipeline :openapi do
10 plug OpenApiSpex.Plug.PutApiSpec, module: Avy.ApiSpec
11 end
12
13 1 pipeline :browser do
14 plug :accepts, ["html"]
15 plug :fetch_session
16 # plug :fetch_live_flash
17 plug :put_root_layout, html: {AvyWeb.Layouts, :root}
18 plug :protect_from_forgery
19 plug :put_secure_browser_headers
20 plug Sentry.Plug.LiveViewContext
21 end
22
23 2 get "/version", AvyWeb.AvyController, :version
24
25 scope "/", AvyWeb do
26 pipe_through :fdsn
27 26 get "/extent", AvyController, :extent
28 4 post "/extent", AvyController, :extent_post
29 15 get "/query", AvyController, :query
30 2 post "/query", AvyController, :query
31 1 get "/statetimeline", AvyController, :statetimeline
32 end
33
34 scope "/" do
35 pipe_through :browser
36
37 # To serve the UI, a static path has to be given at compile time.
38 # I declare here the official FDSN location it should work in
39 # most cases.
40 1 get "/", OpenApiSpex.Plug.SwaggerUI, path: Application.compile_env(:avy, :open_api_path)
41 end
42
43 scope "/" do
44 pipe_through :openapi
45 0 get "/openapi", OpenApiSpex.Plug.RenderSpec, []
46 end
47
48 # Enable LiveDashboard and Swoosh mailbox preview in development
49 if Application.compile_env(:avy, :dev_routes) do
50 # If you want to use the LiveDashboard in production, you should put
51 # it behind authentication and allow only admins to access it.
52 # If your application does not have an admins-only section yet,
53 # you can use Plug.BasicAuth to set up some basic authentication
54 # as long as you are also using SSL (which you should anyway).
55 import Phoenix.LiveDashboard.Router
56
57 scope "/dev" do
58 pipe_through [:fetch_session, :protect_from_forgery]
59
60 live_dashboard "/dashboard",
61 metrics: AvyWeb.Telemetry,
62 additional_pages: [
63 flame_on: FlameOn.DashboardPage
64 ]
65
66 forward "/mailbox", Plug.Swoosh.MailboxPreview
67 end
68 end
69 end