|
| 1 | +defmodule CodeCorps.GitHub.EagerAPI do |
| 2 | + @moduledoc """ |
| 3 | + Eager loads a resource from the GitHub API by fetching all of its pages in |
| 4 | + parallel. |
| 5 | + """ |
| 6 | + |
| 7 | + def get_all(url, headers, options) do |
| 8 | + HTTPoison.start |
| 9 | + {:ok, response} = HTTPoison.get(url, headers, options) |
| 10 | + |
| 11 | + first_page = Poison.decode!(response.body) |
| 12 | + case response.headers |> retrieve_total_pages do |
| 13 | + 1 -> first_page |
| 14 | + total -> |
| 15 | + first_page ++ get_remaining_pages(total, url, headers, options) |> List.flatten |
| 16 | + end |
| 17 | + end |
| 18 | + |
| 19 | + defp get_remaining_pages(total, url, headers, options) do |
| 20 | + 2..total |
| 21 | + |> Enum.to_list |
| 22 | + |> Enum.map(&Task.async(fn -> |
| 23 | + params = options[:params] ++ [page: &1] |
| 24 | + HTTPoison.get(url, headers, options ++ [params: params]) |
| 25 | + end)) |
| 26 | + |> Enum.map(&Task.await(&1, 10000)) |
| 27 | + |> Enum.map(&handle_page_response/1) |
| 28 | + end |
| 29 | + |
| 30 | + defp handle_page_response({:ok, %{body: body}}), do: Poison.decode!(body) |
| 31 | + |
| 32 | + def retrieve_total_pages(headers) do |
| 33 | + case headers |> List.keyfind("Link", 0, nil) do |
| 34 | + nil -> 1 |
| 35 | + {"Link", value} -> value |> extract_total_pages |
| 36 | + end |
| 37 | + end |
| 38 | + |
| 39 | + defp extract_total_pages(links_string) do |
| 40 | + # We use regex to parse the pagination info from the GitHub API response |
| 41 | + # headers. |
| 42 | + # |
| 43 | + # The headers render pages in the following format: |
| 44 | + # |
| 45 | + # ``` |
| 46 | + # {"Link", '<https://api.github.com/search/code?q=addClass+user%3Amozilla&page=15>; rel="next", |
| 47 | + # <https://api.github.com/search/code?q=addClass+user%3Amozilla&page=34>; rel="last", |
| 48 | + # <https://api.github.com/search/code?q=addClass+user%3Amozilla&page=1>; rel="first", |
| 49 | + # <https://api.github.com/search/code?q=addClass+user%3Amozilla&page=13>; rel="prev"' |
| 50 | + # ``` |
| 51 | + # |
| 52 | + # If the response has no list header, then we have received all the records |
| 53 | + # from the only possible page. |
| 54 | + # |
| 55 | + # If the response has a list header, the value will contain at least the |
| 56 | + # "last" relation. |
| 57 | + links_string |
| 58 | + |> String.split(", ") |
| 59 | + |> Enum.map(fn link -> |
| 60 | + rel = get_rel(link) |
| 61 | + page = get_page(link) |
| 62 | + {rel, page} |
| 63 | + end) |
| 64 | + |> Enum.into(%{}) |
| 65 | + |> Map.get("last") |
| 66 | + end |
| 67 | + |
| 68 | + defp get_rel(link) do |
| 69 | + # Searches for `rel=` |
| 70 | + Regex.run(~r{rel="([a-z]+)"}, link) |> List.last() |
| 71 | + end |
| 72 | + |
| 73 | + defp get_page(link) do |
| 74 | + # Searches for the following variations: |
| 75 | + # ``` |
| 76 | + # ?page={match}> |
| 77 | + # ?page={match}&... |
| 78 | + # &page={match}> |
| 79 | + # &page={match}&... |
| 80 | + # ``` |
| 81 | + Regex.run(~r{[&/?]page=([^>&]+)}, link) |> List.last |> String.to_integer |
| 82 | + end |
| 83 | +end |
0 commit comments