diff --git a/LICENSE b/LICENSE new file mode 100644 --- /dev/null +++ b/LICENSE @@ -0,0 +1,32 @@ +This module is under this "3 clause" BSD license: + +Copyright (c) 2025-2025, Daniil Iaitskov +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * The names of the contributors may not be used to endorse or + promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/a-piece-of-flake.cabal b/a-piece-of-flake.cabal new file mode 100644 --- /dev/null +++ b/a-piece-of-flake.cabal @@ -0,0 +1,244 @@ +cabal-version: 3.0 +name: a-piece-of-flake +version: 0.0.1 + +synopsis: Nix flake repository for everyone +description: + The main idea behind this project is to provide a quick and simple + interface for publishing Nix flakes. + + Flakes have become relatively mature and address the central repository + issue. However, the Nixpkgs repository on GitHub still has more that 5k + open issues and a comparable number of pull requests, and continues to + receive many commits every day. Getting a pull request for a new tool + merged into Nixpkgs can be difficult - the Nixpkgs README explicitly + discourages people from submitting their “pet” projects. + + The Nixpkgs repository is huge. It contains more than 120k packages, but + the majority of them are not native to Nix. For example, about 10% are + Haskell packages imported. Therefore, this large number cannot be used + as a reliable measure of how well the publishing process is developed in + Nix. For instance, the PyPy repository alone currently contains almost + 900k packages. + + It is also important to note Python is the most popular general-purpose + programming language, and its publishing process was designed by + programmers for programmers. Yet there is no pull-request step in the + workflow. The interface is essentially “upload and forget”, which has a + significant positive impact on the conversion funnel of Python packages. + + Flakes are easy to install, but the publishing workflow is not yet + polished enough. The current approach to distributing flakes appears to + have inherinted many characteristics of the Nixpkgs workflow. + + For Nixpkgs, this was the natural way of development, because all + derivations form a large and coupled Nix expression split across many + files within a single Git repository. + + == Service + #service# + + The service is deployed at + + + == Development environment + #development-environment# + + > $ nix develop + > $ emacs & + > $ cabal build + > $ cabal test + + === Update cachix + #update-cachix# + + > nix build --no-link --print-out-paths | cachix push piece-of-flake + + ==== Cachix public key + #cachix-public-key# + + > piece-of-flake.cachix.org-1:JamWDEABLpvWhGHIK7Xn/qYFbdZqb6ne7IAF62fXyFY= + + == Release + #release# + + > $ nix build $(e -static true) + > $ ./result/bin/a-piece-of-flake run +homepage: http://github.com/yaitskov/a-piece-of-flake +license: BSD-3-Clause +license-file: LICENSE +author: Daniil Iaitskov +maintainer: dyaitskov@gmail.com +copyright: Daniil Iaitkov 2026 +category: System +build-type: Simple +bug-reports: https://github.com/yaitskov/a-piece-of-flake/issues +extra-source-files: + assets/favicon.svg + assets/github.svg + assets/sitemap.xml + assets/github.svg + assets/flake.svg + assets/app.js + assets/style.css + assets/bulma.min.css + -- brotli + assets/favicon.svg.br + assets/github.svg.br + assets/flake.svg.br + assets/app.js.br + assets/style.css.br + assets/bulma.min.css.br + -- gzipped + assets/favicon.svg.gz + assets/github.svg.gz + assets/flake.svg.gz + assets/app.js.gz + assets/style.css.gz + assets/bulma.min.css.gz + + assets/robots.txt + assets/flush.mp3 + assets/snow.mp3 + assets/avalanche.mp3 + trace-embrace.yaml + +extra-doc-files: + changelog.md +tested-with: + GHC == 9.12.2 + +source-repository head + type: + git + location: + https://github.com/yaitskov/a-piece-of-flake.git + +common base + default-language: GHC2024 + ghc-options: -Wall + default-extensions: + DefaultSignatures + DuplicateRecordFields + NoImplicitPrelude + OverloadedRecordDot + OverloadedStrings + TemplateHaskell + TypeFamilies + ViewPatterns + build-depends: + , base >=4.7 && < 5 + , optparse-applicative < 1 + , relude >= 1.2.2 && < 2 + , tagged < 1 + , unliftio < 1 + , yesod-core < 1.8 + +library + import: base + hs-source-dirs: src + exposed-modules: + PieceOfFlake.Acid + PieceOfFlake.Aeson + PieceOfFlake.CmdArgs + PieceOfFlake.CmdRun + PieceOfFlake.Fetcher + PieceOfFlake.Flake + PieceOfFlake.Flake.Repo + PieceOfFlake.Http + PieceOfFlake.Index + PieceOfFlake.Page + PieceOfFlake.Prelude + PieceOfFlake.Req + PieceOfFlake.Stats + PieceOfFlake.Stm + PieceOfFlake.SubmitList + PieceOfFlake.Th + PieceOfFlake.TotalMath + PieceOfFlake.WebService + PieceOfFlake.Yesod + other-modules: + Paths_a_piece_of_flake + autogen-modules: + Paths_a_piece_of_flake + build-depends: + , acid-state < 1 + , add-dependent-file < 1 + , aeson < 3 + , aeson-pretty < 1 + , binary < 1 + , blaze-markup < 1 + , bytestring < 1 + , cryptohash-sha1 < 1 + , deepseq < 2 + , either < 6 + , exceptions < 1 + , file-embed < 1 + , filepath < 2 + , full-text-search < 1 + , generics-sop < 1 + , hostname < 2 + , http-types < 1 + , lens < 6 + , list-t < 2 + , lrucaching < 1 + , monad-logger < 1 + , memory < 1 + , mtl < 3 + , network < 4 + , non-empty < 1 + , non-negative-time-diff >= 0.0.2 && < 1 + , prettyprinter < 2 + , process < 2 + , psqueues < 1 + , ref-tf < 1 + , refined < 1 + , regex-tdfa < 2 + , req < 4 + , retry < 1 + , ring-buffer < 1 + , safecopy < 1 + , sop-core < 1 + , statistics < 1 + , stm-containers < 2 + , stm < 3 + , tar < 1 + , template-haskell < 3 + , text-zipper < 1 + , time < 2 + , time-units < 2 + , tokenize < 1 + , trace-embrace >= 1.3 && < 2 + , vector < 1 + , wai < 4 + , wai-extra < 4 + , warp < 4 + , warp-tls < 4 + , wl-pprint-text < 2 + +test-suite test + import: base + type: exitcode-stdio-1.0 + main-is: Driver.hs + other-modules: + PieceOfFlake.Test.Aeson + Discovery + hs-source-dirs: + test + ghc-options: -Wall -rtsopts -threaded -main-is Driver + build-depends: + , a-piece-of-flake + , aeson + , QuickCheck + , tasty + , tasty-discover + , tasty-hunit + , tasty-quickcheck + +executable a-piece-of-flake + import: base + ghc-options: -threaded + main-is: PieceOfFlake.hs + hs-source-dirs: app + build-depends: + , a-piece-of-flake diff --git a/app/PieceOfFlake.hs b/app/PieceOfFlake.hs new file mode 100644 --- /dev/null +++ b/app/PieceOfFlake.hs @@ -0,0 +1,8 @@ +module Main where + +import PieceOfFlake.CmdArgs +import PieceOfFlake.CmdRun +import PieceOfFlake.Prelude + +main :: IO () +main = execWithArgs runCmd =<< getArgs diff --git a/assets/app.js b/assets/app.js new file mode 100644 --- /dev/null +++ b/assets/app.js @@ -0,0 +1,206 @@ +/* a piece of flake + github:yaitskov/add-dependent-file +*/ +const xhrError = (xhr) => { + if (xhr.responseText && xhr.responseText.length > 1) { + try { + const respO = JSON.parse(xhr.responseText); + if (respO.errors instanceof Array) { + return respO.errors.join("\n"); + } + } catch (e) { + console.error(`${xhr.responseText} => ${e}`); + } + return xhr.responseText; + } else { + if (xhr.statusText && xhr.statusText.length > 1) { + return xhr.statusText; + } else { + return "Server is down or it is offline"; + } + } +}; + +const xhrPost = (urlPath, payload, okCb, opsCb) => { + const xhr = new XMLHttpRequest(); + xhr.open("POST", urlPath, true); + xhr.setRequestHeader('Content-Type', 'application/json'); + xhr.setRequestHeader('Accept', 'application/json'); + xhr.send(JSON.stringify(payload)); + + xhr.onload = (e) => { + if (xhr.readyState === 4) { + if (xhr.status === 200) { + var r = null; + try { + r = JSON.parse(xhr.responseText); + } catch (je) { + opsCb(`Failed to parse response as JSON: ${je}`); + return; + } + okCb(r); + } else { + opsCb(xhrError(xhr)); + } + } else { + opsCb(xhrError(xhr)); + } + }; + xhr.onerror = (e) => { + opsCb(`${e} => ${xhrError(xhr)}`); + }; + return xhr; +}; + +const ready = () => { + console.log(`Document ready hook is triggered`); + const navBarBurger = document.querySelector('#navbar-burger'); + const navBarBurgerCss = navBarBurger.className; + const navBarMenu = document.querySelector('#navbar-menu'); + const navBarMenuCss = navBarMenu.className; + const flakeState = document.querySelector('#flake-is-not-indexed'); + + if (flakeState) { + document.addEventListener("visibilitychange", (event) => { + if (document.visibilityState == "visible") { + window.location.reload(); + } + }); + } + + + navBarBurger.addEventListener( + "click", (e) => { + if (navBarMenu.className.indexOf("is-active") < 0) { + navBarMenu.className = `${navBarMenu.className} is-active`; + navBarBurger.className = `${navBarBurger.className} is-active`; + } else { + navBarMenu.className = navBarMenuCss; + navBarBurger.className = navBarBurgerCss; + } + } + ); +}; + +document.addEventListener("DOMContentLoaded", ready); + +const tokenize = s => s.split(/[ \t\n]/).filter(w => w.length > 0); + +const flakeViewUrl = (fu) => `/flake/${encodeURIComponent(fu)}`; + +const searchFlakesBy = (pattern) => { + const searchHint = document.querySelector('#search-hint'); + const flakesTbody = document.querySelector('#search-results'); + const foundFlakes = document.querySelector('#found-flakes'); + const errorOutput = document.querySelector('#error-output'); + const noFlakesFound = document.querySelector('#no-flakes-found'); + flakesTbody.innerHTML = ""; + errorOutput.innerHTML = ""; + errorOutput.parentNode.className = "is-hidden"; + foundFlakes.className = "is-hidden"; + noFlakesFound.className = "is-hidden"; + console.log(`Find flakes matching pattern [${pattern}]`); + xhrPost( + "find-flakes", + {skipBroken: false, searchPattern: tokenize(pattern)}, + (foundflakeUrls) => { + if (foundflakeUrls.length == 0) { + noFlakesFound.className = ""; + searchHint.parentNode.className = ""; + } else { + foundFlakes.className = ""; + searchHint.parentNode.className = "is-hidden"; + } + for (const u in foundflakeUrls) { + const tr = document.createElement("tr"); + const td = document.createElement("td"); + tr.appendChild(td); + const a = document.createElement("a"); + td.appendChild(a); + a.setAttribute('href', flakeViewUrl(foundflakeUrls[u])); + a.innerText = foundflakeUrls[u]; + flakesTbody.appendChild(tr); + } + }, + (err) => { + errorOutput.innerText = err; + errorOutput.parentNode.className = ""; + }); + return false; +}; + +const rewriteUrlToGithubLink = (url) => + url.replace(/^https:[/]{2}github[.]com[/]/, "github:"); + +const startSnowfall = () => { + const aniflakes = document.getElementById("aniflakes"); + aniflakes.className = "ani-run"; +}; + +const stopSnowfall = () => { + const aniflakes = document.getElementById("aniflakes"); + aniflakes.className = ""; +}; + +const submitFlake = (url) => { + const badUrl = document.getElementById("bad-url"); + + badUrl.className = "is-hidden"; + url = rewriteUrlToGithubLink(url.trim()); + if (!url.match(/^github:[a-zA-Z0-9._-]+[/][a-zA-Z0-9._-]+$/)) { + badUrl.className = ""; + return false; + } + startSnowfall(); + const sound = Math.random() >= 0.5 ? new Audio("/snow.mp3") + : (Math.random() >= 0.5 ? new Audio("/avalanche.mp3") : new Audio("/flush.mp3")); + + console.log(`Sumbitting flake ${url}`); + + const sumbittedNotification = document.getElementById("sumbitted-notification"); + const errorOutputHid = document.getElementById("error-output-hid"); + const errorOutput = document.getElementById("error-output"); + const flakeLink = document.getElementById("flake-link"); + flakeLink.setAttribute('href', '#'); + errorOutput.innerText = ""; + errorOutputHid.className = "is-hidden"; + sumbittedNotification.className = "is-hidden"; + + xhrPost( + "submit-flake", + url, + (flake) => { + console.log(`${flake}`); + switch(flake.tag) { + case "SubmittedFlake": + case "FlakeIsBeingFetched": + /* play sound */ + sound.onended = (e) => { + window.location = flakeViewUrl(url); + }; + sound.play(); + flakeLink.setAttribute('href', flakeViewUrl(url)); + sumbittedNotification.className = ""; + break; + case "FlakeFetched": + case "FlakeIndexed": + /* just redirect to flake view page */ + flakeLink.setAttribute('href', flakeViewUrl(url)); + sumbittedNotification.className = ""; + setTimeout(() => window.location = flakeViewUrl(url), 1000); + break; + case "BadFlake": + default: + stopSnowfall(); + errorOutput.innerText = flake.error; + errorOutputHid.className = ''; + } + }, + (err) => { + stopSnowfall(); + errorOutput.innerText = `Submition of flake ${url} has failed: ${err}`; + errorOutputHid.className = ''; + } + ); + return false; +}; diff --git a/assets/app.js.br b/assets/app.js.br new file mode 100644 Binary files /dev/null and b/assets/app.js.br differ diff --git a/assets/app.js.gz b/assets/app.js.gz new file mode 100644 Binary files /dev/null and b/assets/app.js.gz differ diff --git a/assets/avalanche.mp3 b/assets/avalanche.mp3 new file mode 100644 Binary files /dev/null and b/assets/avalanche.mp3 differ diff --git a/assets/bulma.min.css b/assets/bulma.min.css new file mode 100644 --- /dev/null +++ b/assets/bulma.min.css @@ -0,0 +1,3 @@ +@charset "UTF-8"; +/*! bulma.io v1.0.4 | MIT License | github.com/jgthms/bulma */:root{--bulma-control-radius:var(--bulma-radius);--bulma-control-border-width:1px;--bulma-control-height:2.5em;--bulma-control-line-height:1.5;--bulma-control-padding-vertical:calc(0.5em - 1px);--bulma-control-padding-horizontal:calc(0.75em - 1px);--bulma-control-size:var(--bulma-size-normal);--bulma-scheme-h:221;--bulma-scheme-s:14%;--bulma-light-l:90%;--bulma-light-invert-l:20%;--bulma-hover-background-l-delta:-5%;--bulma-active-background-l-delta:-10%;--bulma-hover-border-l-delta:-10%;--bulma-active-border-l-delta:-20%;--bulma-scheme-main-l:100%;--bulma-background-l:96%;--bulma-border-l:86%;--bulma-text-strong-l:21%;--bulma-scheme-invert-l:4%;--bulma-family-primary:Inter,SF Pro,Segoe UI,Roboto,Oxygen,Ubuntu,Helvetica Neue,Helvetica,Arial,sans-serif;--bulma-size-small:0.75rem;--bulma-size-normal:1rem;--bulma-weight-normal:400;--bulma-weight-medium:500;--bulma-weight-semibold:600;--bulma-weight-extrabold:800;--bulma-block-spacing:1.5rem;--bulma-duration:294ms;--bulma-easing:ease-out;--bulma-radius:0.375rem;--bulma-burger-h:var(--bulma-link-h);--bulma-burger-s:var(--bulma-link-s);--bulma-burger-l:var(--bulma-link-l);--bulma-burger-border-radius:0.5em;--bulma-burger-gap:5px;--bulma-burger-item-height:2px;--bulma-burger-item-width:20px;--bulma-black:hsla(var(--bulma-black-h),var(--bulma-black-s),var(--bulma-black-l),1);--bulma-black-h:221deg;--bulma-black-s:14%;--bulma-black-l:4%;--bulma-black-invert-l:100%;--bulma-black-invert:#fff;--bulma-black-on-scheme-l:4%;--bulma-light:hsla(var(--bulma-light-h),var(--bulma-light-s),var(--bulma-light-l),1);--bulma-light-h:221deg;--bulma-light-s:14%;--bulma-light-l:96%;--bulma-light-invert-l:21%;--bulma-light-invert:#2e333d;--bulma-light-on-scheme-l:36%;--bulma-text:hsla(var(--bulma-text-h),var(--bulma-text-s),var(--bulma-text-l),1);--bulma-text-h:221deg;--bulma-text-s:14%;--bulma-text-l:29%;--bulma-link:hsla(var(--bulma-link-h),var(--bulma-link-s),var(--bulma-link-l),1);--bulma-link-h:233deg;--bulma-link-s:100%;--bulma-link-l:63%;--bulma-link-100-l:98%;--bulma-link-invert-l:var(--bulma-link-100-l);--bulma-link-on-scheme-l:58%;--bulma-info:hsla(var(--bulma-info-h),var(--bulma-info-s),var(--bulma-info-l),1);--bulma-info-h:198deg;--bulma-info-s:100%;--bulma-info-l:70%;--bulma-info-10-l:10%;--bulma-info-15-l:15%;--bulma-info-invert-l:var(--bulma-info-10-l);--bulma-info-invert:hsla(var(--bulma-info-h),var(--bulma-info-s),var(--bulma-info-invert-l),1);--bulma-info-light-invert-l:var(--bulma-info-15-l);--bulma-info-on-scheme-l:25%;--bulma-success:hsla(var(--bulma-success-h),var(--bulma-success-s),var(--bulma-success-l),1);--bulma-success-h:153deg;--bulma-success-s:53%;--bulma-success-l:53%;--bulma-success-10-l:8%;--bulma-success-15-l:13%;--bulma-success-invert-l:var(--bulma-success-10-l);--bulma-success-invert:hsla(var(--bulma-success-h),var(--bulma-success-s),var(--bulma-success-invert-l),1);--bulma-success-light-invert-l:var(--bulma-success-15-l);--bulma-success-on-scheme-l:23%;--bulma-warning:hsla(var(--bulma-warning-h),var(--bulma-warning-s),var(--bulma-warning-l),1);--bulma-warning-h:42deg;--bulma-warning-s:100%;--bulma-warning-l:53%;--bulma-warning-10-l:8%;--bulma-warning-15-l:13%;--bulma-warning-invert-l:var(--bulma-warning-10-l);--bulma-warning-invert:hsla(var(--bulma-warning-h),var(--bulma-warning-s),var(--bulma-warning-invert-l),1);--bulma-warning-light-invert-l:var(--bulma-warning-15-l);--bulma-warning-on-scheme-l:23%;--bulma-danger:hsla(var(--bulma-danger-h),var(--bulma-danger-s),var(--bulma-danger-l),1);--bulma-danger-h:348deg;--bulma-danger-s:100%;--bulma-danger-l:70%;--bulma-danger-05-l:5%;--bulma-danger-25-l:25%;--bulma-danger-invert-l:var(--bulma-danger-05-l);--bulma-danger-invert:hsla(var(--bulma-danger-h),var(--bulma-danger-s),var(--bulma-danger-invert-l),1);--bulma-danger-light-invert-l:var(--bulma-danger-25-l);--bulma-danger-on-scheme-l:40%;--bulma-size-3:2rem;--bulma-scheme-main:hsl(var(--bulma-scheme-h),var(--bulma-scheme-s),var(--bulma-scheme-main-l));--bulma-border:hsl(var(--bulma-scheme-h),var(--bulma-scheme-s),var(--bulma-border-l));--bulma-text:hsl(var(--bulma-text-h),var(--bulma-text-s),var(--bulma-text-l));--bulma-text-strong:hsl(var(--bulma-text-h),var(--bulma-text-s),var(--bulma-text-strong-l));--bulma-link:hsl(var(--bulma-link-h),var(--bulma-link-s),var(--bulma-link-l));--bulma-link-text:hsl(var(--bulma-link-h),var(--bulma-link-s),var(--bulma-link-on-scheme-l));--bulma-focus-h:var(--bulma-link-h);--bulma-focus-s:var(--bulma-link-s);--bulma-focus-l:var(--bulma-link-l);--bulma-focus-offset:1px;--bulma-focus-style:solid;--bulma-focus-width:2px;--bulma-focus-shadow-size:0 0 0 0.1875em;--bulma-focus-shadow-alpha:0.25}@media (prefers-color-scheme:light){:root{--bulma-scheme-h:221;--bulma-scheme-s:14%;--bulma-light-l:90%;--bulma-light-invert-l:20%;--bulma-hover-background-l-delta:-5%;--bulma-active-background-l-delta:-10%;--bulma-hover-border-l-delta:-10%;--bulma-active-border-l-delta:-20%;--bulma-scheme-main-l:100%;--bulma-background-l:96%;--bulma-border-l:86%;--bulma-text-strong-l:21%;--bulma-scheme-invert-l:4%;--bulma-family-primary:Inter,SF Pro,Segoe UI,Roboto,Oxygen,Ubuntu,Helvetica Neue,Helvetica,Arial,sans-serif;--bulma-size-small:0.75rem;--bulma-size-normal:1rem;--bulma-weight-normal:400;--bulma-weight-medium:500;--bulma-weight-semibold:600;--bulma-weight-extrabold:800;--bulma-block-spacing:1.5rem;--bulma-duration:294ms;--bulma-easing:ease-out;--bulma-radius:0.375rem;--bulma-burger-h:var(--bulma-link-h);--bulma-burger-s:var(--bulma-link-s);--bulma-burger-l:var(--bulma-link-l);--bulma-burger-border-radius:0.5em;--bulma-burger-gap:5px;--bulma-burger-item-height:2px;--bulma-burger-item-width:20px;--bulma-black:hsla(var(--bulma-black-h),var(--bulma-black-s),var(--bulma-black-l),1);--bulma-black-h:221deg;--bulma-black-s:14%;--bulma-black-l:4%;--bulma-black-invert-l:100%;--bulma-black-invert:#fff;--bulma-black-on-scheme-l:4%;--bulma-light:hsla(var(--bulma-light-h),var(--bulma-light-s),var(--bulma-light-l),1);--bulma-light-h:221deg;--bulma-light-s:14%;--bulma-light-l:96%;--bulma-light-invert-l:21%;--bulma-light-invert:#2e333d;--bulma-light-on-scheme-l:36%;--bulma-text:hsla(var(--bulma-text-h),var(--bulma-text-s),var(--bulma-text-l),1);--bulma-text-h:221deg;--bulma-text-s:14%;--bulma-text-l:29%;--bulma-link:hsla(var(--bulma-link-h),var(--bulma-link-s),var(--bulma-link-l),1);--bulma-link-h:233deg;--bulma-link-s:100%;--bulma-link-l:63%;--bulma-link-100-l:98%;--bulma-link-invert-l:var(--bulma-link-100-l);--bulma-link-on-scheme-l:58%;--bulma-info:hsla(var(--bulma-info-h),var(--bulma-info-s),var(--bulma-info-l),1);--bulma-info-h:198deg;--bulma-info-s:100%;--bulma-info-l:70%;--bulma-info-10-l:10%;--bulma-info-15-l:15%;--bulma-info-invert-l:var(--bulma-info-10-l);--bulma-info-invert:hsla(var(--bulma-info-h),var(--bulma-info-s),var(--bulma-info-invert-l),1);--bulma-info-light-invert-l:var(--bulma-info-15-l);--bulma-info-on-scheme-l:25%;--bulma-success:hsla(var(--bulma-success-h),var(--bulma-success-s),var(--bulma-success-l),1);--bulma-success-h:153deg;--bulma-success-s:53%;--bulma-success-l:53%;--bulma-success-10-l:8%;--bulma-success-15-l:13%;--bulma-success-invert-l:var(--bulma-success-10-l);--bulma-success-invert:hsla(var(--bulma-success-h),var(--bulma-success-s),var(--bulma-success-invert-l),1);--bulma-success-light-invert-l:var(--bulma-success-15-l);--bulma-success-on-scheme-l:23%;--bulma-warning:hsla(var(--bulma-warning-h),var(--bulma-warning-s),var(--bulma-warning-l),1);--bulma-warning-h:42deg;--bulma-warning-s:100%;--bulma-warning-l:53%;--bulma-warning-10-l:8%;--bulma-warning-15-l:13%;--bulma-warning-invert-l:var(--bulma-warning-10-l);--bulma-warning-invert:hsla(var(--bulma-warning-h),var(--bulma-warning-s),var(--bulma-warning-invert-l),1);--bulma-warning-light-invert-l:var(--bulma-warning-15-l);--bulma-warning-on-scheme-l:23%;--bulma-danger:hsla(var(--bulma-danger-h),var(--bulma-danger-s),var(--bulma-danger-l),1);--bulma-danger-h:348deg;--bulma-danger-s:100%;--bulma-danger-l:70%;--bulma-danger-05-l:5%;--bulma-danger-25-l:25%;--bulma-danger-invert-l:var(--bulma-danger-05-l);--bulma-danger-invert:hsla(var(--bulma-danger-h),var(--bulma-danger-s),var(--bulma-danger-invert-l),1);--bulma-danger-light-invert-l:var(--bulma-danger-25-l);--bulma-danger-on-scheme-l:40%;--bulma-size-3:2rem}}@media (prefers-color-scheme:dark){:root{--bulma-black-on-scheme-l:-51%;--bulma-light-on-scheme-l:96%;--bulma-link-on-scheme-l:73%;--bulma-info-on-scheme-l:70%;--bulma-success-on-scheme-l:53%;--bulma-warning-on-scheme-l:53%;--bulma-danger-on-scheme-l:70%;--bulma-scheme-main-l:9%;--bulma-background-l:14%;--bulma-border-l:24%;--bulma-text-l:71%;--bulma-text-strong-l:93%;--bulma-hover-background-l-delta:5%;--bulma-active-background-l-delta:10%;--bulma-hover-border-l-delta:10%;--bulma-active-border-l-delta:20%}} +/*! minireset.css v0.0.6 | MIT License | github.com/jgthms/minireset.css */body,html{margin:0;padding:0}button,input{margin:0}html{box-sizing:border-box}*,:after,:before{box-sizing:inherit}img{height:auto;max-width:100%}table{border-collapse:collapse;border-spacing:0}td{padding:0}td:not([align]){text-align:inherit}:root{--bulma-body-background-color:var(--bulma-scheme-main);--bulma-body-size:1em;--bulma-body-min-width:300px;--bulma-body-rendering:optimizeLegibility;--bulma-body-family:var(--bulma-family-primary);--bulma-body-overflow-x:hidden;--bulma-body-overflow-y:scroll;--bulma-body-color:var(--bulma-text);--bulma-body-font-size:1em;--bulma-body-weight:var(--bulma-weight-normal);--bulma-body-line-height:1.5}html{background-color:var(--bulma-body-background-color);font-size:var(--bulma-body-size);-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;min-width:var(--bulma-body-min-width);overflow-x:var(--bulma-body-overflow-x);overflow-y:var(--bulma-body-overflow-y);text-rendering:var(--bulma-body-rendering);text-size-adjust:100%}section{display:block}body,button,input{font-family:var(--bulma-body-family)}body{color:var(--bulma-body-color);font-size:var(--bulma-body-font-size);font-weight:var(--bulma-body-weight);line-height:var(--bulma-body-line-height)}a,button{cursor:pointer}a:focus-visible,button:focus-visible{outline-color:hsl(var(--bulma-focus-h),var(--bulma-focus-s),var(--bulma-focus-l));outline-offset:var(--bulma-focus-offset);outline-style:var(--bulma-focus-style);outline-width:var(--bulma-focus-width)}a:focus-visible:active,button:focus-visible:active{outline-width:1px}a:active,button:active{outline-width:1px}a{color:var(--bulma-link-text);cursor:pointer;text-decoration:none;transition-duration:var(--bulma-duration);transition-property:background-color,border-color,color}button{appearance:none;background:none;border:none;color:inherit;font-family:inherit;font-size:1em;margin:0;padding:0;transition-duration:var(--bulma-duration);transition-property:background-color,border-color,color}img{height:auto;max-width:100%}span{font-style:inherit;font-weight:inherit}svg{height:auto;width:auto}table td{vertical-align:top}table td:not([align]){text-align:inherit}@keyframes spinAround{0%{transform:rotate(0deg)}to{transform:rotate(359deg)}}@keyframes pulsate{50%{opacity:.5}}.content:not(:last-child),.field:not(:last-child),.message:not(:last-child),.notification:not(:last-child),.table:not(:last-child),.title:not(:last-child){margin-bottom:var(--bulma-block-spacing)}.button,.input{align-items:center;appearance:none;border-color:transparent;border-radius:var(--bulma-control-radius);border-style:solid;border-width:var(--bulma-control-border-width);box-shadow:none;display:inline-flex;font-size:var(--bulma-control-size);height:var(--bulma-control-height);justify-content:flex-start;line-height:var(--bulma-control-line-height);padding:var(--bulma-control-padding-vertical) var(--bulma-control-padding-horizontal);position:relative;transition-duration:var(--bulma-duration);transition-property:background-color,border-color,box-shadow,color;vertical-align:top}.button:active,.button:focus,.button:focus-visible,.button:focus-within,.input:active,.input:focus,.input:focus-visible,.input:focus-within,.is-active.button,.is-active.input{outline:none}.navbar-burger{appearance:none;background:none;border:none;color:inherit;font-family:inherit;font-size:1em;margin:0;padding:0}.button{-webkit-touch-callout:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.button{--bulma-button-weight:var(--bulma-weight-medium);--bulma-button-border-style:solid;--bulma-button-border-width:var(--bulma-control-border-width);--bulma-button-padding-vertical:0.5em;--bulma-button-padding-horizontal:1em;--bulma-button-h:var(--bulma-scheme-h);--bulma-button-s:var(--bulma-scheme-s);--bulma-button-background-l:var(--bulma-scheme-main-l);--bulma-button-background-l-delta:0%;--bulma-button-hover-background-l-delta:var(--bulma-hover-background-l-delta);--bulma-button-active-background-l-delta:var(--bulma-active-background-l-delta);--bulma-button-color-l:var(--bulma-text-strong-l);--bulma-button-border-l:var(--bulma-border-l);--bulma-button-border-l-delta:0%;--bulma-button-hover-border-l-delta:var(--bulma-hover-border-l-delta);--bulma-button-active-border-l-delta:var(--bulma-active-border-l-delta);--bulma-button-outer-shadow-h:0;--bulma-button-outer-shadow-s:0%;--bulma-button-outer-shadow-l:20%;--bulma-button-outer-shadow-a:0.05;background-color:hsl(var(--bulma-button-h),var(--bulma-button-s),calc(var(--bulma-button-background-l) + var(--bulma-button-background-l-delta)));border-color:hsl(var(--bulma-button-h),var(--bulma-button-s),calc(var(--bulma-button-border-l) + var(--bulma-button-border-l-delta)));border-style:var(--bulma-button-border-style);border-width:var(--bulma-button-border-width);box-shadow:0 .0625em .125em hsla(var(--bulma-button-outer-shadow-h),var(--bulma-button-outer-shadow-s),var(--bulma-button-outer-shadow-l),var(--bulma-button-outer-shadow-a)),0 .125em .25em hsla(var(--bulma-button-outer-shadow-h),var(--bulma-button-outer-shadow-s),var(--bulma-button-outer-shadow-l),var(--bulma-button-outer-shadow-a));color:hsl(var(--bulma-button-h),var(--bulma-button-s),var(--bulma-button-color-l));cursor:pointer;font-weight:var(--bulma-button-weight);height:auto;justify-content:center;padding:calc(var(--bulma-button-padding-vertical) - var(--bulma-button-border-width)) calc(var(--bulma-button-padding-horizontal) - var(--bulma-button-border-width));text-align:center;white-space:nowrap}.button .icon{height:1.5em;width:1.5em}.button .icon:first-child:not(:last-child){margin-inline-end:calc(var(--bulma-button-padding-horizontal)*.25);margin-inline-start:calc(var(--bulma-button-padding-horizontal)*-.5)}.button .icon:last-child:not(:first-child){margin-inline-end:calc(var(--bulma-button-padding-horizontal)*-.5);margin-inline-start:calc(var(--bulma-button-padding-horizontal)*.25)}.button .icon:first-child:last-child{margin-inline-end:calc(var(--bulma-button-padding-horizontal)*-.5);margin-inline-start:calc(var(--bulma-button-padding-horizontal)*-.5)}.button:hover{--bulma-button-background-l-delta:var(--bulma-button-hover-background-l-delta);--bulma-button-border-l-delta:var(--bulma-button-hover-border-l-delta)}.button:focus-visible{--bulma-button-border-width:1px;border-color:hsl(var(--bulma-focus-h),var(--bulma-focus-s),var(--bulma-focus-l));box-shadow:var(--bulma-focus-shadow-size) hsla(var(--bulma-focus-h),var(--bulma-focus-s),var(--bulma-focus-l),var(--bulma-focus-shadow-alpha))}.button.is-active,.button:active{--bulma-button-background-l-delta:var(--bulma-button-active-background-l-delta);--bulma-button-border-l-delta:var(--bulma-button-active-border-l-delta);--bulma-button-outer-shadow-a:0}.button.is-black{--bulma-button-h:var(--bulma-black-h);--bulma-button-s:var(--bulma-black-s);--bulma-button-background-l:var(--bulma-black-l);--bulma-button-border-l:var(--bulma-black-l);--bulma-button-border-width:0px;--bulma-button-color-l:var(--bulma-black-invert-l);--bulma-button-outer-shadow-a:0}.button.is-black:focus-visible{--bulma-button-border-width:1px}.button.is-light{--bulma-button-h:var(--bulma-light-h);--bulma-button-s:var(--bulma-light-s);--bulma-button-background-l:var(--bulma-light-l);--bulma-button-border-l:var(--bulma-light-l);--bulma-button-border-width:0px;--bulma-button-color-l:var(--bulma-light-invert-l);--bulma-button-outer-shadow-a:0}.button.is-light:focus-visible{--bulma-button-border-width:1px}.button.is-info{--bulma-button-h:var(--bulma-info-h);--bulma-button-s:var(--bulma-info-s);--bulma-button-background-l:var(--bulma-info-l);--bulma-button-border-l:var(--bulma-info-l);--bulma-button-border-width:0px;--bulma-button-color-l:var(--bulma-info-invert-l);--bulma-button-outer-shadow-a:0}.button.is-info:focus-visible{--bulma-button-border-width:1px}.button.is-info.is-light{--bulma-button-background-l:var(--bulma-light-l);--bulma-button-color-l:var(--bulma-info-light-invert-l)}.button.is-success{--bulma-button-h:var(--bulma-success-h);--bulma-button-s:var(--bulma-success-s);--bulma-button-background-l:var(--bulma-success-l);--bulma-button-border-l:var(--bulma-success-l);--bulma-button-border-width:0px;--bulma-button-color-l:var(--bulma-success-invert-l);--bulma-button-outer-shadow-a:0}.button.is-success:focus-visible{--bulma-button-border-width:1px}.button.is-success.is-light{--bulma-button-background-l:var(--bulma-light-l);--bulma-button-color-l:var(--bulma-success-light-invert-l)}.button.is-warning{--bulma-button-h:var(--bulma-warning-h);--bulma-button-s:var(--bulma-warning-s);--bulma-button-background-l:var(--bulma-warning-l);--bulma-button-border-l:var(--bulma-warning-l);--bulma-button-border-width:0px;--bulma-button-color-l:var(--bulma-warning-invert-l);--bulma-button-outer-shadow-a:0}.button.is-warning:focus-visible{--bulma-button-border-width:1px}.button.is-warning.is-light{--bulma-button-background-l:var(--bulma-light-l);--bulma-button-color-l:var(--bulma-warning-light-invert-l)}.button.is-danger{--bulma-button-h:var(--bulma-danger-h);--bulma-button-s:var(--bulma-danger-s);--bulma-button-background-l:var(--bulma-danger-l);--bulma-button-border-l:var(--bulma-danger-l);--bulma-button-border-width:0px;--bulma-button-color-l:var(--bulma-danger-invert-l);--bulma-button-outer-shadow-a:0}.button.is-danger:focus-visible{--bulma-button-border-width:1px}.button.is-danger.is-light{--bulma-button-background-l:var(--bulma-light-l);--bulma-button-color-l:var(--bulma-danger-light-invert-l)}.content{--bulma-content-block-margin-bottom:1em;--bulma-content-table-cell-border:1px solid var(--bulma-border);--bulma-content-table-cell-border-width:0 0 1px;--bulma-content-table-cell-padding:0.5em 0.75em}.content table:not(:last-child){margin-bottom:var(--bulma-content-block-margin-bottom)}.content table td{border:var(--bulma-content-table-cell-border);border-width:var(--bulma-content-table-cell-border-width);padding:var(--bulma-content-table-cell-padding);vertical-align:top}.icon{--bulma-icon-dimensions:1.5rem}.icon{align-items:center;display:inline-flex;flex-shrink:0;height:var(--bulma-icon-dimensions);justify-content:center;transition-duration:var(--bulma-duration);transition-property:color;width:var(--bulma-icon-dimensions)}.image{display:block;position:relative}.image img{display:block;height:auto;width:100%}.notification{--bulma-notification-h:var(--bulma-scheme-h);--bulma-notification-s:var(--bulma-scheme-s);--bulma-notification-background-l:var(--bulma-background-l);--bulma-notification-color-l:var(--bulma-text-strong-l);--bulma-notification-radius:var(--bulma-radius);--bulma-notification-padding:1.375em 1.5em;background-color:hsl(var(--bulma-notification-h),var(--bulma-notification-s),var(--bulma-notification-background-l));border-radius:var(--bulma-notification-radius);color:hsl(var(--bulma-notification-h),var(--bulma-notification-s),var(--bulma-notification-color-l));padding:var(--bulma-notification-padding);position:relative}.notification a:not(.button):not(.dropdown-item){color:currentColor;text-decoration:underline}.notification .content,.notification .title{color:currentColor}.notification.is-black{--bulma-notification-h:var(--bulma-black-h);--bulma-notification-s:var(--bulma-black-s);--bulma-notification-background-l:var(--bulma-black-l);--bulma-notification-color-l:var(--bulma-black-invert-l)}.notification.is-black.is-light{--bulma-notification-background-l:90%;--bulma-notification-color-l:var(--bulma-black-light-invert-l)}.notification.is-light{--bulma-notification-h:var(--bulma-light-h);--bulma-notification-s:var(--bulma-light-s);--bulma-notification-background-l:var(--bulma-light-l);--bulma-notification-color-l:var(--bulma-light-invert-l)}.notification.is-light.is-light{--bulma-notification-background-l:90%;--bulma-notification-color-l:var(--bulma-light-light-invert-l)}.notification.is-info{--bulma-notification-h:var(--bulma-info-h);--bulma-notification-s:var(--bulma-info-s);--bulma-notification-background-l:var(--bulma-info-l);--bulma-notification-color-l:var(--bulma-info-invert-l)}.notification.is-info.is-light{--bulma-notification-background-l:90%;--bulma-notification-color-l:var(--bulma-info-light-invert-l)}.notification.is-success{--bulma-notification-h:var(--bulma-success-h);--bulma-notification-s:var(--bulma-success-s);--bulma-notification-background-l:var(--bulma-success-l);--bulma-notification-color-l:var(--bulma-success-invert-l)}.notification.is-success.is-light{--bulma-notification-background-l:90%;--bulma-notification-color-l:var(--bulma-success-light-invert-l)}.notification.is-warning{--bulma-notification-h:var(--bulma-warning-h);--bulma-notification-s:var(--bulma-warning-s);--bulma-notification-background-l:var(--bulma-warning-l);--bulma-notification-color-l:var(--bulma-warning-invert-l)}.notification.is-warning.is-light{--bulma-notification-background-l:90%;--bulma-notification-color-l:var(--bulma-warning-light-invert-l)}.notification.is-danger{--bulma-notification-h:var(--bulma-danger-h);--bulma-notification-s:var(--bulma-danger-s);--bulma-notification-background-l:var(--bulma-danger-l);--bulma-notification-color-l:var(--bulma-danger-invert-l)}.notification.is-danger.is-light{--bulma-notification-background-l:90%;--bulma-notification-color-l:var(--bulma-danger-light-invert-l)}@keyframes moveIndeterminate{0%{background-position:200% 0}to{background-position:-200% 0}}.table{--bulma-table-color:var(--bulma-text-strong);--bulma-table-background-color:var(--bulma-scheme-main);--bulma-table-cell-border-color:var(--bulma-border);--bulma-table-cell-border-style:solid;--bulma-table-cell-border-width:0 0 1px;--bulma-table-cell-padding:0.5em 0.75em;background-color:var(--bulma-table-background-color)}.table,.table td{color:var(--bulma-table-color)}.table td{background-color:var(--bulma-table-cell-background-color);border-color:var(--bulma-table-cell-border-color);border-style:var(--bulma-table-cell-border-style);border-width:var(--bulma-table-cell-border-width);padding:var(--bulma-table-cell-padding);vertical-align:top}.table td.is-black{--bulma-table-color:var(--bulma-black-invert);--bulma-table-cell-background-color:var(--bulma-black);--bulma-table-cell-border-color:var(--bulma-black)}.table td.is-light{--bulma-table-color:var(--bulma-light-invert);--bulma-table-cell-background-color:var(--bulma-light);--bulma-table-cell-border-color:var(--bulma-light)}.table td.is-info{--bulma-table-color:var(--bulma-info-invert);--bulma-table-cell-background-color:var(--bulma-info);--bulma-table-cell-border-color:var(--bulma-info)}.table td.is-success{--bulma-table-color:var(--bulma-success-invert);--bulma-table-cell-background-color:var(--bulma-success);--bulma-table-cell-border-color:var(--bulma-success)}.table td.is-warning{--bulma-table-color:var(--bulma-warning-invert);--bulma-table-cell-background-color:var(--bulma-warning);--bulma-table-cell-border-color:var(--bulma-warning)}.table td.is-danger{--bulma-table-color:var(--bulma-danger-invert);--bulma-table-cell-background-color:var(--bulma-danger);--bulma-table-cell-border-color:var(--bulma-danger)}.table tr.is-black{--bulma-table-color:var(--bulma-black-invert);--bulma-table-cell-background-color:var(--bulma-black);--bulma-table-cell-border-color:var(--bulma-black)}.table tr.is-light{--bulma-table-color:var(--bulma-light-invert);--bulma-table-cell-background-color:var(--bulma-light);--bulma-table-cell-border-color:var(--bulma-light)}.table tr.is-info{--bulma-table-color:var(--bulma-info-invert);--bulma-table-cell-background-color:var(--bulma-info);--bulma-table-cell-border-color:var(--bulma-info)}.table tr.is-success{--bulma-table-color:var(--bulma-success-invert);--bulma-table-cell-background-color:var(--bulma-success);--bulma-table-cell-border-color:var(--bulma-success)}.table tr.is-warning{--bulma-table-color:var(--bulma-warning-invert);--bulma-table-cell-background-color:var(--bulma-warning);--bulma-table-cell-border-color:var(--bulma-warning)}.table tr.is-danger{--bulma-table-color:var(--bulma-danger-invert);--bulma-table-cell-background-color:var(--bulma-danger);--bulma-table-cell-border-color:var(--bulma-danger)}.table.is-bordered td{border-width:1px}.table.is-bordered tr:last-child td{border-bottom-width:1px}.title{--bulma-title-color:var(--bulma-text-strong);--bulma-title-size:var(--bulma-size-3);--bulma-title-weight:var(--bulma-weight-extrabold);--bulma-title-line-height:1.125;word-break:break-word}.title span{font-weight:inherit}.title{color:var(--bulma-title-color);font-size:var(--bulma-title-size);font-weight:var(--bulma-title-weight);line-height:var(--bulma-title-line-height)}.title:not(.is-spaced):has(+.subtitle){margin-bottom:0}.title.is-4{font-size:1.5rem}.title.is-5{font-size:1.25rem}.title.is-6{font-size:1rem}.control,.input{--bulma-input-h:var(--bulma-scheme-h);--bulma-input-s:var(--bulma-scheme-s);--bulma-input-border-l:var(--bulma-border-l);--bulma-input-border-l-delta:0%;--bulma-input-border-color:hsl(var(--bulma-input-h),var(--bulma-input-s),calc(var(--bulma-input-border-l) + var(--bulma-input-border-l-delta)));--bulma-input-hover-border-l-delta:var(--bulma-hover-border-l-delta);--bulma-input-active-border-l-delta:var(--bulma-active-border-l-delta);--bulma-input-focus-h:var(--bulma-focus-h);--bulma-input-focus-s:var(--bulma-focus-s);--bulma-input-focus-l:var(--bulma-focus-l);--bulma-input-focus-shadow-size:var(--bulma-focus-shadow-size);--bulma-input-focus-shadow-alpha:var(--bulma-focus-shadow-alpha);--bulma-input-color-l:var(--bulma-text-strong-l);--bulma-input-background-l:var(--bulma-scheme-main-l);--bulma-input-background-l-delta:0%;--bulma-input-placeholder-color:hsla(var(--bulma-text-h),var(--bulma-text-s),var(--bulma-text-strong-l),0.3);--bulma-input-radius:var(--bulma-radius)}.input{background-color:hsl(var(--bulma-input-h),var(--bulma-input-s),calc(var(--bulma-input-background-l) + var(--bulma-input-background-l-delta)));border-color:var(--bulma-input-border-color);border-radius:var(--bulma-input-radius);color:hsl(var(--bulma-input-h),var(--bulma-input-s),var(--bulma-input-color-l))}.input::-moz-placeholder{color:var(--bulma-input-placeholder-color)}.input::-webkit-input-placeholder{color:var(--bulma-input-placeholder-color)}.input:-moz-placeholder{color:var(--bulma-input-placeholder-color)}.input:-ms-input-placeholder{color:var(--bulma-input-placeholder-color)}.input:hover{--bulma-input-border-l-delta:var(--bulma-input-hover-border-l-delta)}.input:active,.is-active.input{--bulma-input-border-l-delta:var(--bulma-input-active-border-l-delta)}.input:focus,.input:focus-within{border-color:hsl(var(--bulma-input-focus-h),var(--bulma-input-focus-s),var(--bulma-input-focus-l));box-shadow:var(--bulma-input-focus-shadow-size) hsla(var(--bulma-input-focus-h),var(--bulma-input-focus-s),var(--bulma-input-focus-l),var(--bulma-input-focus-shadow-alpha))}.input{box-shadow:inset 0 .0625em .125em hsla(var(--bulma-scheme-h),var(--bulma-scheme-s),var(--bulma-scheme-invert-l),.05);max-width:100%;width:100%}.is-black.input{--bulma-input-h:var(--bulma-black-h);--bulma-input-s:var(--bulma-black-s);--bulma-input-focus-h:var(--bulma-black-h);--bulma-input-focus-s:var(--bulma-black-s);--bulma-input-focus-l:var(--bulma-black-l);--bulma-input-border-l:var(--bulma-black-l)}.is-light.input{--bulma-input-h:var(--bulma-light-h);--bulma-input-s:var(--bulma-light-s);--bulma-input-focus-h:var(--bulma-light-h);--bulma-input-focus-s:var(--bulma-light-s);--bulma-input-focus-l:var(--bulma-light-l);--bulma-input-border-l:var(--bulma-light-l)}.is-info.input{--bulma-input-h:var(--bulma-info-h);--bulma-input-s:var(--bulma-info-s);--bulma-input-focus-h:var(--bulma-info-h);--bulma-input-focus-s:var(--bulma-info-s);--bulma-input-focus-l:var(--bulma-info-l);--bulma-input-border-l:var(--bulma-info-l)}.is-success.input{--bulma-input-h:var(--bulma-success-h);--bulma-input-s:var(--bulma-success-s);--bulma-input-focus-h:var(--bulma-success-h);--bulma-input-focus-s:var(--bulma-success-s);--bulma-input-focus-l:var(--bulma-success-l);--bulma-input-border-l:var(--bulma-success-l)}.is-warning.input{--bulma-input-h:var(--bulma-warning-h);--bulma-input-s:var(--bulma-warning-s);--bulma-input-focus-h:var(--bulma-warning-h);--bulma-input-focus-s:var(--bulma-warning-s);--bulma-input-focus-l:var(--bulma-warning-l);--bulma-input-border-l:var(--bulma-warning-l)}.is-danger.input{--bulma-input-h:var(--bulma-danger-h);--bulma-input-s:var(--bulma-danger-s);--bulma-input-focus-h:var(--bulma-danger-h);--bulma-input-focus-s:var(--bulma-danger-s);--bulma-input-focus-l:var(--bulma-danger-l);--bulma-input-border-l:var(--bulma-danger-l)}:root{--bulma-label-color:var(--bulma-text-strong);--bulma-label-spacing:0.5em;--bulma-help-size:var(--bulma-size-small);--bulma-field-block-spacing:0.75rem}.label{color:var(--bulma-label-color);display:block;font-size:var(--bulma-size-normal);font-weight:var(--bulma-weight-semibold)}.label:not(:last-child){margin-bottom:var(--bulma-label-spacing)}.help{display:block;font-size:var(--bulma-help-size);margin-top:.25rem}.help.is-black{color:hsl(var(--bulma-black-h),var(--bulma-black-s),var(--bulma-black-on-scheme-l))}.help.is-light{color:hsl(var(--bulma-light-h),var(--bulma-light-s),var(--bulma-light-on-scheme-l))}.help.is-info{color:hsl(var(--bulma-info-h),var(--bulma-info-s),var(--bulma-info-on-scheme-l))}.help.is-success{color:hsl(var(--bulma-success-h),var(--bulma-success-s),var(--bulma-success-on-scheme-l))}.help.is-warning{color:hsl(var(--bulma-warning-h),var(--bulma-warning-s),var(--bulma-warning-on-scheme-l))}.help.is-danger{color:hsl(var(--bulma-danger-h),var(--bulma-danger-s),var(--bulma-danger-on-scheme-l))}.field{--bulma-block-spacing:var(--bulma-field-block-spacing)}.field.is-grouped{display:flex;gap:.75rem;justify-content:flex-start}.field.is-grouped>.control{flex-shrink:0}.control{box-sizing:border-box;clear:both;font-size:var(--bulma-size-normal);position:relative;text-align:inherit}.message{--bulma-message-radius:var(--bulma-radius);--bulma-message-h:var(--bulma-scheme-h);--bulma-message-s:var(--bulma-scheme-s);--bulma-message-color-l:var(--bulma-text-l);border-radius:var(--bulma-message-radius);color:hsl(var(--bulma-message-h),var(--bulma-message-s),var(--bulma-message-color-l));font-size:var(--bulma-size-normal)}.message a:not(.button):not(.tag):not(.dropdown-item){color:currentColor;text-decoration:underline}.message.is-black{--bulma-message-h:var(--bulma-black-h);--bulma-message-s:var(--bulma-black-s);--bulma-message-color-l:var(--bulma-black-on-scheme-l)}.message.is-light{--bulma-message-h:var(--bulma-light-h);--bulma-message-s:var(--bulma-light-s);--bulma-message-color-l:var(--bulma-light-on-scheme-l)}.message.is-info{--bulma-message-h:var(--bulma-info-h);--bulma-message-s:var(--bulma-info-s);--bulma-message-color-l:var(--bulma-info-on-scheme-l)}.message.is-success{--bulma-message-h:var(--bulma-success-h);--bulma-message-s:var(--bulma-success-s);--bulma-message-color-l:var(--bulma-success-on-scheme-l)}.message.is-warning{--bulma-message-h:var(--bulma-warning-h);--bulma-message-s:var(--bulma-warning-s);--bulma-message-color-l:var(--bulma-warning-on-scheme-l)}.message.is-danger{--bulma-message-h:var(--bulma-danger-h);--bulma-message-s:var(--bulma-danger-s);--bulma-message-color-l:var(--bulma-danger-on-scheme-l)}:root{--bulma-navbar-height:3.25rem}.navbar{--bulma-navbar-h:var(--bulma-scheme-h);--bulma-navbar-s:var(--bulma-scheme-s);--bulma-navbar-background-color:var(--bulma-scheme-main);--bulma-navbar-z:30;--bulma-navbar-item-background-a:0;--bulma-navbar-item-background-l:var(--bulma-scheme-main-l);--bulma-navbar-item-background-l-delta:0%;--bulma-navbar-item-hover-background-l-delta:var(--bulma-hover-background-l-delta);--bulma-navbar-item-active-background-l-delta:var(--bulma-active-background-l-delta);--bulma-navbar-item-color-l:var(--bulma-text-l);--bulma-navbar-item-color:hsl(var(--bulma-navbar-h),var(--bulma-navbar-s),var(--bulma-navbar-item-color-l));--bulma-navbar-item-selected-h:var(--bulma-link-h);--bulma-navbar-item-selected-s:var(--bulma-link-s);--bulma-navbar-item-selected-background-l:var(--bulma-link-l);--bulma-navbar-item-selected-color-l:var(--bulma-link-invert-l);--bulma-navbar-item-img-max-height:1.75rem;--bulma-navbar-burger-color:var(--bulma-link);background-color:var(--bulma-navbar-background-color);min-height:var(--bulma-navbar-height);position:relative;z-index:var(--bulma-navbar-z)}.navbar.is-black{--bulma-navbar-h:var(--bulma-black-h);--bulma-navbar-s:var(--bulma-black-s);--bulma-burger-h:var(--bulma-black-h);--bulma-burger-s:var(--bulma-black-s);--bulma-burger-l:var(--bulma-black-invert-l);--bulma-navbar-background-color:var(--bulma-black);--bulma-navbar-item-background-l:var(--bulma-black-l);--bulma-navbar-item-color-l:var(--bulma-black-invert-l);--bulma-navbar-item-selected-h:var(--bulma-black-h);--bulma-navbar-item-selected-s:var(--bulma-black-s);--bulma-navbar-item-selected-background-l:var(--bulma-black-l);--bulma-navbar-item-selected-color-l:var(--bulma-black-invert-l)}.navbar.is-light{--bulma-navbar-h:var(--bulma-light-h);--bulma-navbar-s:var(--bulma-light-s);--bulma-burger-h:var(--bulma-light-h);--bulma-burger-s:var(--bulma-light-s);--bulma-burger-l:var(--bulma-light-invert-l);--bulma-navbar-background-color:var(--bulma-light);--bulma-navbar-item-background-l:var(--bulma-light-l);--bulma-navbar-item-color-l:var(--bulma-light-invert-l);--bulma-navbar-item-selected-h:var(--bulma-light-h);--bulma-navbar-item-selected-s:var(--bulma-light-s);--bulma-navbar-item-selected-background-l:var(--bulma-light-l);--bulma-navbar-item-selected-color-l:var(--bulma-light-invert-l)}.navbar.is-info{--bulma-navbar-h:var(--bulma-info-h);--bulma-navbar-s:var(--bulma-info-s);--bulma-burger-h:var(--bulma-info-h);--bulma-burger-s:var(--bulma-info-s);--bulma-burger-l:var(--bulma-info-invert-l);--bulma-navbar-background-color:var(--bulma-info);--bulma-navbar-item-background-l:var(--bulma-info-l);--bulma-navbar-item-color-l:var(--bulma-info-invert-l);--bulma-navbar-item-selected-h:var(--bulma-info-h);--bulma-navbar-item-selected-s:var(--bulma-info-s);--bulma-navbar-item-selected-background-l:var(--bulma-info-l);--bulma-navbar-item-selected-color-l:var(--bulma-info-invert-l)}.navbar.is-success{--bulma-navbar-h:var(--bulma-success-h);--bulma-navbar-s:var(--bulma-success-s);--bulma-burger-h:var(--bulma-success-h);--bulma-burger-s:var(--bulma-success-s);--bulma-burger-l:var(--bulma-success-invert-l);--bulma-navbar-background-color:var(--bulma-success);--bulma-navbar-item-background-l:var(--bulma-success-l);--bulma-navbar-item-color-l:var(--bulma-success-invert-l);--bulma-navbar-item-selected-h:var(--bulma-success-h);--bulma-navbar-item-selected-s:var(--bulma-success-s);--bulma-navbar-item-selected-background-l:var(--bulma-success-l);--bulma-navbar-item-selected-color-l:var(--bulma-success-invert-l)}.navbar.is-warning{--bulma-navbar-h:var(--bulma-warning-h);--bulma-navbar-s:var(--bulma-warning-s);--bulma-burger-h:var(--bulma-warning-h);--bulma-burger-s:var(--bulma-warning-s);--bulma-burger-l:var(--bulma-warning-invert-l);--bulma-navbar-background-color:var(--bulma-warning);--bulma-navbar-item-background-l:var(--bulma-warning-l);--bulma-navbar-item-color-l:var(--bulma-warning-invert-l);--bulma-navbar-item-selected-h:var(--bulma-warning-h);--bulma-navbar-item-selected-s:var(--bulma-warning-s);--bulma-navbar-item-selected-background-l:var(--bulma-warning-l);--bulma-navbar-item-selected-color-l:var(--bulma-warning-invert-l)}.navbar.is-danger{--bulma-navbar-h:var(--bulma-danger-h);--bulma-navbar-s:var(--bulma-danger-s);--bulma-burger-h:var(--bulma-danger-h);--bulma-burger-s:var(--bulma-danger-s);--bulma-burger-l:var(--bulma-danger-invert-l);--bulma-navbar-background-color:var(--bulma-danger);--bulma-navbar-item-background-l:var(--bulma-danger-l);--bulma-navbar-item-color-l:var(--bulma-danger-invert-l);--bulma-navbar-item-selected-h:var(--bulma-danger-h);--bulma-navbar-item-selected-s:var(--bulma-danger-s);--bulma-navbar-item-selected-background-l:var(--bulma-danger-l);--bulma-navbar-item-selected-color-l:var(--bulma-danger-invert-l)}.navbar-brand{align-items:stretch;display:flex;flex-shrink:0;min-height:var(--bulma-navbar-height)}.navbar-burger{align-items:center;appearance:none;background:none;border:none;border-radius:var(--bulma-burger-border-radius);color:hsl(var(--bulma-burger-h),var(--bulma-burger-s),var(--bulma-burger-l));cursor:pointer;display:inline-flex;flex-direction:column;flex-shrink:0;height:2.5rem;justify-content:center;position:relative;vertical-align:top;width:2.5rem}.navbar-burger span{background-color:currentColor;display:block;height:var(--bulma-burger-item-height);left:calc(50% - (var(--bulma-burger-item-width))/2);position:absolute;transform-origin:center;transition-duration:var(--bulma-duration);transition-property:background-color,color,opacity,transform;transition-timing-function:var(--bulma-easing);width:var(--bulma-burger-item-width)}.navbar-burger span:first-child,.navbar-burger span:nth-child(2){top:calc(50% - (var(--bulma-burger-item-height))/2)}.navbar-burger span:nth-child(3){bottom:calc(50% + var(--bulma-burger-gap))}.navbar-burger span:nth-child(4){top:calc(50% + var(--bulma-burger-gap))}.navbar-burger:hover{background-color:hsla(var(--bulma-burger-h),var(--bulma-burger-s),var(--bulma-burger-l),.1)}.navbar-burger:active{background-color:hsla(var(--bulma-burger-h),var(--bulma-burger-s),var(--bulma-burger-l),.2)}.navbar-burger.is-active span:first-child{transform:rotate(-45deg)}.navbar-burger.is-active span:nth-child(2){transform:rotate(45deg)}.navbar-burger.is-active span:nth-child(3),.navbar-burger.is-active span:nth-child(4){opacity:0}.navbar-burger{align-self:center;color:var(--bulma-navbar-burger-color);margin-inline-end:.375rem;margin-inline-start:auto}.navbar-menu{display:none}.navbar-item{color:var(--bulma-navbar-item-color);display:block;gap:.75rem;line-height:1.5;padding:.5rem .75rem;position:relative}.navbar-item .icon:only-child{margin-left:-.25rem;margin-right:-.25rem}a.navbar-item{background-color:hsla(var(--bulma-navbar-h),var(--bulma-navbar-s),calc(var(--bulma-navbar-item-background-l) + var(--bulma-navbar-item-background-l-delta)),var(--bulma-navbar-item-background-a));cursor:pointer}a.navbar-item:focus,a.navbar-item:focus-within,a.navbar-item:hover{--bulma-navbar-item-background-l-delta:var(--bulma-navbar-item-hover-background-l-delta);--bulma-navbar-item-background-a:1}a.navbar-item:active{--bulma-navbar-item-background-l-delta:var(--bulma-navbar-item-active-background-l-delta);--bulma-navbar-item-background-a:1}a.navbar-item.is-active{--bulma-navbar-h:var(--bulma-navbar-item-selected-h);--bulma-navbar-s:var(--bulma-navbar-item-selected-s);--bulma-navbar-item-background-l:var(--bulma-navbar-item-selected-background-l);--bulma-navbar-item-background-a:1;--bulma-navbar-item-color-l:var(--bulma-navbar-item-selected-color-l)}.navbar-item{flex-grow:0;flex-shrink:0}.navbar-item img,.navbar-item svg{max-height:var(--bulma-navbar-item-img-max-height)}@media screen and (max-width:1023px){.navbar-brand .navbar-item{align-items:center;display:flex}.navbar-menu{background-color:var(--bulma-navbar-background-color);box-shadow:0 .5em 1em hsla(var(--bulma-scheme-h),var(--bulma-scheme-s),var(--bulma-scheme-invert-l),.1);padding:.5rem 0}.navbar-menu.is-active{display:block}}@media screen and (min-width:1024px){.navbar,.navbar-menu,.navbar-start{align-items:stretch;display:flex}.navbar{min-height:var(--bulma-navbar-height)}.navbar-burger{display:none}.navbar-item{align-items:center;display:flex}.navbar-menu{flex-grow:1;flex-shrink:0}.navbar-start{justify-content:flex-start;margin-inline-end:auto}}:root{}.section{--bulma-section-padding:3rem 1.5rem;--bulma-section-padding-desktop:3rem 3rem;padding:var(--bulma-section-padding)}@media screen and (min-width:1024px){.section{padding:var(--bulma-section-padding-desktop)}}:root{}.mb-1{margin-bottom:.25rem!important}.mb-3{margin-bottom:.75rem!important}.p-1{padding:.25rem!important}.pt-3{padding-top:.75rem!important}.pt-4{padding-top:1rem!important}.pt-5{padding-top:1.5rem!important}.is-size-5{font-size:1.25rem!important}.has-text-justified{text-align:justify!important}.has-text-weight-bold{font-weight:700!important}.is-hidden{display:none!important} diff --git a/assets/bulma.min.css.br b/assets/bulma.min.css.br new file mode 100644 Binary files /dev/null and b/assets/bulma.min.css.br differ diff --git a/assets/bulma.min.css.gz b/assets/bulma.min.css.gz new file mode 100644 Binary files /dev/null and b/assets/bulma.min.css.gz differ diff --git a/assets/favicon.svg b/assets/favicon.svg new file mode 100644 --- /dev/null +++ b/assets/favicon.svg @@ -0,0 +1,35 @@ + + + + + + + + + + + + + diff --git a/assets/favicon.svg.br b/assets/favicon.svg.br new file mode 100644 Binary files /dev/null and b/assets/favicon.svg.br differ diff --git a/assets/favicon.svg.gz b/assets/favicon.svg.gz new file mode 100644 Binary files /dev/null and b/assets/favicon.svg.gz differ diff --git a/assets/flake.svg b/assets/flake.svg new file mode 100644 --- /dev/null +++ b/assets/flake.svg @@ -0,0 +1,50 @@ + + + + diff --git a/assets/flake.svg.br b/assets/flake.svg.br new file mode 100644 Binary files /dev/null and b/assets/flake.svg.br differ diff --git a/assets/flake.svg.gz b/assets/flake.svg.gz new file mode 100644 Binary files /dev/null and b/assets/flake.svg.gz differ diff --git a/assets/flush.mp3 b/assets/flush.mp3 new file mode 100644 Binary files /dev/null and b/assets/flush.mp3 differ diff --git a/assets/github.svg b/assets/github.svg new file mode 100644 --- /dev/null +++ b/assets/github.svg @@ -0,0 +1,31 @@ + + + + + + + diff --git a/assets/github.svg.br b/assets/github.svg.br new file mode 100644 Binary files /dev/null and b/assets/github.svg.br differ diff --git a/assets/github.svg.gz b/assets/github.svg.gz new file mode 100644 Binary files /dev/null and b/assets/github.svg.gz differ diff --git a/assets/robots.txt b/assets/robots.txt new file mode 100644 --- /dev/null +++ b/assets/robots.txt @@ -0,0 +1,5 @@ +User-agent: * +Allow: /index.html$ +Allow: /$ +Disallow: / +Sitemap: https://pieceofflakenixrepository.org/sitemap.xml diff --git a/assets/sitemap.xml b/assets/sitemap.xml new file mode 100644 --- /dev/null +++ b/assets/sitemap.xml @@ -0,0 +1,7 @@ + + + + https://pieceofflakenixrepository.org/index.html + 2026-03-14 + + diff --git a/assets/snow.mp3 b/assets/snow.mp3 new file mode 100644 Binary files /dev/null and b/assets/snow.mp3 differ diff --git a/assets/style.css b/assets/style.css new file mode 100644 --- /dev/null +++ b/assets/style.css @@ -0,0 +1,132 @@ +/* A Piece of Flake */ + +@property --navy-logo-color { + syntax: ""; + inherits: false; + initial-value: #5277c3; +} + +@property --blue-logo-color { + syntax: ""; + inherits: false; + initial-value: #7ebae4; +} + +.ani-run .ani { + position: fixed; + z-index: 111; + display: block; + animation-play-state: running; + animation-delay: 0s; + animation-direction: normal; + animation-iteration-count: infinite; + animation-timing-function: cubic-bezier(0, 0.38, 1, 1); +} + +.ani { + display: none; + width: 7vh; + height: 7vh; + animation-play-state: paused; +} + +@keyframes fall1 { + from { + transform: translateY(-6vh) translateX(-6vw); + opacity: 1; + } + to { + transform: translateY(100vh) translateX(10vw) rotate(360deg); + opacity: 0.2; + } +} +@keyframes fall2 { + from { + transform: translateY(-6vh) translateX(16vw); + opacity: 1; + } + to { + transform: translateY(100vh) translateX(30vw) rotate(360deg); + opacity: 0.3; + } +} +@keyframes fall3 { + from { + transform: translateY(-6vh) translateX(44vw); + opacity: 1; + } + to { + transform: translateY(100vh) translateX(70vw) rotate(-360deg); + opacity: 0.2; + } +} +@keyframes fall4 { + from { + transform: translateY(-6vh) translateX(90vw); + opacity: 1; + } + to { + transform: translateY(100vh) translateX(70vw) rotate(-360deg); + opacity: 0.4; + } +} + +.fall1 { animation-duration: 3s; animation-name: fall1; } +.fall2 { animation-duration: 2.2s; animation-name: fall2; } +.fall3 { animation-duration: 3.4s; animation-name: fall3; } +.fall4 { animation-duration: 2.7s; animation-name: fall4; animation-delay: 0.3s; } + +.navbar .navbar-brand a.navbar-item { + color: var(--navy-logo-color); +} + +button.is-black { + font-weight: bold; +} + +body { + background: radial-gradient(circle, var(--blue-logo-color) 0%, var(--blue-logo-color) 80%, var(--navy-logo-color) 100%); + color: #171717; + min-height: 100vh; + overflow: scroll !important; + --bulma-body-overflow-x: scroll !important; +} + +body pre.error { + white-space: pre-wrap; +} + +body .table { + background: transparent; +} + +body .table.is-bordered td { + border-width: 2px; +} + +body .table td { + border-color: hsl(221.1, 17.8%, 61.4%); +} + +.input.is-success { + border-width: 4px !important; + box-shadow: 20px 10px 20px green !important; + background-color: var(--bulma-success-25-invert); +} + +.input.is-danger { + border-width: 4px !important; + box-shadow: 10px 5px 5px var(--bulma-danger) !important; + background-color: var(--bulma-danger-25-invert); +} + +img.github { + width: 8vh; + opacity: 0.8; + padding-top: 5vh; +} + +.content p { + overflow-wrap: break-word; + text-wrap: pretty; +} diff --git a/assets/style.css.br b/assets/style.css.br new file mode 100644 Binary files /dev/null and b/assets/style.css.br differ diff --git a/assets/style.css.gz b/assets/style.css.gz new file mode 100644 Binary files /dev/null and b/assets/style.css.gz differ diff --git a/changelog.md b/changelog.md new file mode 100644 --- /dev/null +++ b/changelog.md @@ -0,0 +1,4 @@ +# a-piece-of-flake changelog + +## Version 0.0.1 2026-02-26 + * init diff --git a/src/PieceOfFlake/Acid.hs b/src/PieceOfFlake/Acid.hs new file mode 100644 --- /dev/null +++ b/src/PieceOfFlake/Acid.hs @@ -0,0 +1,57 @@ +{-# LANGUAGE TypeFamilies #-} +module PieceOfFlake.Acid where + +import Data.Acid + ( query, + update, + openLocalStateFrom, + makeAcidic, + AcidState, + Query, + Update ) +import PieceOfFlake.CmdArgs ( AcidFlakesPath ) +import Data.SafeCopy ( deriveSafeCopy, base ) +import PieceOfFlake.Flake ( FlakeUrl, Flake ) +import PieceOfFlake.Prelude +import PieceOfFlake.Stm ( TQueue, readTQueue, peekTQueue ) +import UnliftIO.Retry + ( fibonacciBackoff, limitRetries, recoverAll ) + + +newtype AcidFlakes = AcidFlakes { unAcidFlakes :: [(FlakeUrl, Flake)] } + +$(deriveSafeCopy 0 'base ''AcidFlakes) + +type Message = (FlakeUrl, Flake) +-- Transactions are defined to run in either the 'Update' monad +-- or the 'Query' monad. +addMessage :: Message -> Update AcidFlakes () +addMessage msg + = do AcidFlakes messages <- get + put $ AcidFlakes (msg:messages) + +viewMessages :: Query AcidFlakes [Message] +viewMessages = unAcidFlakes <$> ask + +-- This will define @ViewMessage@ and @AddMessage@ for us. +$(makeAcidic ''AcidFlakes ['addMessage, 'viewMessages]) + +openFlakeDb :: MonadIO m => Tagged AcidFlakesPath FilePath -> m (AcidState AcidFlakes) +openFlakeDb (Tagged dbLocation) = + liftIO $ openLocalStateFrom dbLocation (AcidFlakes []) + +updateDb :: PoF m => AcidState AcidFlakes -> Message -> m () +updateDb db m = do + liftIO $ update db (AddMessage m) + $(logInfo) $ "Persisted flake " <> show (fst m) + +loadFromDb :: MonadIO m => AcidState AcidFlakes -> m [(FlakeUrl, Flake)] +loadFromDb db = liftIO (query db ViewMessages) + +runPersistQueue :: PoF m => AcidState AcidFlakes -> TQueue (FlakeUrl, Flake) -> m () +runPersistQueue db q = finally go (void $ atomically $ readTQueue q) + where + go = + recoverAll + (fibonacciBackoff 100_000 <> limitRetries 3) + (\_ -> updateDb db =<< atomically (peekTQueue q)) diff --git a/src/PieceOfFlake/Aeson.hs b/src/PieceOfFlake/Aeson.hs new file mode 100644 --- /dev/null +++ b/src/PieceOfFlake/Aeson.hs @@ -0,0 +1,25 @@ +module PieceOfFlake.Aeson + ( module Data.Aeson + , Some (..) + , someToList + ) where + +import Data.Aeson + ( Value(Array, Number, String, Bool, Object, Null), + FromJSON(parseJSON) ) +import PieceOfFlake.Prelude ( Eq, Show, (<$>) ) + +data Some x = Atom x | More [x] deriving (Show, Eq) + +instance FromJSON x => FromJSON (Some x) where + parseJSON n@Number {} = Atom <$> parseJSON n + parseJSON n@String {} = Atom <$> parseJSON n + parseJSON n@Bool {} = Atom <$> parseJSON n + parseJSON n@Object {} = Atom <$> parseJSON n + parseJSON n@Null {} = Atom <$> parseJSON n + parseJSON n@Array {} = More <$> parseJSON n + +someToList :: Some a -> [a] +someToList = \case + Atom x -> [x] + More xs -> xs diff --git a/src/PieceOfFlake/CmdArgs.hs b/src/PieceOfFlake/CmdArgs.hs new file mode 100644 --- /dev/null +++ b/src/PieceOfFlake/CmdArgs.hs @@ -0,0 +1,354 @@ +{-# LANGUAGE DuplicateRecordFields #-} +{-# LANGUAGE TemplateHaskell #-} +module PieceOfFlake.CmdArgs where + +import Data.Char (isDigit) +import Data.Either.Combinators ( mapLeft ) +import GHC.TypeLits (KnownSymbol) +import Network.HostName ( getHostName ) +import Options.Applicative +import PieceOfFlake.Flake ( FetcherId(..) ) +import PieceOfFlake.Prelude +import PieceOfFlake.Prelude qualified as P +import PieceOfFlake.Req ( DynamicUrl(UrlHttp), http, port, parseUrl ) +import PieceOfFlake.WebService + ( FetcherHeartbeatPeriod, FetcherSecret, NoSubmitionHeartbeatSec ) +import Text.Show ( Show(show) ) + +data HttpPort +data Cert +data CertKey +data AcidFlakesPath +data StaticCacheSeconds +data RawNixCacheOutput +data BaseUrl +data ResubmitPeriod +type IndexQueryCacheSize = "index-query-cache-size" +type BadFlakeMaxAge = "bad-flake-max-age" + +newtype RingBufferSize = RingBufferSize (Refined (FromTo 1 12) Int) + +instance Show RingBufferSize where + show = P.show . unrefine . coerce + + +data WsCmdArgs + = WsCmdArgs + { httpPortToListen :: Tagged HttpPort Int + , certFile :: Maybe (Tagged Cert FilePath) + , keyFile :: Maybe (Tagged CertKey FilePath) + , acidFlakes :: Tagged AcidFlakesPath FilePath + , staticCache :: Tagged StaticCacheSeconds Word32 + , baseUrl :: Tagged BaseUrl Text + , fetcherSecretPath :: Tagged FetcherSecret FilePath + , noSubmitionHeartbeat :: Tagged NoSubmitionHeartbeatSec Second + , allowResubmitBadFlakeIn :: Tagged ResubmitPeriod NominalDiffTime + , logLevel :: LogLevel + , indexQueryCacheSize :: Tagged IndexQueryCacheSize Word + , ringBufferSize :: RingBufferSize + , badFlakeMaxAge :: Tagged BadFlakeMaxAge Second + , allowResubmitIndexedFlakeIn :: Tagged ResubmitPeriod NominalDiffTime + , fetcherHeartbeatPeriod :: Tagged FetcherHeartbeatPeriod Second + } + deriving Show + +type RawNixCacheMaxAge = "nix-cache-max-age" +type RawNixCacheErrorMaxAge = "nix-cache-err-max-age" +type LooseFlakes = "loose-flake" +type NixGCDiskCriticalCapacity = "nix-gc-used" +data FetcherCmdArgs + = FetcherCmdArgs + { webServiceUrl :: DynamicUrl + , rawNixCache :: Tagged RawNixCacheOutput (Maybe FilePath) + , fetcherId :: FetcherId + , fetcherSecretPath :: Tagged FetcherSecret FilePath + , rawNixCacheMaxAge :: Tagged RawNixCacheMaxAge NominalDiffTime + , rawNixCacheErrMaxAge :: Tagged RawNixCacheErrorMaxAge NominalDiffTime + , logLevel :: LogLevel + , looseFlakes :: Tagged LooseFlakes Bool + , runNixGCIfUsedMoreThan :: Tagged NixGCDiskCriticalCapacity Percent + } + deriving (Show) + +data SubmitListOfFlakesArgs + = SubmitListOfFlakesArgs + { webServiceUrl :: DynamicUrl + , logLevel :: LogLevel + , indexTimeoutIn :: Tagged ResubmitPeriod NominalDiffTime + } + deriving (Show) + +data CmdArgs + = WebService WsCmdArgs + | FetcherJob FetcherCmdArgs + | SubmitListOfFlakes SubmitListOfFlakesArgs + | PieceOfFlakeVersion + deriving (Show) + +execWithArgs :: MonadIO m => (CmdArgs -> m a) -> [String] -> m a +execWithArgs a args = a =<< liftIO (handleParseResult $ execParserPure defaultPrefs (info (cmdp <**> helper) phelp) args) + where + serviceP = WebService <$> (WsCmdArgs <$> portOption <*> certO <*> + certKeyO <*> acidOption <*> cacheSecondsO <*> + baseUrlO <*> fetcherSecretPathO <*> noSubmitionHeartbeatO <*> + allowResubmitBadFlakeInO <*> logLevelO <*> + indexQueryCacheSizeO <*> ringBufferSizeO <*> badFlakeMaxAgeO <*> + allowResubmitIndexedFlakeInO <*> fetcherHeartbeatPeriodO) + fetcherP = FetcherJob <$> (FetcherCmdArgs <$> urlOption <*> rawNixCacheO <*> + customFetcherIdO <*> fetcherSecretPathO <*> + rawNixCacheMaxAgeO @RawNixCacheMaxAge (7 * 24 * 3600) <*> + rawNixCacheMaxAgeO @RawNixCacheErrorMaxAge 800 <*> + logLevelO <*> looseFlakesO <*> nixGcDiskUsedO) + submitListP = SubmitListOfFlakes <$> + (SubmitListOfFlakesArgs <$> urlOption <*> logLevelO <*> allowResubmitBadFlakeInO) + cmdp = + hsubparser + ( command "web" (infoP serviceP "launch web service") + <> command "fetcher" (infoP fetcherP "launch fetcher job") + <> command "submit-list" (infoP submitListP "submit list of flakes read for stdin - an url per line") + <> command "version" (infoP (pure PieceOfFlakeVersion) "print program version")) + + infoP p h = info p (progDesc h <> fullDesc) + phelp = + progDesc + "Nix Flake repository" + +defaultPort :: Int +defaultPort = 3003 + +newtype Percent = Percent Word8 deriving newtype (Eq, Ord) +instance Show Percent where + show (Percent p) = P.show p <> "%" + +parsePercent :: String -> Maybe Percent +parsePercent s = + case span isDigit s of + (digits@(_:_), "%") -> Percent <$> readMaybe digits + _bad -> Nothing + +nixGcDiskUsedO :: Parser (Tagged NixGCDiskCriticalCapacity Percent) +nixGcDiskUsedO = Tagged <$> + option (maybeReader parsePercent) + ( long (symbolVal $ Proxy @NixGCDiskCriticalCapacity) + <> showDefault + <> value (Percent 90) + <> help "percent of root volume usage when nix GC is triggerred before fetching a flake" + ) + +allowResubmitBadFlakeInO :: Parser (Tagged ResubmitPeriod NominalDiffTime) +allowResubmitBadFlakeInO = Tagged <$> + option auto + ( long "resubmit-bad-flake-interval" + <> showDefault + <> value 600 + <> help "how soon bad flake can be resubmitted" + ) + +allowResubmitIndexedFlakeInO :: Parser (Tagged ResubmitPeriod NominalDiffTime) +allowResubmitIndexedFlakeInO = Tagged <$> + option auto + ( long "resubmit-indexed-flake-interval" + <> showDefault + <> value (toNominal (2 :: Hour)) + <> help "how soon an indexed flake can be resubmitted" + ) + +looseFlakesO :: Parser (Tagged LooseFlakes Bool) +looseFlakesO = Tagged <$> + option auto + ( long (symbolVal $ Proxy @LooseFlakes) + <> showDefault + <> value False + <> help "loose fetched flake - debugging web service response" + ) + +logLevelO :: Parser LogLevel +logLevelO = + option auto + ( long "log-level" + <> short 'l' + <> showDefault + <> value LevelDebug + <> help "app log level" + <> metavar "LOG" + ) + +rawNixCacheMaxAgeO :: forall a. KnownSymbol a => NominalDiffTime -> Parser (Tagged a NominalDiffTime) +rawNixCacheMaxAgeO defVal = + Tagged <$> + option auto + ( long (symbolVal $ Proxy @a) + <> showDefault + <> value defVal + <> help "max age of cached entity" + ) + +badFlakeMaxAgeO :: Parser (Tagged BadFlakeMaxAge Second) +badFlakeMaxAgeO = Tagged <$> + option auto + ( long (symbolVal $ Proxy @BadFlakeMaxAge) + <> showDefault + <> value (8 * 3600) + <> help "bad flake max age" + ) + +fetcherHeartbeatPeriodO :: Parser (Tagged FetcherHeartbeatPeriod Second) +fetcherHeartbeatPeriodO = Tagged <$> + option auto + ( long "fetcher-heartbeat" + <> showDefault + <> value 10 + <> help "period of heartbeats from fetcher to make WS keep flake association with fetcher" + ) + +noSubmitionHeartbeatO :: Parser (Tagged NoSubmitionHeartbeatSec Second) +noSubmitionHeartbeatO = Tagged <$> + option auto + ( long "heartbeat" + <> short 'b' + <> showDefault + <> value 600 + <> help "period for putting empty submition request into fetcher queue (in seconds) when queue is empty" + <> metavar "HEARTBEAT" + ) + +ringBufferSizeO :: Parser RingBufferSize +ringBufferSizeO = + option (eitherReader (mapLeft toString . readEither >=> fmap RingBufferSize . mapLeft P.show . refine)) + ( long "ring-buffer" + <> showDefault + <> value (RingBufferSize $$(refineTH 3)) + <> help "ring buffer size that is used for mean values" + ) + +indexQueryCacheSizeO :: Parser (Tagged IndexQueryCacheSize Word) +indexQueryCacheSizeO = Tagged <$> + option auto + ( long (symbolVal $ Proxy @IndexQueryCacheSize) + <> showDefault + <> value 100 + <> help "index query cache size" + ) + +cacheSecondsO :: Parser (Tagged StaticCacheSeconds Word32) +cacheSecondsO = Tagged <$> + option auto + ( long "static-cache" + <> showDefault + <> value 1 + <> help "cache duration for static content (used in HTTP header)" + <> metavar "STATIC_CACHE" + ) + +rawNixCacheO :: Parser (Tagged RawNixCacheOutput (Maybe FilePath)) +rawNixCacheO = Tagged <$> + option (eitherReader parse ) + ( long "raw-nix-cache-output" + <> short 'c' + <> showDefault + <> value (pure "fetcher-raw-nix-cache") + <> help "path to fetcher cache of nix raw output" + <> metavar "RAW_NIX_CACHE" + ) + where + parse = \case + "" -> pure Nothing + "null" -> pure Nothing + "-" -> pure Nothing + o -> pure $ Just o + +customFetcherIdO :: Parser FetcherId +customFetcherIdO = FetcherId . toText <$> + option str + ( long "fetcher-id" + <> short 'i' + <> showDefault + <> value (unsafePerformIO getHostName) + <> help "fetcher id" + <> metavar "FID" + ) + +fetcherSecretPathO :: Parser (Tagged FetcherSecret FilePath) +fetcherSecretPathO = Tagged <$> + option str + ( long "fetcher-secret" + <> short 's' + <> showDefault + <> value ".fetcher-secret" + <> help "path to file with secret for fetcher authentication on web service" + <> metavar "SECRET" + ) + +acidOption :: Parser (Tagged AcidFlakesPath FilePath) +acidOption = Tagged <$> + option str + ( long "acid" + <> short 'a' + <> showDefault + <> value "acid-flakes/" + <> help "path do ACID flake store" + <> metavar "ACID" + ) + +baseUrlO :: Parser (Tagged BaseUrl Text) +baseUrlO = Tagged <$> + option str + ( long "base-url" + <> short 'u' + <> showDefault + <> value ("http://localhost:" <> P.show defaultPort) + <> help "base web service url for HTML links" + <> metavar "URL" + ) + +urlOption :: Parser DynamicUrl +urlOption = + option (eitherReader (mapLeft toString . parseUrl . toText)) + ( long "url" + <> short 'u' + <> showDefault + <> value (UrlHttp (http "localhost") (port defaultPort)) + <> help "web service url for fetching flake submition requests and uploading flake meta back" + <> metavar "URL" + ) + +portOption :: Parser (Tagged HttpPort Int) +portOption = Tagged <$> + option auto + ( long "port" + <> short 'p' + <> showDefault + <> value defaultPort + <> help "HTTP(S) port to listen" + <> metavar "PORT" + ) + +emptyToNothing :: FilePath -> Maybe FilePath +emptyToNothing "" = Nothing +emptyToNothing s = Just s + +zeroToNothing :: Int -> Maybe Int +zeroToNothing s | s <= 0 = Nothing + | otherwise = Just s + +pured :: (Applicative g, Applicative f) => g a -> g (f a) +pured = fmap pure + +certO :: Parser (Maybe (Tagged Cert FilePath)) +certO = pured . emptyToNothing <$> + strOption + ( long "certificate" + <> short 'c' + <> value "" + <> help "path to SSL certificate file (./certificate.pem)" + <> metavar "CERT" + ) +certKeyO :: Parser (Maybe (Tagged CertKey FilePath)) +certKeyO = pured . emptyToNothing <$> + strOption + ( long "key" + <> short 'k' + <> value "" + <> help "path to key file of SSL certificate (./key.pem)" + <> metavar "KEY" + ) diff --git a/src/PieceOfFlake/CmdRun.hs b/src/PieceOfFlake/CmdRun.hs new file mode 100644 --- /dev/null +++ b/src/PieceOfFlake/CmdRun.hs @@ -0,0 +1,102 @@ +module PieceOfFlake.CmdRun where + +import Data.Version (showVersion) +import Data.ByteString.Char8 qualified as S8 +import Paths_a_piece_of_flake ( version ) +import PieceOfFlake.Acid ( loadFromDb, openFlakeDb, runPersistQueue ) +import PieceOfFlake.CmdArgs + ( WsCmdArgs(staticCache, indexQueryCacheSize, acidFlakes, + ringBufferSize, logLevel, fetcherSecretPath, noSubmitionHeartbeat, fetcherHeartbeatPeriod), + CmdArgs(..), + FetcherCmdArgs(fetcherSecretPath, FetcherCmdArgs, logLevel), + SubmitListOfFlakesArgs(logLevel) ) +import PieceOfFlake.Fetcher ( runFetcher ) +import PieceOfFlake.Flake.Repo + ( FlakeRepo(flakeIndex, wsArgs, acidFlakes, acidQueue, repoStats, + flakes), + mkFlakeRepo, + removeOldBadFlakes, + sendEmptyFlakeSubmition, + resubmitFlakesFetchingByZombie ) +import PieceOfFlake.Http ( runWebService ) +import PieceOfFlake.Index + ( mkFlakeIndex, loadIndexFromScratch, consumeIndexQueue ) +import PieceOfFlake.Page ( Ypp(Ypp) ) +import PieceOfFlake.Prelude +import PieceOfFlake.Stats ( mkRepoStats ) +import PieceOfFlake.SubmitList ( runSubmitList ) +import PieceOfFlake.WebService + ( FetcherSecret(..), NoSubmitionHeartbeatSec ) +import StmContainers.Map ( newIO ) +import UnliftIO.Concurrent ( forkFinally ) +import UnliftIO.Retry ( fibonacciBackoff, recoverAll ) + +initRepo :: PoF m => FetcherSecret -> WsCmdArgs -> m FlakeRepo +initRepo fsec wsa = do + flakesMap <- liftIO newIO + fi <- mkFlakeIndex wsa.indexQueryCacheSize + acidFlakeStorage <- openFlakeDb wsa.acidFlakes + rs <- mkRepoStats wsa.ringBufferSize + loadIndexFromScratch rs fi flakesMap . reverse =<< loadFromDb acidFlakeStorage + mkFlakeRepo fsec wsa fi flakesMap acidFlakeStorage rs + +launchBackgroundThreads :: PoF m => + Tagged NoSubmitionHeartbeatSec Second -> FlakeRepo -> m () +launchBackgroundThreads period fr = do + forkForever "Lost flake resubmition" $ do + resubmitFlakesFetchingByZombie fr + threadDelay . (* 3) $ untag fr.wsArgs.fetcherHeartbeatPeriod + forkForever "Flake persistence" $ do + recoverAll + (fibonacciBackoff 10000) + (\_ -> runPersistQueue fr.acidFlakes fr.acidQueue) + forkForever "Flake text search indexer" + $ consumeIndexQueue fr.repoStats fr.flakes fr.flakeIndex + forkForever "Empty Submition" $ do + threadDelay $ untag period + sendEmptyFlakeSubmition fr + forkForever "Old Bad Flake Collector" $ removeOldBadFlakes fr + where + forkForever thrName f = do + tid <- forkFinally + (forever f) + (\case + Left e -> $(logError) $ thrName <> " thread ended: " <> show e + Right () -> $(logInfo) $ thrName <> " thread ended without errors") + $(logInfo) $ thrName <> " thread is forked with tid: " <> show tid + +-- commented lines below are excuted via: @ghciwatch --enable-eval@ +-- $> import PieceOfFlake.CmdArgs +-- $> execWithArgs runCmd . (:[]) =<< (fromMaybe "web" <$> PieceOfFlake.Prelude.lookupEnv "E") +runCmd :: CmdArgs -> IO () +runCmd = \case + WebService ws -> + withLogs ws.logLevel $ do + $(logInfo) $ "Start WebService " <> show ws + fr <- (`initRepo` ws) =<< loadFetcherSecret ws.fetcherSecretPath + launchBackgroundThreads ws.noSubmitionHeartbeat fr + runWebService ws $ Ypp fr ws.staticCache + FetcherJob fa@FetcherCmdArgs {} -> + withLogs fa.logLevel $ do + $(logInfo) $ "Start Fetcher " <> show fa + runFetcher fa =<< loadFetcherSecret fa.fetcherSecretPath + SubmitListOfFlakes sla -> + withLogs sla.logLevel $ runSubmitList sla + PieceOfFlakeVersion -> + putStrLn $ "Version " <> showVersion version + +withLogs :: MonadIO m => LogLevel -> LoggingT m a -> m a +withLogs minLogL m = + runFlushingStdoutLoggingT $ filterLogger (\_ l -> l >= minLogL) m + where + runFlushingStdoutLoggingT = (`runLoggingT` flushingOutput stdout) + where + -- without explicit flushing - syslog does to get logs + flushingOutput h loc src level msg = S8.hPutStr h ls >> hFlush h + where + ls = defaultLogStrBS loc src level msg + defaultLogStrBS a b c d = fromLogStr $ defaultLogStr a b c d + + +loadFetcherSecret :: MonadIO m => Tagged FetcherSecret FilePath -> m FetcherSecret +loadFetcherSecret a = readFileTxt (untag a) <&> FetcherSecret diff --git a/src/PieceOfFlake/Fetcher.hs b/src/PieceOfFlake/Fetcher.hs new file mode 100644 --- /dev/null +++ b/src/PieceOfFlake/Fetcher.hs @@ -0,0 +1,413 @@ +module PieceOfFlake.Fetcher where + +import Crypto.Hash.SHA1 (hashlazy) +import Data.Aeson qualified as A +import Data.Aeson + ( FromJSON(parseJSON), + eitherDecodeStrict, + genericParseJSON, + defaultOptions, + Options(fieldLabelModifier) ) +import Data.Char ( toLower ) +import Data.ByteArray.Encoding (Base(Base16), convertToBase) +import Data.ByteString.Char8 qualified as BS8 +import Data.ByteString.Lazy qualified as LBS +import Data.Map.Strict qualified as M +import Data.Text qualified as T +import PieceOfFlake.Aeson ( Some, someToList ) +import PieceOfFlake.CmdArgs +import PieceOfFlake.Flake + ( FlakeUrl(..), + Architecture(..), + PackageName(..), + PackageInfo(..), + MetaFlake(..) ) +import PieceOfFlake.Flake.Repo ( FetcherReq(FetcherReq) ) +import PieceOfFlake.Prelude +import PieceOfFlake.Req + ( DynamicUrl, + setResponseTimeout, + (/:), + defaultHttpConfig, + isStatusCodeException, + jsonResponse, + responseBody, + responseStatusCode, + runReq, + GET(GET), + HttpConfig(httpConfigRetryJudge, httpConfigRetryJudgeException), + HttpException(VanillaHttpException), + NoReqBody(NoReqBody), + POST(POST), + ReqBodyJson(ReqBodyJson), + dynReq, + dynReq' ) + +import PieceOfFlake.WebService + ( FetcherAutoConfig(httpMinTimeout, heartbeatPeriod), + FetcherHeartbeat(fetcherSecret, FetcherHeartbeat, fetcherId, + workingOn), + FetcherSecret, + Period(Period, unPeriod) ) +import System.Exit ( ExitCode(ExitFailure, ExitSuccess) ) +import System.Process ( callProcess, readProcess, readProcessWithExitCode ) +import Text.Regex.TDFA ( AllTextSubmatches(getAllTextSubmatches), (=~) ) +import UnliftIO.Async ( withAsync ) +import UnliftIO.Retry + ( capDelay, + fibonacciBackoff, + limitRetries, + recoverAll, + recovering ) +import UnliftIO.Directory + ( doesFileExist, + createDirectoryIfMissing, + removeFile ) +import System.FilePath ( () ) + + +showDigest :: ByteString -> String +showDigest = BS8.unpack . convertToBase Base16 + +readJsonDirect :: (FromJSON a, MonadIO m) => Text -> [Text] -> m a +readJsonDirect prg prgArgs = do + prgOut <- liftIO (readProcess (toString prg) (toString <$> prgArgs) "") + case eitherDecodeStrict $ encodeUtf8 prgOut of + Right x -> pure x + Left e -> + throwIO . stringException $ + "Failed to parse output as JSON:\n" <> prgOut <> "\nFrom: " <> + toString (T.intercalate " " prgArgs) <> "\nError: " <> e + +newtype ReadJsonCached = ReadJsonCached FilePath + +{- hash path cut 2 chars join with mkdir -p + by that path create files out.json and cmd.sh +-} + +isFileOlderThan :: (MonadIO m, ClockMonad m) => Tagged a FilePath -> Tagged a NominalDiffTime -> m Bool +isFileOlderThan (Tagged fp) (Tagged age) = do + getModificationTime fp >>= (`doAfter` \mt -> do now <- getTimeAfter mt; pure $ now `diffUTCTime` mt > age) + +readJsonCached :: (FromJSON a, FetcherM m) => FilePath -> Text -> [Text] -> m a +readJsonCached cacheDir prg prgArgs = do + let cmd = LBS.intercalate " " (encodeUtf8 <$> (prg : prgArgs)) + (cmdH2, cmdHashRest) = splitAt 2 $ showDigest $ hashlazy cmd + callDir = cacheDir cmdH2 cmdHashRest + exOut = Tagged @RawNixCacheErrorMaxAge $ callDir "err.txt" + outJson = Tagged @RawNixCacheMaxAge $ callDir "out.json" + cmdFile = callDir "cmd.sh" + doDirect = do + $(logDebug) $ "Direct read " <> show cmd + createDirectoryIfMissing True callDir + prgOut <- liftIO $ do + readProcessWithExitCode (toString prg) (toString <$> prgArgs) "" >>= \case + (ExitSuccess, out, _err) -> pure out + (ExitFailure ec, _, err) -> do + writeFileBS (untag exOut) $ encodeUtf8 err + throwIO . stringException $ "Command [" <> show cmd <> "] exited with: " <> + show ec <> " and output: " <> err + let prgOutBs = encodeUtf8 prgOut + writeFileBS (untag outJson) prgOutBs + writeFileLBS cmdFile cmd + case eitherDecodeStrict prgOutBs of + Right x -> do + pure x + Left e -> + throwIO . stringException $ + "Failed to parse output as JSON:\n" <> prgOut <> "\nFrom: " <> + toString (T.intercalate " " prgArgs) <> "\nError: " <> e + + $(logInfo) $ "readJsonCached " <> show cmd + ifM (doesFileExist $ untag exOut) + (do + $(logDebug) $ "Ex cache exist for " <> show cmd + ifM (isFileOlderThan exOut . rawNixCacheErrMaxAge =<< asks fetcherArgs) + (do $(logDebug) $ "Ex cache is expired for " <> show cmd + removeFile $ untag exOut + doDirect) + (throwIO . stringException . show =<< readFileBS (untag exOut)) + ) + (ifM (doesFileExist $ untag outJson) + (do + $(logDebug) $ "Out cache exist for " <> show cmd + ifM (isFileOlderThan outJson . rawNixCacheMaxAge =<< asks fetcherArgs) + doDirect + (do + $(logDebug) $ "Read cache from: " <> show outJson + jsonLbs <- readFileBS $ untag outJson + case eitherDecodeStrict jsonLbs of + Right x -> pure x + Left e -> + throwIO . stringException $ + "Failed to parse output as JSON:\n" <> show jsonLbs <> "\nFrom: " <> + toString (T.intercalate " " prgArgs) <> "\nError: " <> e + ) + ) + doDirect + ) + +data FetcherConf + = FetcherConf + { fetUrl :: DynamicUrl + , arch :: Architecture + , fetcherSecret :: FetcherSecret + , fetcherArgs :: FetcherCmdArgs + , flakeLoser :: Maybe (FlakeUrl, Either Text MetaFlake) -> Maybe (FlakeUrl, Either Text MetaFlake) + , confFromWs :: FetcherAutoConfig + } + +type FetcherM m = (MonadReader FetcherConf m, PoF m) + +readJson :: (FetcherM m, FromJSON a) => Text -> [Text] -> m a +readJson prg prgArgs = do + asks fetcherArgs <&> rawNixCache >>= \case + Tagged Nothing -> readJsonDirect prg prgArgs + Tagged (Just c) -> readJsonCached c prg prgArgs + +data RawFlakeOrigin + = RawFlakeOrigin + { flakeOriginType :: Text + , flakeOriginOwner :: Maybe Text + , flakeOriginRepo :: Maybe Text + , flakeOriginRev :: Maybe Text + , flakeOriginUrl :: Maybe Text + } deriving (Show, Eq, Generic) + +instance FromJSON RawFlakeOrigin where + parseJSON = genericParseJSON + (defaultOptions { fieldLabelModifier = fmap toLower . drop (length ("flakeOrigin" :: String)) }) + +data RawNode + = RawNode + { flake :: Maybe Bool + , original :: Maybe RawFlakeOrigin + } deriving (Show, Eq, Generic) +instance FromJSON RawNode + +newtype RawLocks + = RawLocks + { nodes :: Map Text RawNode + } deriving (Show, Eq, Generic) +instance FromJSON RawLocks + +data RawFlakeInfo + = RawFlakeInfo + { revision :: Text + , description :: Maybe Text + , lastModified :: Integer + , locks :: RawLocks + } deriving (Show, Eq, Generic) +instance FromJSON RawFlakeInfo + +-- Filesystem Size Used Avail Use% Mounted on +-- /dev/nvme0n1p5 907G 485G 376G 57% / +dfOutputRegex :: String +dfOutputRegex = " (1?[0-9]{1,2}[%])" + +parseDfLine :: String -> Maybe Percent +parseDfLine l = + case getAllTextSubmatches (l =~ dfOutputRegex) of + ([_full, pG ] :: [String]) -> parsePercent pG + _ -> Nothing + +rootDiskUsedSpacePercent :: FetcherM m => m Percent +rootDiskUsedSpacePercent = do + liftIO (readProcess "df" ["-h", "/"] "") <&> mapMaybe (parseDfLine . toString) . lines . toText >>= \case + [] -> throwIO . stringException $ "No result from df -h /" + [r] -> do + $(logDebug) $ "Root volume used space: " <> show r + pure r + mu -> throwIO . stringException $ "Multiple results " <> show mu + +runNixCG :: FetcherM m => m () +runNixCG = liftIO $ callProcess "nix-store" ["--gc"] + +checkDiskFreeSpace :: FetcherM m => m () +checkDiskFreeSpace = do + Tagged criticalDiskUsage <- runNixGCIfUsedMoreThan <$> asks fetcherArgs + diskUsage <- rootDiskUsedSpacePercent + if criticalDiskUsage <= diskUsage + then runNixCG + else $(logDebug) "Skip nix GC step" + +nixFlakeInfo :: FetcherM m => FlakeUrl -> m RawFlakeInfo +nixFlakeInfo (FlakeUrl fu) = + readJson "nix" ["flake", "metadata", "--json", fu] + +nixCurrentArch :: MonadIO m => m Architecture +nixCurrentArch = + Architecture . toText <$> + liftIO (readProcess "nix" (toString <$> words "eval --impure --raw --expr builtins.currentSystem") "") + +data RawFlakeOutputs + = RawFlakeOutputs + { packages :: Maybe (Map Architecture (Map PackageName ())) + , nixosModules :: Maybe (Map Text ()) + } deriving (Show, Eq, Generic) + +instance FromJSON RawFlakeOutputs + +nixFlakeShow :: FetcherM m => FlakeUrl -> m RawFlakeOutputs +nixFlakeShow (FlakeUrl fu) = + readJson "nix" ["flake", "show", "--json", fu] + +data RawLicense + = RawLicense + { deprecated :: Bool + , free :: Bool + , fullName :: Text + , shortName :: Text + , spdxId :: Maybe Text + , url :: Maybe Text + } deriving (Show, Eq, Generic) + +instance FromJSON RawLicense where + +newtype RawPlatform = RawPlatform { unRawPlatform :: Text } deriving (Show, Eq) + +instance FromJSON RawPlatform where + parseJSON (A.String x) = pure $ RawPlatform x + parseJSON _ = pure $ RawPlatform "trash" + +data RawPackage + = RawPackage + { broken :: Maybe Bool + , description :: Maybe Text + , insecure :: Maybe Bool + , license :: Maybe (Some RawLicense) + , name :: Maybe Text + , platforms :: Maybe [RawPlatform] + , unfree :: Maybe Bool + , unsupported :: Maybe Bool + } deriving (Show, Eq, Generic) + +instance FromJSON RawPackage + +nixEvalPkgMeta :: FetcherM m => FlakeUrl -> Architecture -> PackageName -> m RawPackage +nixEvalPkgMeta (FlakeUrl fu) (Architecture arch) (PackageName pn) = + readJson "nix" ["eval", "--json", fu <> "#packages." <> arch <> "." <> pn <> ".meta" ] + +rawPackageToPackageInfo :: PackageName -> RawPackage -> PackageInfo +rawPackageToPackageInfo pn rp = + PackageInfo + { description = rp.description + , license = shortName <$> join (maybeToList (fmap someToList rp.license)) + , name = maybe pn PackageName rp.name + , platforms = unRawPlatform <$> fromMaybe [] rp.platforms + , unfree = rp.unfree + , broken = rp.broken + } + +forkWhile :: MonadUnliftIO m => (t -> m a) -> (t -> m b) -> t -> m b +forkWhile f m a = withAsync (f a) (\_ -> m a) + + +metaFlakeFromUrl :: FetcherM m => FlakeUrl -> m MetaFlake +metaFlakeFromUrl fu = do + checkDiskFreeSpace + rfi <- nixFlakeInfo fu + curArch <- asks arch + rfo <- nixFlakeShow fu + let archPkgs = fmap M.keys . M.filterWithKey (\a _ps -> a == curArch) $ fromMaybe mempty rfo.packages + metaPackages <- M.fromList <$> mapM mapArchPkgs (M.toList archPkgs) + pure MetaFlake + { description = rfi.description + , packages = metaPackages + , rev = rfi.revision + , hasNixOsModules = not $ null rfo.nixosModules + , flakeDeps = [] + } + where + getPackageInfo arch pkgName = nixEvalPkgMeta fu arch pkgName <&> rawPackageToPackageInfo pkgName + mapArchPkgs (arch, pkgNames) = + (arch, ) . M.fromList <$> mapM (\pkgName -> (pkgName,) <$> getPackageInfo arch pkgName) + (filter (/= PackageName "default") pkgNames) + +uploadFlakeAndFetch :: (FetcherM m) => + Maybe (FlakeUrl, Either Text MetaFlake) + -> m () +uploadFlakeAndFetch f = do + fctx <- ask + let fetr = FetcherReq fctx.fetcherArgs.fetcherId f fctx.fetcherSecret + rqb = ReqBodyJson fetr + $(logDebug) $ "sending " <> show fetr + jr <- recovering + (fibonacciBackoff 100_000 <> limitRetries 6) + [ \_rs -> Handler $ + \case + he@VanillaHttpException {} -> do + $(logError) $ "Vanila Exception " <> show he + case isStatusCodeException he of + Nothing -> pure False + Just r -> + case responseStatusCode r of + scode -> + pure $ scode < 400 || scode >= 500 + oe -> do + $(logError) $ "Other Exception " <> show oe + pure False + ] + (\_ -> + runReq noRetry $ dynReq POST fctx.fetUrl "fetch-new-flake-submitions" rqb jsonResponse) + $(logDebug) $ "Response from WS for " <> show (fmap fst f) <> ": " <> show (responseBody jr) + case responseBody jr of + Nothing -> uploadFlakeAndFetch Nothing + Just fu -> catchAny (go fctx fu) (onEx fctx fu) + where + onEx fctx fu e = do + $(logError) $ "nix failed for " <> show fu <> " with " <> show e + uploadFlakeAndFetch $ fctx.flakeLoser (Just (fu, Left $ show e)) + go fctx fu = + uploadFlakeAndFetch . fctx.flakeLoser . Just . (fu,) . Right + =<< forkWhile launchHeartbeats metaFlakeFromUrl fu + +noRetry :: HttpConfig +noRetry = + defaultHttpConfig + { httpConfigRetryJudgeException = \_ _ -> False + , httpConfigRetryJudge = \_ _ -> False + } + +runFetcher :: PoF m => FetcherCmdArgs -> FetcherSecret -> m () +runFetcher fa fsec = do + fac <- loadFetcherAutoConf fa.webServiceUrl + let serviceUrl = setResponseTimeout fa.webServiceUrl . coerce . (1 + ) $ untag fac.httpMinTimeout + $(logInfo) $ "Fetcher " <> show fa.fetcherId <> " started for " <> show serviceUrl <> " with " <> show fac + ca <- nixCurrentArch + forever $ do + recoverAll + (capDelay (toMs (6 :: Second)) (fibonacciBackoff (toMs (1 :: Second)))) + (\_ -> + runReaderT (uploadFlakeAndFetch Nothing) + $ FetcherConf serviceUrl ca fsec fa looseF fac) + where + looseF :: forall a. Maybe a -> Maybe a + looseF = if untag fa.looseFlakes + then const Nothing + else id + +launchHeartbeats :: FetcherM m => FlakeUrl -> m () +launchHeartbeats fu = do + $(logInfo) "Fetcher Heartbeat thread is forked" + ctx <- ask + forever $ do + sendHeartbeat fu + threadDelay . unPeriod $ untag ctx.confFromWs.heartbeatPeriod + +sendHeartbeat :: FetcherM m => FlakeUrl -> m () +sendHeartbeat fu = do + ctx <- ask + responseBody <$> runReq noRetry + (dynReq' POST ctx.fetUrl (\u -> u /: "fetcher" /: "heartbeat") (ReqBodyJson $ fhb ctx) jsonResponse) + where + fhb ctx = FetcherHeartbeat + { fetcherId = ctx.fetcherArgs.fetcherId + , workingOn = fu + , fetcherSecret = ctx.fetcherSecret + } + +loadFetcherAutoConf :: PoF m => DynamicUrl -> m FetcherAutoConfig +loadFetcherAutoConf serviceUrl = do + r <- runReq defaultHttpConfig $ dynReq' GET serviceUrl (\ur -> ur /: "fetcher" /: "config") NoReqBody jsonResponse + pure $ responseBody r diff --git a/src/PieceOfFlake/Flake.hs b/src/PieceOfFlake/Flake.hs new file mode 100644 --- /dev/null +++ b/src/PieceOfFlake/Flake.hs @@ -0,0 +1,128 @@ +{-# LANGUAGE DuplicateRecordFields #-} +module PieceOfFlake.Flake where + +import Data.Aeson ( FromJSONKey, ToJSONKey, encode ) +import Data.Map.Strict qualified as M +import Data.SafeCopy ( deriveSafeCopy, base ) +import Data.Text qualified as T +import PieceOfFlake.Prelude hiding (Map) +import Text.Blaze ( ToMarkup ) +import Yesod.Core + ( typeJson, + ToContent(..), + ToTypedContent(..), + TypedContent(TypedContent), PathPiece ) + +newtype RawFlakeUrl = RawFlakeUrl Text + deriving newtype (Show, FromJSON, ToJSON) + +newtype FlakeUrl = FlakeUrl { unFlakeUrl :: Text } + deriving newtype (Show, Read, Eq, Ord, ToMarkup, ToJSON, FromJSON, + Hashable, ToContent, ToTypedContent, IsString, ToText, PathPiece) + +deriveSafeCopy 1 'base ''FlakeUrl + +instance ToContent (Maybe FlakeUrl) where + toContent = toContent . encode +instance ToTypedContent (Maybe FlakeUrl) where + toTypedContent = TypedContent typeJson . toContent + +instance ToContent [FlakeUrl] where + toContent = toContent . encode +instance ToTypedContent [FlakeUrl] where + toTypedContent = TypedContent typeJson . toContent + +repoOfFlakeUrl :: FlakeUrl -> Text +repoOfFlakeUrl (FlakeUrl fu) = T.drop 1 $ T.dropWhile (/= '/') fu + +newtype Architecture = Architecture Text + deriving newtype + ( Show, Eq, Ord, Hashable + , ToJSON, FromJSON, ToJSONKey, FromJSONKey + , IsString, ToText, ToMarkup + ) +deriveSafeCopy 1 'base ''Architecture +newtype PackageName = PackageName Text + deriving newtype + ( Show, Eq, Ord, Hashable + , ToJSON, FromJSON, ToJSONKey, FromJSONKey + , IsString, ToText, ToMarkup + ) +deriveSafeCopy 1 'base ''PackageName +data PackageInfo + = PackageInfo + { description :: Maybe Text + , license :: [ Text ] + , name :: PackageName + , unfree :: Maybe Bool + , platforms :: [ Text ] + , broken :: Maybe Bool + } deriving (Show, Eq, Generic) +deriveSafeCopy 1 'base ''PackageInfo +instance ToJSON PackageInfo +instance FromJSON PackageInfo + +data MetaFlake + = MetaFlake + { description :: Maybe Text + , packages :: M.Map Architecture (M.Map PackageName PackageInfo) + , hasNixOsModules :: Bool + , rev :: Text + , flakeDeps :: [ FlakeUrl ] + } deriving (Show, Eq, Generic) +deriveSafeCopy 1 'base ''MetaFlake + +instance ToJSON MetaFlake +instance FromJSON MetaFlake + +newtype IpAdr = IpAdr Text deriving newtype (Show, Eq, Ord, ToJSON, FromJSON, ToMarkup) +deriveSafeCopy 1 'base ''IpAdr +newtype FetcherId = FetcherId Text deriving newtype (Show, Eq, Ord, ToJSON, FromJSON, ToMarkup, Read, Hashable) +deriveSafeCopy 1 'base ''FetcherId + +data Flake + = SubmittedFlake + { flakeUrl :: FlakeUrl + , submittedAt :: UtcBox + , submittedFrom :: IpAdr + } + | FlakeIsBeingFetched + { flakeUrl :: FlakeUrl + , submitionFetchedAt :: UtcBox + , fetcherId :: FetcherId + } + | BadFlake + { flakeUrl :: FlakeUrl + , fetcherRespondedAt :: UtcBox + , error :: Text + } + | FlakeFetched + { flakeUrl :: FlakeUrl + , uploadedAt :: UtcBox + , meta :: MetaFlake + } + | FlakeIndexed + { flakeUrl :: FlakeUrl + , indexedAt :: UtcBox + , meta :: MetaFlake + } + deriving (Show, Eq, Generic) + +deriveSafeCopy 1 'base ''Flake +instance ToJSON Flake +instance FromJSON Flake + +instance ToContent Flake where + toContent = toContent . encode +instance ToTypedContent Flake where + toTypedContent = TypedContent typeJson . toContent + +isIndexed :: Flake -> Bool +isIndexed FlakeIndexed {} = True +isIndexed _ = False + +linkUrl :: FlakeUrl -> Text +linkUrl (FlakeUrl (T.stripPrefix "github:" -> Just s)) = + "https://github.com/" <> s +linkUrl (FlakeUrl s) = + "#not-gh-link-" <> s diff --git a/src/PieceOfFlake/Flake/Repo.hs b/src/PieceOfFlake/Flake/Repo.hs new file mode 100644 --- /dev/null +++ b/src/PieceOfFlake/Flake/Repo.hs @@ -0,0 +1,398 @@ +module PieceOfFlake.Flake.Repo where + +import Data.Acid ( AcidState ) +import ListT qualified as L +import PieceOfFlake.Acid ( AcidFlakes ) +import PieceOfFlake.Flake + ( FlakeUrl(FlakeUrl), + Flake(indexedAt, FlakeIndexed, submitionFetchedAt, FlakeFetched, + BadFlake, FlakeIsBeingFetched, SubmittedFlake, submittedAt, + submittedFrom, fetcherId, flakeUrl, fetcherRespondedAt), + FetcherId, + IpAdr(..), + MetaFlake, + RawFlakeUrl(..) ) +import PieceOfFlake.CmdArgs + ( WsCmdArgs(allowResubmitIndexedFlakeIn, fetcherHeartbeatPeriod, + badFlakeMaxAge, allowResubmitBadFlakeIn) ) +import PieceOfFlake.Index ( FlakeIndex, indexNewFlake ) +import PieceOfFlake.Prelude hiding (Map, show) +import PieceOfFlake.Prelude qualified as P +import PieceOfFlake.Stats + ( RepoStatsF(totalFlakeUploadsSinceRestart, badFlakes, + fetchingFlakes, fetchedFlakes, meanFetchTime, submittedFlakes, meanTimeInFetchQueue), + addTimeDif ) +import PieceOfFlake.Stm ( newTQueueIO, readTQueue, writeTQueue, TQueue, atomicalog ) +import PieceOfFlake.WebService + ( FetcherHeartbeat(fetcherId), FetcherSecret ) +import StmContainers.Map + ( Map, delete, insert, listTNonAtomic, lookup, newIO ) +import Text.Regex.TDFA ( (=~) ) +import UnliftIO.Exception (bracket_) + +data FetcherState + = FetcherState + { workingOn :: Maybe FlakeUrl + , lastHeartbeatAt :: UtcBox + } + +data FlakeRepo + = FlakeRepo + { flakes :: Map FlakeUrl Flake + , fetcherSecret :: FetcherSecret + , flakeIndex :: FlakeIndex + , wsArgs :: WsCmdArgs + , acidFlakes :: AcidState AcidFlakes + , repoStats :: RepoStatsF TVar + , fetchers :: Map FetcherId FetcherState + , fetcherQueue :: TQueue (Maybe FlakeUrl) + , fetcherQueueLen :: TVar Int + , acidQueue :: TQueue (FlakeUrl, Flake) + , pollingFetchers :: TVar Int + } + +mkFlakeRepo :: MonadIO m => + FetcherSecret -> + WsCmdArgs -> + FlakeIndex -> + Map FlakeUrl Flake -> + AcidState AcidFlakes -> + RepoStatsF TVar -> + m FlakeRepo +mkFlakeRepo fetSec cmdA fi flakesMap acidFlakeStorage rs = do + liftIO $ + FlakeRepo flakesMap fetSec fi cmdA acidFlakeStorage rs <$> + newIO <*> + newTQueueIO <*> + newTVarIO 0 <*> + newTQueueIO <*> + newTVarIO 0 + +trySubmitFlakeToRepo :: PoF m => IpAdr -> FlakeRepo -> FlakeUrl -> m (Either Text Flake) +trySubmitFlakeToRepo ip fr fu = do + atomicalog $ do + lift (lookup fu fr.flakes) >>= \case + Nothing -> submitFlakeToRepo . mkUtcBox =<< getCurrentTime + Just bf@BadFlake {} -> + doAfter bf.fetcherRespondedAt + (\fra -> do + now <- lift $ getTimeAfter fra + if now `diffUTCTime` fra > untag fr.wsArgs.allowResubmitBadFlakeIn then do + $(logInfo) $ "Resubmit bad flake " <> P.show fu + submitFlakeToRepo $ mkUtcBox now + else + pure . Left $ "Flake resubmitted within " <> P.show (untag fr.wsArgs.allowResubmitBadFlakeIn)) + Just fi@FlakeIndexed {} -> + doAfter fi.indexedAt + (\fra -> do + now <- lift $ getTimeAfter fra + if now `diffUTCTime` fra > untag fr.wsArgs.allowResubmitIndexedFlakeIn then do + $(logInfo) $ "Resubmit indexed flake " <> P.show fu + submitFlakeToRepo $ mkUtcBox now + else + pure . Left $ "Indexed flake resubmitted within " <> P.show (untag fr.wsArgs.allowResubmitIndexedFlakeIn)) + Just f -> + pure $ Right f + where + submitFlakeToRepo now = do + ql <- lift (readTVar fr.fetcherQueueLen) + if ql > 1000 + then pure $ Left "Submition Queue is full" + else do + lift $ do + modifyTVar' fr.repoStats.submittedFlakes (1 +) + modifyTVar' fr.repoStats.totalFlakeUploadsSinceRestart (1 +) + modifyTVar' fr.fetcherQueueLen (1 +) + writeTQueue fr.fetcherQueue $ Just fu + fql <- lift $ readTVar fr.fetcherQueueLen + $(logInfo) $ "Fetcher queue increased to " <> P.show fql + let f = SubmittedFlake fu now ip in do + lift $ insert f fu fr.flakes + pure $ Right f + +stmBracket_ :: PoF m => STM a -> STM b -> WriterLoggingT STM c -> m c +stmBracket_ init' recycle mainAction = + bracket_ (atomically init') (atomically recycle) (atomicalog mainAction) + +popFlakeSubmition :: PoF m => FlakeRepo -> FetcherId -> m (Maybe FlakeUrl) +popFlakeSubmition fr ftid = + stmBracket_ + (modifyTVar' fr.pollingFetchers (1 + )) + (modifyTVar' fr.pollingFetchers ((-1) + )) + (popFlakeSubmitionStm ftid fr) >>= mapM + (\(fu, Tagged ifq) -> do + addTimeDif fr.repoStats.meanTimeInFetchQueue ifq + pure fu) + +data TimeInFetchQueue + +popFlakeSubmitionStm :: + FetcherId -> + FlakeRepo -> + WriterLoggingT STM (Maybe (FlakeUrl, Tagged TimeInFetchQueue NominalDiffTime)) +popFlakeSubmitionStm ftid fr = + lift (lookup ftid fr.fetchers) >>= \case + Nothing -> do + $(logDebug) $ "Fetcher " <> P.show ftid <> " asks for flake url from scratch" + fromScratch + Just FetcherState { workingOn = Nothing } -> do + $(logDebug) $ "Fetcher " <> P.show ftid <> " asks for flake url from scratch2" + fromScratch + Just FetcherState { workingOn = Just lostFu } -> do + $(logWarn) $ "Fetcher " <> P.show ftid <> " asked for a next flake url, but " + <> "has not responded about " <> P.show lostFu + fetcherResume lostFu + where + fromScratch = do + fSub <- lift $ readTQueue fr.fetcherQueue + lift $ modifyTVar' fr.fetcherQueueLen (\x -> x - 1) + fql <- lift $ readTVar fr.fetcherQueueLen + $(logInfo) $ "Fetcher queue decreased to " <> P.show fql + case fSub of + Nothing -> pure Nothing + Just fu -> resumeWithFu fu + fetcherResume fu = do + lift (lookup fu fr.flakes) >>= \case + Just (FlakeIsBeingFetched { flakeUrl, submitionFetchedAt }) + | flakeUrl == fu -> do + doAfter submitionFetchedAt (\sa -> do + now <- getTimeAfter sa + pure $ Just (flakeUrl, Tagged $ now `diffUTCTime` sa)) + | otherwise -> do + $(logError) $ "Error flake url mismatch " <> P.show fu <> " <> " <> P.show flakeUrl + popFlakeSubmitionStm ftid fr + Just ufs -> do + $(logError) $ "Expected SumbittedFlake state but:" <> P.show ufs + popFlakeSubmitionStm ftid fr + Nothing -> do + $(logError) $ "Error flake " <> P.show fu <> " is missing in map" + popFlakeSubmitionStm ftid fr + + resumeWithFu fu = do + lift (lookup fu fr.flakes) >>= \case + Just (SubmittedFlake { flakeUrl, submittedAt }) + | flakeUrl == fu -> do + lift $ do + modifyTVar' fr.repoStats.submittedFlakes (flip (-) 1) + modifyTVar' fr.repoStats.fetchingFlakes (1 +) + + doAfter submittedAt (\sa -> do + now <- getTimeAfter sa + assocFlakeWithFetcher now flakeUrl ftid fr.fetchers fr.flakes + lift $ insert (FlakeIsBeingFetched flakeUrl (mkUtcBox now) ftid) fu fr.flakes + pure $ Just (flakeUrl, Tagged $ now `diffUTCTime` sa)) + | otherwise -> do + $(logError) $ "Error flake url mismatch " <> P.show fu <> " <> " <> P.show flakeUrl + popFlakeSubmitionStm ftid fr + Just ufs -> do + $(logError) $ "Expected SumbittedFlake state but:" <> P.show ufs + popFlakeSubmitionStm ftid fr + Nothing -> do + $(logError) $ "Error flake " <> P.show fu <> " is missing in map" + popFlakeSubmitionStm ftid fr + +deassocFlakeFromFetcher :: + (MonadTrans t, Hashable key, MonadLogger (t STM), Show key) => + UTCTime n -> FlakeUrl -> key -> Map key FetcherState -> t STM () +deassocFlakeFromFetcher now fu ftid fetchers = do + lift (lookup ftid fetchers) >>= \case + Nothing -> do + $(logWarn) $ "Fetcher " <> P.show ftid <> " has no state" + lift $ insert efs ftid fetchers + Just FetcherState { workingOn = Nothing } -> + $(logWarn) $ "Fetcher " <> P.show ftid <> " was not associated with any flake" + Just FetcherState { workingOn = Just afu } + | afu == fu -> do + lift $ insert efs ftid fetchers + $(logDebug) $ "Fetcher " <> P.show ftid <> " is diassociated from flake " <> P.show afu + | otherwise -> do + $(logWarn) $ "Fetcher " <> P.show ftid <> " was diassociated with flake " <> P.show afu + <> " but returned " <> P.show fu + lift $ insert efs ftid fetchers + where + efs = FetcherState { workingOn = Nothing, lastHeartbeatAt = mkUtcBox now } + +assocFlakeWithFetcher :: (MonadTrans t, Hashable a, MonadLogger (t STM), Show a) => + UTCTime n -> FlakeUrl -> a -> Map a FetcherState -> Map FlakeUrl Flake -> t STM () +assocFlakeWithFetcher now fu ftid fetchers flakes = do + lift (lookup ftid fetchers) >>= \case + Nothing -> do + $(logDebug) $ "Fetcher " <> P.show ftid <> " got flake " <> P.show fu + lift $ insert FetcherState { workingOn = Just fu, lastHeartbeatAt = mkUtcBox now } ftid fetchers + Just FetcherState { workingOn = Nothing } -> do + $(logDebug) $ "Fetcher " <> P.show ftid <> " got flake' " <> P.show fu + lift $ insert FetcherState { workingOn = Just fu, lastHeartbeatAt = mkUtcBox now } ftid fetchers + Just FetcherState { workingOn = Just lostFu } + | lostFu == fu -> + $(logDebug) $ "Fetcher " <> P.show ftid <> " resumes on " <> P.show fu + | otherwise -> do + lift $ insert FetcherState { workingOn = Just fu, lastHeartbeatAt = mkUtcBox now } ftid fetchers + lift (lookup lostFu flakes) >>= \case + Nothing -> + $(logWarn) $ "Fetcher " <> P.show ftid <> + " lost a flake that is missing in the flake map " <> P.show lostFu + Just fbf@FlakeIsBeingFetched {} + | lostFu == fbf.flakeUrl -> do + $(logWarn) $ "Flake " <> P.show lostFu <> " has been lost on fetcher " <> P.show ftid + lift $ insert (BadFlake lostFu (mkUtcBox now) "Flake has been lost on fetcher. Try resubmit.") + lostFu flakes + | otherwise -> do + $(logError) $ "Lost flake url " <> P.show lostFu <> " mismatch with url in map " <> P.show fbf + Just ue -> + $(logWarn) $ "Flake " <> P.show fu <> " is bound to fetcher " <> + P.show ftid <> " with strange state " <> P.show ue + +data FetcherReq + = FetcherReq + { fetcherId :: FetcherId + , fetcherResponse :: Maybe (FlakeUrl, Either Text MetaFlake) + , fetcherSecret :: FetcherSecret + } deriving (Show, Eq, Generic) + +instance FromJSON FetcherReq +instance ToJSON FetcherReq + +-- | Store meta data for flake and ask for next flake submition +addFetchedFlake :: PoF m => + FlakeRepo -> + FetcherId -> + (FlakeUrl, Either Text MetaFlake) -> + m (Maybe FlakeUrl) +addFetchedFlake fr ftid (fu, fetchedFlake) = do + mapM_ (addTimeDif fr.repoStats.meanFetchTime) =<< atomicalog go + popFlakeSubmition fr ftid + where + go = do + lift (lookup fu fr.flakes) >>= \case + Just (FlakeIsBeingFetched exFu past _fid) + | fu == exFu -> + doAfter past $ \p -> do + now <- lift $ getTimeAfter p + deassocFlakeFromFetcher now fu ftid fr.fetchers + case fetchedFlake of + Left e -> do + lift $ do + modifyTVar' fr.repoStats.fetchingFlakes (flip (-) 1) + modifyTVar' fr.repoStats.badFlakes (1 +) + insert (BadFlake fu (mkUtcBox now) e) fu fr.flakes + $(logInfo) $ "Fetching flake " <> P.show fu <> " failed in " <> P.show (now `diffUTCTime` p) + pure Nothing + Right meta -> do + let f = FlakeFetched fu (mkUtcBox now) meta + lift $ do + modifyTVar' fr.repoStats.fetchingFlakes (flip (-) 1) + modifyTVar' fr.repoStats.fetchedFlakes (1 +) + insert f fu fr.flakes + let fetchTime = now `diffUTCTime` p + $(logInfo) $ "Flake " <> P.show fu <> " is fetched in " <> P.show fetchTime + lift $ writeTQueue fr.acidQueue (fu, f) + indexNewFlake fr.flakeIndex fu + pure $ Just fetchTime + | otherwise -> do + $(logError) $ "Error flake url mismatch " <> P.show fu <> " <> " <> P.show exFu + pure Nothing + Just ufs -> do + $(logError) $ "Expected FlakeIsBeingFetched state but: " <> P.show ufs + pure Nothing + Nothing -> do + $(logError) $ "Error flake " <> P.show fu <> " is missing in map" + pure Nothing + +sendEmptyFlakeSubmition :: PoF m => FlakeRepo -> m () +sendEmptyFlakeSubmition fr = + atomicalog $ do + fql <- lift $ readTVar fr.fetcherQueueLen + when (fql == 0) $ do + numOfBlockedThreads <- lift $ readTVar fr.pollingFetchers + $(logInfo) $ "Send " <> P.show numOfBlockedThreads <> " empty Flake Submition(s)" + replicateM_ numOfBlockedThreads $ do + lift $ do + writeTQueue fr.fetcherQueue Nothing + modifyTVar' fr.fetcherQueueLen (1 +) + +selectBadOldFlakes :: MonadIO m => FlakeRepo -> m [ FlakeUrl ] +selectBadOldFlakes fr = + liftIO (L.foldMaybe filterBad [] (listTNonAtomic fr.flakes)) + where + filterBad selected = \case + (_, BadFlake { flakeUrl }) -> pure . Just $ flakeUrl : selected + _ -> pure Nothing + +removeOldBadFlakes :: PoF m => FlakeRepo -> m () +removeOldBadFlakes fr = do + threadDelay $ untag fr.wsArgs.badFlakeMaxAge `div` 2 + badFlakes <- selectBadOldFlakes fr + forM_ badFlakes $ \fu -> + atomicalog $ do + lift (lookup fu fr.flakes) >>= \case + Just bf@BadFlake {} -> + doAfter bf.fetcherRespondedAt $ \fra -> do + now <- getTimeAfter fra + when (now `diffUTCTime` fra > toNominal (untag fr.wsArgs.badFlakeMaxAge)) $ do + $(logInfo) $ "Delete old bad flake " <> P.show fu + lift $ delete fu fr.flakes + _ -> pure () + +validateRawFlakeUrl :: RawFlakeUrl -> Maybe FlakeUrl +validateRawFlakeUrl (RawFlakeUrl rfu) = + if rfu =~ ("^github:[a-zA-Z0-9._-]+[/][a-zA-Z0-9._-]+$" :: Text) + then pure $ FlakeUrl rfu + else Nothing + +fetcherIsAlive :: PoF m => FlakeRepo -> FetcherHeartbeat -> m () +fetcherIsAlive fr fhb = + atomicalog $ do + now <- getCurrentTime + lift $ do + lookup fhb.fetcherId fr.fetchers >>= \case + Nothing -> + insert + FetcherState { workingOn = Nothing + , lastHeartbeatAt = mkUtcBox now + } + fhb.fetcherId fr.fetchers + Just ftst -> + insert ftst { lastHeartbeatAt = mkUtcBox now } fhb.fetcherId fr.fetchers + +resubmitFlakesFetchingByZombie :: PoF m => FlakeRepo -> m () +resubmitFlakesFetchingByZombie fr = do + fs <- liftIO (L.toList (listTNonAtomic fr.fetchers)) + atomicalog $ do + forM_ fs $ \(ftid, ftst) -> do + case ftst.workingOn of + Nothing -> do + $(logInfo) $ "Delete state of " <> P.show ftid <> " fetcher because of idling" + lift $ delete ftid fr.fetchers + Just wasWorkingOnFu -> do + doAfter ftst.lastHeartbeatAt $ \ha -> do + now <- getTimeAfter ha + let maxPeriod = (*2) . toNominal $ untag fr.wsArgs.fetcherHeartbeatPeriod + when (now `diffUTCTime` ha > maxPeriod) $ do + $(logWarn) $ "Fetcher " <> P.show ftid <> " must be dead since " <> P.show now + lift $ delete ftid fr.fetchers + lift (lookup wasWorkingOnFu fr.flakes) >>= \case + Nothing -> + $(logError) $ "Fetcher " <> P.show ftid <> " was working an ghost flake: " <> P.show wasWorkingOnFu + Just fif@FlakeIsBeingFetched {} + | fif.fetcherId /= ftid -> + $(logError) $ "Failed to resubmit flake " <> P.show wasWorkingOnFu <> + " due it is processed by another fetcher " <> P.show fif.fetcherId + | fif.flakeUrl /= wasWorkingOnFu -> do + $(logWarn) $ "Flake map is corrupted on" <> + P.show wasWorkingOnFu <> " /= " <> P.show fif.flakeUrl + | otherwise -> do + $(logInfo) $ "Resubmit flake " <> P.show wasWorkingOnFu <> + " after fetcher " <> P.show ftid <> " died" + lift $ do + insert + SubmittedFlake + { flakeUrl = wasWorkingOnFu + , submittedAt = mkUtcBox now + , submittedFrom = IpAdr "127.0.0.1" + } + wasWorkingOnFu fr.flakes + modifyTVar' fr.fetcherQueueLen (1 +) + writeTQueue fr.fetcherQueue $ Just wasWorkingOnFu + Just o -> + $(logError) $ "Failed to resubmit flake " <> P.show wasWorkingOnFu <> + " due it is not in IsBeingFetching state but " <> P.show o diff --git a/src/PieceOfFlake/Http.hs b/src/PieceOfFlake/Http.hs new file mode 100644 --- /dev/null +++ b/src/PieceOfFlake/Http.hs @@ -0,0 +1,118 @@ +module PieceOfFlake.Http where + +import Data.ByteString qualified as BS +import Language.Haskell.TH.Syntax (qLocation) +import Network.Wai.Handler.WarpTLS ( runTLS, tlsSettings, TLSSettings ) +import Network.Wai.Handler.Warp + ( Settings, + setBeforeMainLoop, + setMaxTotalHeaderLength, + setOnException, + setPort, + setServerName, + setSlowlorisSize, + setTimeout, + runSettings, + defaultSettings, + defaultShouldDisplayException ) +import Network.Wai +import Network.Wai.Middleware.Autohead +import Network.Wai.Middleware.AcceptOverride +import Network.Wai.Middleware.MethodOverride ( methodOverride ) +import Network.Wai.Middleware.Gzip + ( gzip, defaultGzipSettings, GzipSettings(gzipCheckMime) ) +import Network.Wai.Middleware.RequestLogger + ( OutputFormat(Apache), + defaultRequestLoggerSettings, + mkRequestLogger, + Destination(Logger), + RequestLoggerSettings(outputFormat, destination), + IPAddrSource(FromSocket) ) +import PieceOfFlake.CmdArgs +import PieceOfFlake.Page ( Ypp ) +import PieceOfFlake.Prelude +import Yesod.Core + ( Yesod(makeLogger, messageLoggerSource), YesodDispatch + , toWaiAppYre, makeSessionBackend, getGetMaxExpires + , defaultGen + ) +import Yesod.Core.Types ( Logger, YesodRunnerEnv (..), loggerSet ) + +toWaiApp :: YesodDispatch site => site -> IO Application +toWaiApp site = do + logger <- makeLogger site + toWaiAppLogger logger site + +toWaiAppLogger :: YesodDispatch site => Logger -> site -> IO Application +toWaiAppLogger logger site = do + sb <- makeSessionBackend site + getMaxExpires <- getGetMaxExpires + let yre = YesodRunnerEnv + { yreLogger = logger + , yreSite = site + , yreSessionBackend = sb + , yreGen = defaultGen + , yreGetMaxExpires = getMaxExpires + } + messageLoggerSource + site + logger + $(qLocation >>= liftLoc) + "yesod-core" + LevelInfo + (toLogStr ("Application launched" :: ByteString)) + middleware <- mkDefaultMiddlewares logger + return $ middleware $ toWaiAppYre yre + +mkDefaultMiddlewares :: Logger -> IO Middleware +mkDefaultMiddlewares logger = do + logWare <- mkRequestLogger + defaultRequestLoggerSettings + { destination = Network.Wai.Middleware.RequestLogger.Logger $ loggerSet logger + , outputFormat = Apache FromSocket + } + return $ logWare . defaultMiddlewaresNoLogging + +defaultMiddlewaresNoLogging :: Middleware +defaultMiddlewaresNoLogging = acceptOverride . autohead . gzip gzipSettings . methodOverride + where + gzipSettings = defaultGzipSettings { gzipCheckMime = check } + check = BS.isPrefixOf "text/html" + +mkSettings :: Ypp -> WsCmdArgs -> Logger -> Settings +mkSettings yp ca logger = + setPort port $ + setServerName "PieceOfFlake" $ + setOnException onEx $ + setSlowlorisSize 1024 $ + setMaxTotalHeaderLength 1024 $ + setBeforeMainLoop (putTextLn $ "Go " <> untag ca.baseUrl) $ + -- expected that Nginx restricts keepalive for non fetcher connections + setTimeout (2 * fromIntegral (untag ca.noSubmitionHeartbeat)) + defaultSettings + + where + port = untag ca.httpPortToListen + shouldLog' = defaultShouldDisplayException + onEx _ e = + when (shouldLog' e) $ + messageLoggerSource + yp + logger + $(qLocation >>= liftLoc) + "yesod-core" + LevelError + (toLogStr $ "Exception from Warp: " ++ show e) + +mkTlsSettings :: Tagged Cert FilePath -> Tagged CertKey FilePath -> TLSSettings +mkTlsSettings cert key = tlsSettings (untag cert) (untag key) + +runPlain :: Settings -> Application -> IO () +runPlain = runSettings + +runWebService :: MonadIO m => WsCmdArgs -> Ypp -> m () +runWebService ws y = liftIO $ do + logger <- makeLogger y + case liftA2 mkTlsSettings ws.certFile ws.keyFile of + Nothing -> runPlain (mkSettings y ws logger) =<< toWaiApp y + Just tlsSngs -> runTLS tlsSngs (mkSettings y ws logger) =<< toWaiApp y diff --git a/src/PieceOfFlake/Index.hs b/src/PieceOfFlake/Index.hs new file mode 100644 --- /dev/null +++ b/src/PieceOfFlake/Index.hs @@ -0,0 +1,223 @@ +module PieceOfFlake.Index where + +import Data.HashPSQ qualified as PSQ +import Data.Map.Strict qualified as M +import Data.LruCache qualified as LRU +import Data.LruCache.Internal qualified as LRU +import Data.LruCache (LruCache) +import Data.SearchEngine + ( Term, + SearchConfig(documentFeatureValue, SearchConfig, documentKey, + extractDocumentTerms, transformQueryTerm), + SearchRankParameters(paramAutosuggestPostfilterLimit, + SearchRankParameters, paramK1, paramB, paramFieldWeights, + paramFeatureWeights, paramFeatureFunctions, + paramResultsetSoftLimit, paramResultsetHardLimit, + paramAutosuggestPrefilterLimit), + SearchEngine, + FeatureFunction(LogarithmicFunction), + query, + initSearchEngine, + insertDoc, + queryAutosuggest, + ResultsFilter(NoFilter) ) +import ListT qualified as L +import NLP.Tokenize.Text ( tokenize ) +import PieceOfFlake.Flake + ( FlakeUrl, + repoOfFlakeUrl, + PackageInfo(broken, description, license, name, unfree), + MetaFlake(hasNixOsModules, description, packages), + Flake(meta, uploadedAt, FlakeIndexed, FlakeFetched, flakeUrl), + isIndexed ) +import PieceOfFlake.CmdArgs ( IndexQueryCacheSize ) +import PieceOfFlake.Prelude hiding (pi, Map) +import PieceOfFlake.Stm + ( readTQueue, writeTQueue, TQueue, newTQueueIO, atomicalog ) +import StmContainers.Map ( insert, listTNonAtomic, lookup, Map ) +import PieceOfFlake.Stats + ( RepoStatsF(meanTimeInIndexQueue, fetchedFlakes, indexedFlakes, + meanIndexTime, meanSearchTime), + addTimeDif ) +import PieceOfFlake.TotalMath ( notZero, divNz, realToFracNz ) + +type FlakeSearchEngine = SearchEngine (FlakeUrl, MetaFlake) FlakeUrl () () + +data FlakeIndex + = FlakeIndex + { searchEngine :: TVar FlakeSearchEngine + , indexerQueue :: TQueue FlakeUrl + , indexerQueueLen :: TVar Int + , queryCache :: TVar (LruCache Text UtcBox) + , searchRequestCounter :: TVar Integer + } + +mkFlakeIndex :: MonadIO m => Tagged IndexQueryCacheSize Word -> m FlakeIndex +mkFlakeIndex (Tagged cs) = do + liftIO $ + FlakeIndex <$> + newTVarIO emptySearchEngine <*> + newTQueueIO <*> + newTVarIO 0 <*> + newTVarIO (LRU.empty $ fromIntegral cs) <*> + newTVarIO 0 + +packageInfoToTerms :: PackageInfo -> [Term] +packageInfoToTerms pi = + concatMap tokenize pi.description <> + concatMap tokenize pi.license <> + [ toText pi.name ] + <> maybeToList (fmap (\c -> if c then "unfree" else "free") pi.unfree) + <> maybeToList (fmap (\c -> if c then "broken" else "unbroken") pi.broken) + +extractTerms :: (FlakeUrl, MetaFlake) -> () -> [Term] +extractTerms (fu, mf) () = + concatMap tokenize (maybeToList mf.description) + <> [repoOfFlakeUrl fu] <> (toText <$> M.keys mf.packages) + <> (concatMap packageInfoToTerms . concatMap M.elems $ M.elems mf.packages) + <> memptyIfFalse mf.hasNixOsModules ["nixosModules"] + +emptySearchEngine :: FlakeSearchEngine +emptySearchEngine = + initSearchEngine + SearchConfig + { documentKey = fst + , extractDocumentTerms = extractTerms + , transformQueryTerm = const + , documentFeatureValue = \_doc () -> 1 + } + SearchRankParameters + { paramK1 = 2 -- weight of repetition + , paramB = const 0.4 -- normalize by length + , paramFieldWeights = const 1 + , paramFeatureWeights = const 1 + , paramFeatureFunctions = const $ LogarithmicFunction 2 + , paramResultsetSoftLimit = 10 + , paramResultsetHardLimit = 30 + , paramAutosuggestPrefilterLimit = 10 + , paramAutosuggestPostfilterLimit = 10 + } + +data FlakeIndexedSuc + = FlakeIndexedSuc + { inIndexQueue :: NominalDiffTime + , indexingTook :: NominalDiffTime + } + +nothing :: Monad m => m a1 -> m (Maybe a2) +nothing m = do + void m + pure Nothing + +consumeIndexQueue :: PoF m => RepoStatsF TVar -> Map FlakeUrl Flake -> FlakeIndex -> m () +consumeIndexQueue rs fs fi = do + r <- atomicalog $ do + $(logInfo) "Wait for flakes to index for full text search" + fu <- lift $ readTQueue fi.indexerQueue + lift $ modifyTVar' fi.indexerQueueLen (\x -> x - 1) + ql <- lift $ readTVar fi.indexerQueueLen + $(logInfo) $ "Start index flake " <> show fu <> "; index queue " <> show ql + lift (lookup fu fs) >>= \case + Nothing -> do + $(logError) $ "Flake " <> show fu <> " is missing" + pure Nothing + Just f -> indexFlake rs fi fs f + mapM_ (\FlakeIndexedSuc { inIndexQueue, indexingTook } -> do + addTimeDif rs.meanIndexTime indexingTook + addTimeDif rs.meanTimeInIndexQueue inIndexQueue + ) r + +indexNewFlake :: (MonadLogger (t STM), MonadTrans t) => FlakeIndex -> FlakeUrl -> t STM () +indexNewFlake fi fu = do + iql <- lift $ do + writeTQueue fi.indexerQueue fu + modifyTVar' fi.indexerQueueLen (1 +) + readTVar fi.indexerQueueLen + $(logInfo) $ "Indexer queue increased to " <> show iql + +indexFlake :: (MonadLogger (t STM), MonadTrans t) => + RepoStatsF TVar -> FlakeIndex -> Map FlakeUrl Flake -> Flake -> t STM (Maybe FlakeIndexedSuc) +indexFlake rs = + indexFlake' $ do + lift $ do + modifyTVar' rs.fetchedFlakes (flip (-) 1) + modifyTVar' rs.indexedFlakes (1 +) + +indexFlake' :: (MonadLogger (t STM), MonadTrans t) => + t STM () -> FlakeIndex -> Map FlakeUrl Flake -> Flake -> t STM (Maybe FlakeIndexedSuc) +indexFlake' onIndexed fi fs f = + case f of + ff@FlakeFetched { flakeUrl, uploadedAt } -> + doAfter uploadedAt $ \ua -> do + beforeIndex <- lift $ getTimeAfter ua + let fu = flakeUrl + ixf = FlakeIndexed fu (mkUtcBox beforeIndex) ff.meta + onIndexed + lift $ do + insert ixf fu fs + modifyTVar' fi.searchEngine (insertDoc (fu, ff.meta)) + afterIndex <- lift $ getTimeAfter beforeIndex + $(logInfo) $ "Finished index flake " <> show fu + pure . Just $ FlakeIndexedSuc + { inIndexQueue = beforeIndex `diffUTCTime` ua + , indexingTook = afterIndex `diffUTCTime` beforeIndex + } + _nff -> do + $(logError) $ "Flake " <> show f.flakeUrl <> " is not in the fetched state" + pure Nothing + +loadIndexFromScratch :: + PoF m => RepoStatsF TVar -> FlakeIndex -> Map FlakeUrl Flake -> [(FlakeUrl, Flake)] -> m () +loadIndexFromScratch rs fi fsm fs = do + started <- getCurrentTime + totalIndexed :: Int <- atomicalog $ do + $(logInfo) "Started init full text search index population" + r <- foldlM go 0 fs + $(logInfo) "Ended init full text search index population" + pure r + dur <- flip diffUTCTime started <$> getTimeAfter started + forM_ (realToFracNz <$> notZero totalIndexed) $ \ti -> + addTimeDif rs.meanIndexTime $ dur `divNz` ti + where + go (i :: Int) = \case + (_, ff@FlakeFetched {}) -> + (i + ) . maybe 0 (const 1) <$> indexFlake' (lift $ modifyTVar' rs.indexedFlakes (1 +)) fi fsm ff + (fu, nff) -> do + lift $ insert nff fu fsm + pure i + +data FlakeSearchReq + = FlakeSearchReq + { searchPattern :: [Text] + , skipBroken :: Bool + } deriving (Show, Eq, Generic) + +instance FromJSON FlakeSearchReq + +listQueryCache :: PoF m => FlakeIndex -> m [ Text ] +listQueryCache fi = + reverse . fmap (^._1) . sortWith (^._2) . PSQ.toList . LRU.lruQueue <$> readTVarIO fi.queryCache + +findFlakes :: PoF m => + RepoStatsF TVar -> Map FlakeUrl Flake -> FlakeIndex -> FlakeSearchReq -> m [ FlakeUrl ] +findFlakes rs fs fi FlakeSearchReq { searchPattern = ps } = + case concatMap tokenize ps of + [] -> justLoadFirstNFlakes fs 30 + pst@(t1:_) -> do + now <- getCurrentTime + atomicalog (fromIdx now t1 pst) <* + (addTimeDif rs.meanSearchTime . (`diffUTCTime` now) =<< getTimeAfter now) + where + fromIdx now t1 pst = do + se <- lift $ do + modifyTVar' fi.searchRequestCounter (1 +) + modifyTVar' fi.queryCache (LRU.insert (unwords ps) (mkUtcBox now)) + readTVar fi.searchEngine + $(logInfo) $ "Search flakes by " <> show pst + let r = query se pst `alt` (fmap fst . snd $ queryAutosuggest se NoFilter [] t1) + $(logInfo) $ "Found " <> show (length r) <> " by " <> show ps + pure r + +justLoadFirstNFlakes :: MonadIO m => Map FlakeUrl Flake -> Int -> m [ FlakeUrl ] +justLoadFirstNFlakes fs n = + fmap fst . filter (isIndexed . snd) <$> liftIO (L.toList $ L.take (2 * n) (listTNonAtomic fs)) diff --git a/src/PieceOfFlake/Page.hs b/src/PieceOfFlake/Page.hs new file mode 100644 --- /dev/null +++ b/src/PieceOfFlake/Page.hs @@ -0,0 +1,654 @@ +{-# LANGUAGE QuasiQuotes #-} +{-# LANGUAGE MultilineStrings #-} +module PieceOfFlake.Page where + +import Data.Aeson ( encode ) +import Data.Map.Strict (elems) +import PieceOfFlake.CmdArgs + ( WsCmdArgs(logLevel, fetcherHeartbeatPeriod, noSubmitionHeartbeat, + baseUrl), + StaticCacheSeconds ) +import PieceOfFlake.Flake + ( FlakeUrl, + Flake(..), + linkUrl, + IpAdr(IpAdr), + MetaFlake(description, packages, rev), + PackageInfo(broken, name, description, license, unfree) ) +import PieceOfFlake.Flake.Repo + ( FetcherReq(FetcherReq), + FlakeRepo(wsArgs, fetcherSecret, flakes, fetcherQueueLen, + flakeIndex, repoStats), + trySubmitFlakeToRepo, + popFlakeSubmition, + addFetchedFlake, + fetcherIsAlive, + validateRawFlakeUrl ) +import PieceOfFlake.Index ( findFlakes, listQueryCache, FlakeIndex (searchRequestCounter, indexerQueueLen) ) +import PieceOfFlake.Prelude hiding (Map, error, pi, Handler) +import PieceOfFlake.Stats ( greadTraVar, renderRepoStats ) +import PieceOfFlake.Th ( includeFile ) +import PieceOfFlake.WebService + ( FetcherAutoConfig(..), + FetcherHeartbeat(fetcherSecret), + FetcherSecret, + Period(Period) ) +import PieceOfFlake.Yesod + ( bulmaLayout, + clientAdrToDec4, + getClientAdr, + mp3Mime, + sendStaticBs, + staticFile, + ContentEncoding(Br, Gzip), + Mime(Mime), + Ts(Ts), + Unit(..) ) + +import StmContainers.Map ( lookup ) +import Text.Blaze.Internal ( MarkupM ) +import Yesod.Core + + +data Ypp + = Ypp + { repo :: FlakeRepo + , staticCache :: Tagged StaticCacheSeconds Word32 + } + +mkYesod "Ypp" [parseRoutes| +/app.js AppJsR GET +/style.css StyleR GET +/bulma.min.css BulmaR GET +/robots.txt RobotsR GET +/sitemap.xml SiteMapR GET +/ HomeR GET +/search SearchR GET +/stats StatsR GET +/about AboutR GET +/fetcher/heartbeat FetcherHeartbeatR POST +/fetcher/config FetcherAutoConfigR GET +/flake/#FlakeUrl FlakeR GET +/flake/status/#FlakeUrl FlakeStatusR GET +/publication PublicationR GET +/favicon.svg FaviconR GET +/flake.svg FlakeSvgR GET +/github.svg GitHubR GET +/flush.mp3 FlushSoundR GET +/snow.mp3 SnowSoundR GET +/avalanche.mp3 AvalancheSoundR GET +/submit-flake SubmitFlakeR POST +/fetch-new-flake-submitions FetchNewFlakeSubmitionsR POST +/find-flakes FindFlakesR POST +|] + +instance Yesod Ypp where + approot = ApprootMaster $ untag . baseUrl . wsArgs . repo + makeSessionBackend _ = pure Nothing + maximumContentLength _ = pure . \case + Nothing -> 1 + Just HomeR -> 1 + Just SubmitFlakeR -> 200 + Just FetchNewFlakeSubmitionsR -> 100000 + Just _ -> 500 + + shouldLogIO (Ypp {repo}) _ l = + pure $ l >= repo.wsArgs.logLevel + + +getAppJsR, getFaviconR, getRobotsR, getGitHubR, getFlushSoundR :: Handler TypedContent +getSnowSoundR, getSiteMapR, getAvalancheSoundR, getStyleR, getBulmaR, getFlakeSvgR :: Handler TypedContent +getFlakeSvgR = + staticFile (Mime typeSvg) $(includeFile "assets/flake.svg") $ fromList + [ (Gzip, $(includeFile "assets/flake.svg.gz")) + , (Br, $(includeFile "assets/flake.svg.br")) + ] +getFaviconR = + staticFile (Mime typeSvg) $(includeFile "assets/favicon.svg") $ fromList + [ (Gzip, $(includeFile "assets/favicon.svg.gz")) + , (Br, $(includeFile "assets/favicon.svg.br")) + ] +getGitHubR = + staticFile (Mime typeSvg) $(includeFile "assets/github.svg") $ fromList + [ (Gzip, $(includeFile "assets/github.svg.gz")) + , (Br, $(includeFile "assets/github.svg.br")) + ] +getAppJsR = + staticFile (Mime typeJavascript) $(includeFile "assets/app.js") $ fromList + [ (Gzip, $(includeFile "assets/app.js.gz")) + , (Br, $(includeFile "assets/app.js.br")) + ] +getStyleR = + staticFile (Mime typeCss) $(includeFile "assets/style.css") $ fromList + [ (Gzip, $(includeFile "assets/style.css.gz")) + , (Br, $(includeFile "assets/style.css.br")) + ] +getBulmaR = + staticFile (Mime typeCss) $(includeFile "assets/bulma.min.css") $ fromList + [ (Gzip, $(includeFile "assets/bulma.min.css.gz")) + , (Br, $(includeFile "assets/bulma.min.css.br")) + ] +getFlushSoundR = sendStaticBs mp3Mime $(includeFile "assets/flush.mp3") +getSnowSoundR = sendStaticBs mp3Mime $(includeFile "assets/snow.mp3") +getAvalancheSoundR = sendStaticBs mp3Mime $(includeFile "assets/avalanche.mp3") +getSiteMapR = sendStaticBs (Mime typeXml) $(includeFile "assets/sitemap.xml") +getRobotsR = sendStaticBs (Mime typePlain) $(includeFile "assets/robots.txt") + +instance ClockMonad (HandlerFor Ypp) where + getCurrentTime = liftIO getCurrentTime + getTimeAfter x = liftIO $ getTimeAfter x + +getStatsR :: Handler Html +getStatsR = do + Ypp { repo } <- getYesod + queries <- listQueryCache repo.flakeIndex + fetchQueueLen <- Tagged @"fetch" <$> readTVarIO repo.fetcherQueueLen + idxQueueLen <- readTVarIO repo.flakeIndex.indexerQueueLen + searchReq <- readTVarIO repo.flakeIndex.searchRequestCounter + rs <- greadTraVar repo.repoStats + bulmaLayout $ do + setTitle "Stats" + metaTags + navBar + [whamlet| +
+ ^{renderRepoStats searchReq idxQueueLen fetchQueueLen rs} +

+ Popular Queries + $if null queries +
+ No queries + $else +
+