changed invidious to local package

modules/invidious.nix

@@ -2,17 +2,17 @@
 let
   iv_port = "3001";
   domain = "yt.mc-fucker.cool";
+  db_user = "kemal";
+  db = "invidious";
 in
 {
   services.invidious = {
-    #package = pkgs.callPackage /root/invidious {
-    #  lsquic = pkgs.callPackage /root/invidious/lsquic.nix {};
-    #};
+    package = pkgs.callPackage /etc/nixos/packages/invidious { lsquic = pkgs.callPackage /etc/nixos/packages/invidious/lsquic.nix {}; videojs = pkgs.callPackage /etc/nixos/packages/invidious/videojs.nix {}; };
     enable = true;
     port = (lib.strings.toInt iv_port);
     database = {
       createLocally = false;
-      host = "postgres";
+      host = "127.0.0.1";
       passwordFile = "/etc/nixos/keys/invidious-dbpassword";
     };
     settings = {
@@ -23,12 +23,24 @@ in
         };
     };
   };
-  networking.firewall.allowedTCPPorts = [ config.services.invidious.port ];
   systemd.services.invidious.serviceConfig = {
     Restart = "always";
     RestartSec = "2s";
   };
 
+  services.postgresql = {
+    ensureDatabases = [ db ];
+    ensureUsers = [
+      {
+        name = db_user;
+        ensurePermissions = {
+          "DATABASE ${db}" = "ALL PRIVILEGES";
+        };
+      }
+    ];
+    authentication = "host invidious kemal 127.0.0.1/32 trust";
+  };
+
   services.nginx.virtualHosts."${domain}" = {
     forceSSL = true;
     enableACME = true;

packages/invidious/default.nix

@@ -0,0 +1,113 @@
+{ lib, stdenv, crystal, fetchFromGitHub, librsvg, pkg-config, libxml2, openssl, shards, sqlite, lsquic, videojs, nixosTests }:
+let
+  # All versions, revisions, and checksums are stored in ./versions.json.
+  # The update process is the following:
+  #   * pick the latest commit
+  #   * update .invidious.rev, .invidious.version, and .invidious.sha256
+  #   * prefetch the videojs dependencies with scripts/fetch-player-dependencies.cr
+  #     and update .videojs.sha256 (they are normally fetched during build
+  #     but nix's sandboxing does not allow that)
+  #   * if shard.lock changed
+  #     * recreate shards.nix by running crystal2nix
+  #     * update lsquic and boringssl if necessarry, lsquic.cr depends on
+  #       the same version of lsquic and lsquic requires the boringssl
+  #       commit mentioned in its README
+  versions = builtins.fromJSON (builtins.readFile ./versions.json);
+in
+crystal.buildCrystalPackage rec {
+  pname = "invidious";
+  inherit (versions.invidious) version;
+
+  src = fetchFromGitHub {
+    owner = "iv-org";
+    repo = pname;
+    fetchSubmodules = true;
+    inherit (versions.invidious) rev sha256;
+  };
+
+  postPatch =
+    let
+      # Replacing by the value (templates) of the variables ensures that building
+      # fails if upstream changes the way the metadata is formatted.
+      branchTemplate = ''{{ "#{`git branch | sed -n '/* /s///p'`.strip}" }}'';
+      commitTemplate = ''{{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }}'';
+      versionTemplate = ''{{ "#{`git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g`.strip}" }}'';
+      # This always uses the latest commit which invalidates the cache even if
+      # the assets were not changed
+      assetCommitTemplate = ''{{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit -- assets`.strip}" }}'';
+    in
+    ''
+      for d in ${videojs}/*; do ln -s "$d" assets/videojs; done
+
+      # Use the version metadata from the derivation instead of using git at
+      # build-time
+      substituteInPlace src/invidious.cr \
+          --replace ${lib.escapeShellArg branchTemplate} '"master"' \
+          --replace ${lib.escapeShellArg commitTemplate} '"${lib.substring 0 7 versions.invidious.rev}"' \
+          --replace ${lib.escapeShellArg versionTemplate} '"${lib.replaceChars ["-"] ["."] (lib.substring 9 10 version)}"' \
+          --replace ${lib.escapeShellArg assetCommitTemplate} '"${lib.substring 0 7 versions.invidious.rev}"'
+
+      # Patch the assets and locales paths to be absolute
+      substituteInPlace src/invidious.cr \
+          --replace 'public_folder "assets"' 'public_folder "${placeholder "out"}/share/invidious/assets"'
+      substituteInPlace src/invidious/helpers/i18n.cr \
+          --replace 'File.read("locales/' 'File.read("${placeholder "out"}/share/invidious/locales/'
+
+      # Reference sql initialisation/migration scripts by absolute path
+      substituteInPlace src/invidious/database/base.cr \
+            --replace 'config/sql' '${placeholder "out"}/share/invidious/config/sql'
+
+      substituteInPlace src/invidious/user/captcha.cr \
+          --replace 'Process.run(%(rsvg-convert' 'Process.run(%(${lib.getBin librsvg}/bin/rsvg-convert'
+    '';
+
+  nativeBuildInputs = [ pkg-config shards ];
+  buildInputs = [ libxml2 openssl sqlite ];
+
+  format = "crystal";
+  shardsFile = ./shards.nix;
+  crystalBinaries.invidious = {
+    src = "src/invidious.cr";
+    options = [ "--release" "--progress" "--verbose" "--no-debug" "-Dskip_videojs_download" ];
+  };
+
+  postConfigure = ''
+    # lib includes nix store paths which can’t be patched, so the links have to
+    # be dereferenced first.
+    cp -rL lib lib2
+    rm -r lib
+    mv lib2 lib
+    chmod +w -R lib
+    cp ${lsquic}/lib/liblsquic.a lib/lsquic/src/lsquic/ext
+  '';
+
+  postInstall = ''
+    mkdir -p $out/share/invidious/config
+
+    # Copy static parts
+    cp -r assets locales $out/share/invidious
+    cp -r config/sql $out/share/invidious/config
+  '';
+
+  # Invidious tries to open config/config.yml and connect to the database, even
+  # when running --help. This specifies a minimal configuration in an
+  # environment variable. Even though the database is bogus, --help still
+  # works.
+  installCheckPhase = ''
+    INVIDIOUS_CONFIG="database_url: sqlite3:///dev/null" $out/bin/invidious --help
+  '';
+
+  passthru = {
+    inherit lsquic;
+    tests = { inherit (nixosTests) invidious; };
+    updateScript = ./update.sh;
+  };
+
+  meta = with lib; {
+    description = "An open source alternative front-end to YouTube";
+    homepage = "https://invidious.io/";
+    license = licenses.agpl3;
+    maintainers = with maintainers; [ infinisil sbruder ];
+    broken = stdenv.isDarwin && stdenv.isAarch64;
+  };
+}

packages/invidious/lsquic.nix

@@ -0,0 +1,111 @@
+{ lib, boringssl, stdenv, fetchgit, fetchFromGitHub, fetchurl, cmake, zlib, perl, libevent }:
+let
+  versions = builtins.fromJSON (builtins.readFile ./versions.json);
+
+  fetchGitilesPatch = { name, url, sha256 }:
+    fetchurl {
+      url = "${url}%5E%21?format=TEXT";
+      inherit name sha256;
+      downloadToTemp = true;
+      postFetch = ''
+        base64 -d < $downloadedFile > $out
+      '';
+    };
+
+  # lsquic requires a specific boringssl version (noted in its README)
+  boringssl' = boringssl.overrideAttrs ({ preBuild, ... }: {
+    version = versions.boringssl.rev;
+    src = fetchgit {
+      url = "https://boringssl.googlesource.com/boringssl";
+      inherit (versions.boringssl) rev sha256;
+    };
+
+    patches = [
+      # Use /etc/ssl/certs/ca-certificates.crt instead of /etc/ssl/cert.pem
+      ./use-etc-ssl-certs.patch
+
+      # because lsquic requires that specific boringssl version and that
+      # version does not yet include fixes for gcc11 build errors, they
+      # must be backported
+      (fetchGitilesPatch {
+        name = "fix-mismatch-between-header-and-implementation-of-bn_sqr_comba8.patch";
+        url = "https://boringssl.googlesource.com/boringssl/+/139adff9b27eaf0bdaac664ec4c9a7db2fe3f920";
+        sha256 = "05sp602dvh50v46jkzmh4sf4wqnq5bwy553596g2rhxg75bailjj";
+      })
+      (fetchGitilesPatch {
+        name = "use-an-unsized-helper-for-truncated-SHA-512-variants.patch";
+        url = "https://boringssl.googlesource.com/boringssl/+/a24ab549e6ae246b391155d7bed3790ac0e07de2";
+        sha256 = "0483jkpg4g64v23ln2blb74xnmzdjcn3r7w4zk7nfg8j3q5f9lxm";
+      })
+/*
+      # the following patch is too complex, so we will modify the build flags
+      # of crypto/fipsmodule/CMakeFiles/fipsmodule.dir/bcm.c.o in preBuild
+      # and turn off -Werror=stringop-overflow
+      (fetchGitilesPatch {
+        name = "make-md32_common.h-single-included-and-use-an-unsized-helper-for-SHA-256.patch";
+        url = "https://boringssl.googlesource.com/boringssl/+/597ffef971dd980b7de5e97a0c9b7ca26eec94bc";
+        sha256 = "1y0bkkdf1ccd6crx326agp01q22clm4ai4p982y7r6dkmxmh52qr";
+      })
+*/
+      (fetchGitilesPatch {
+        name = "fix-array-parameter-warnings.patch";
+        url = "https://boringssl.googlesource.com/boringssl/+/92c6fbfc4c44dc8462d260d836020d2b793e7804";
+        sha256 = "0h4sl95i8b0dj0na4ngf50wg54raxyjxl1zzwdc810abglp10vnv";
+      })
+    ];
+
+    preBuild = ''
+      ${preBuild}
+      sed -e '/^build crypto\/fipsmodule\/CMakeFiles\/fipsmodule\.dir\/bcm\.c\.o:/,/^ *FLAGS =/ s/^ *FLAGS = -Werror/& -Wno-error=stringop-overflow/' \
+          -i build.ninja
+    '';
+  });
+in
+stdenv.mkDerivation rec {
+  pname = "lsquic";
+  version = versions.lsquic.version;
+
+  src = fetchFromGitHub {
+    owner = "litespeedtech";
+    repo = pname;
+    rev = "v${version}";
+    inherit (versions.lsquic) sha256;
+    fetchSubmodules = true;
+  };
+
+  nativeBuildInputs = [ cmake perl ];
+  buildInputs = [ boringssl' libevent zlib ];
+
+  cmakeFlags = [
+    "-DBORINGSSL_DIR=${lib.getDev boringssl'}"
+    "-DBORINGSSL_LIB_crypto=${lib.getLib boringssl'}/lib/libcrypto.a"
+    "-DBORINGSSL_LIB_ssl=${lib.getLib boringssl'}/lib/libssl.a"
+    "-DZLIB_LIB=${zlib}/lib/libz.so"
+  ];
+
+  # adapted from lsquic.cr’s Dockerfile
+  # (https://github.com/iv-org/lsquic.cr/blob/master/docker/Dockerfile)
+  installPhase = ''
+    runHook preInstall
+
+    mkdir combinedlib
+    cd combinedlib
+    ar -x ${lib.getLib boringssl'}/lib/libssl.a
+    ar -x ${lib.getLib boringssl'}/lib/libcrypto.a
+    ar -x ../src/liblsquic/liblsquic.a
+    ar rc liblsquic.a *.o
+    ranlib liblsquic.a
+    install -D liblsquic.a $out/lib/liblsquic.a
+
+    runHook postInstall
+  '';
+
+  passthru.boringssl = boringssl';
+
+  meta = with lib; {
+    description = "A library for QUIC and HTTP/3 (version for Invidious)";
+    homepage = "https://github.com/litespeedtech/lsquic";
+    maintainers = with maintainers; [ infinisil sbruder ];
+    license = with licenses; [ openssl isc mit bsd3 ]; # statically links against boringssl, so has to include its licenses
+  };
+}

packages/invidious/shards.nix

@@ -0,0 +1,80 @@
+{
+  athena-negotiation = {
+    owner = "athena-framework";
+    repo = "negotiation";
+    rev = "v0.1.1";
+    sha256 = "1vkk59lqrxb0l8kyzs114i3c18zb2bdiah2xhazkk8q7x6fz4yzk";
+  };
+  backtracer = {
+    owner = "sija";
+    repo = "backtracer.cr";
+    rev = "v1.2.1";
+    sha256 = "02r1l7rn2wsljkx495s5s7j04zgn73m2kx0hkzs7620camvlwbqq";
+  };
+  db = {
+    owner = "crystal-lang";
+    repo = "crystal-db";
+    rev = "v0.10.1";
+    sha256 = "03c5h14z6h2mxnx949lihnyqjd19hcj38iasdwq9fp95h8cld376";
+  };
+  exception_page = {
+    owner = "crystal-loot";
+    repo = "exception_page";
+    rev = "v0.2.2";
+    sha256 = "1c8askb9b7621jjz5pjj6b8pdbhw3r1l3dym6swg1saspf5j3jwi";
+  };
+  kemal = {
+    owner = "kemalcr";
+    repo = "kemal";
+    rev = "v1.1.2";
+    sha256 = "1149q4qw0zrws5asqqr4snrdi67xsmisdcq58zcrbgqgsxgly9d0";
+  };
+  kilt = {
+    owner = "jeromegn";
+    repo = "kilt";
+    rev = "v0.6.1";
+    sha256 = "0dpc15y9m8c5l9zdfif6jlf7zmkrlm9w4m2igi5xa22fdjwamwfp";
+  };
+  lsquic = {
+    owner = "iv-org";
+    repo = "lsquic.cr";
+    rev = "v2.18.1-2";
+    sha256 = "0bljk0pwbjb813dfwrhgi00w2ai09k868xvak4hfzdkbmpc7id6y";
+  };
+  pg = {
+    owner = "will";
+    repo = "crystal-pg";
+    rev = "v0.24.0";
+    sha256 = "07i5bqkv5j6y6f8v5cpqdxc5wzzrvgv3ds24znv4mzv6nc84csn4";
+  };
+  protodec = {
+    owner = "iv-org";
+    repo = "protodec";
+    rev = "v0.1.4";
+    sha256 = "15azh9izxqgwpgkpicmivfdz31wkibnwy09rwhxsg0lyc4wf8xj9";
+  };
+  radix = {
+    owner = "luislavena";
+    repo = "radix";
+    rev = "v0.4.1";
+    sha256 = "1l08cydkdidq9yyil1wl240hvk41iycv04jrg6nx5mkvzw4z1bzg";
+  };
+  spectator = {
+    owner = "icy-arctic-fox";
+    repo = "spectator";
+    rev = "v0.10.4";
+    sha256 = "0rcxq2nbslvwrd8m9ajw6dzaw3hagxmkdy9s8p34cgnr4c9dijdq";
+  };
+  sqlite3 = {
+    owner = "crystal-lang";
+    repo = "crystal-sqlite3";
+    rev = "v0.18.0";
+    sha256 = "03nnvpchhq9f9ywsm3pk2rrj4a3figw7xs96zdziwgr5znkz6x93";
+  };
+  ameba = {
+    owner = "crystal-ameba";
+    repo = "ameba";
+    rev = "v0.14.3";
+    sha256 = "1cfr95xi6hsyxw1wlrh571hc775xhwmssk3k14i8b7dgbwfmm5x1";
+  };
+}

packages/invidious/update.sh

@@ -0,0 +1,96 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p curl crystal crystal2nix jq git moreutils nix nix-prefetch pkg-config
+git_url='https://github.com/iv-org/invidious.git'
+git_branch='master'
+git_dir='/var/tmp/invidious.git'
+pkg='invidious'
+
+set -euo pipefail
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
+info() {
+    if [ -t 2 ]; then
+        set -- '\033[32m%s\033[39m\n' "$@"
+    else
+        set -- '%s\n' "$@"
+    fi
+    printf "$@" >&2
+}
+
+json_get() {
+    jq -r "$1" < 'versions.json'
+}
+
+json_set() {
+    jq --arg x "$2" "$1 = \$x" < 'versions.json' | sponge 'versions.json'
+}
+
+old_rev=$(json_get '.invidious.rev')
+old_version=$(json_get '.invidious.version')
+today=$(LANG=C date -u +'%Y-%m-%d')
+
+info "fetching $git_url..."
+if [ ! -d "$git_dir" ]; then
+    git init --initial-branch="$git_branch" "$git_dir"
+    git -C "$git_dir" remote add origin "$git_url"
+fi
+git -C "$git_dir" fetch origin "$git_branch"
+
+# use latest commit before today, we should not call the version *today*
+# because there might still be commits coming
+# use the day of the latest commit we picked as version
+new_rev=$(git -C "$git_dir" log -n 1 --format='format:%H' --before="${today}T00:00:00Z" "origin/$git_branch")
+new_version="unstable-$(TZ=UTC git -C "$git_dir" log -n 1 --date='format-local:%Y-%m-%d' --format='%cd' "$new_rev")"
+info "latest commit before $today: $new_rev"
+
+if [ "$new_rev" = "$old_rev" ]; then
+    info "$pkg is up-to-date."
+    exit
+fi
+
+json_set '.invidious.version' "$new_version"
+json_set '.invidious.rev' "$new_rev"
+new_sha256=$(nix-prefetch -I 'nixpkgs=../../..' "$pkg")
+json_set '.invidious.sha256' "$new_sha256"
+commit_msg="$pkg: $old_version -> $new_version"
+
+# fetch video.js dependencies
+info "Running scripts/fetch-player-dependencies.cr..."
+git -C "$git_dir" reset --hard "$new_rev"
+(cd "$git_dir" && crystal run scripts/fetch-player-dependencies.cr -- --minified)
+rm -f "$git_dir/assets/videojs/.gitignore"
+videojs_new_sha256=$(nix-hash --type sha256 --base32 "$git_dir/assets/videojs")
+json_set '.videojs.sha256' "$videojs_new_sha256"
+
+if git -C "$git_dir" diff-tree --quiet "${old_rev}..${new_rev}" -- 'shard.lock'; then
+    info "shard.lock did not change since $old_rev."
+else
+    info "Updating shards.nix..."
+    crystal2nix -- "$git_dir/shard.lock"  # argv's index seems broken
+
+    lsquic_old_version=$(json_get '.lsquic.version')
+    # lsquic.cr's version tracks lsquic's, so lsquic must be updated to the
+    # version in the shards file
+    lsquic_new_version=$(nix eval --raw -f 'shards.nix' lsquic.rev \
+        | sed -e 's/^v//' -e 's/-[0-9]*$//')
+    if [ "$lsquic_old_version" != "$lsquic_new_version" ]; then
+        info "Updating lsquic to $lsquic_new_version..."
+        json_set '.lsquic.version' "$lsquic_new_version"
+        lsquic_new_sha256=$(nix-prefetch -I 'nixpkgs=../../..' "${pkg}.lsquic")
+        json_set '.lsquic.sha256' "$lsquic_new_sha256"
+
+        info "Updating boringssl..."
+        # lsquic specifies the boringssl commit it requires in its README
+        boringssl_new_rev=$(curl -LSsf "https://github.com/litespeedtech/lsquic/raw/v${lsquic_new_version}/README.md" \
+            | grep -Pom1 '(?<=^git checkout ).*')
+        json_set '.boringssl.rev' "$boringssl_new_rev"
+        boringssl_new_sha256=$(nix-prefetch -I 'nixpkgs=../../..' "${pkg}.lsquic.boringssl")
+        json_set '.boringssl.sha256' "$boringssl_new_sha256"
+        commit_msg="$commit_msg
+
+lsquic: $lsquic_old_version -> $lsquic_new_version"
+    fi
+fi
+
+git commit --verbose --message "$commit_msg" -- versions.json shards.nix

packages/invidious/use-etc-ssl-certs.patch

@@ -0,0 +1,13 @@
+diff --git a/crypto/x509/x509_def.c b/crypto/x509/x509_def.c
+index d2bc3e5c1..329580075 100644
+--- a/crypto/x509/x509_def.c
++++ b/crypto/x509/x509_def.c
+@@ -67,7 +67,7 @@
+ 
+ #define X509_CERT_AREA          OPENSSLDIR
+ #define X509_CERT_DIR           OPENSSLDIR "/certs"
+-#define X509_CERT_FILE          OPENSSLDIR "/cert.pem"
++#define X509_CERT_FILE          "/etc/ssl/certs/ca-certificates.crt"
+ #define X509_PRIVATE_DIR        OPENSSLDIR "/private"
+ #define X509_CERT_DIR_EVP        "SSL_CERT_DIR"
+ #define X509_CERT_FILE_EVP       "SSL_CERT_FILE"

packages/invidious/versions.json

@@ -0,0 +1,18 @@
+{
+  "boringssl": {
+    "rev": "251b5169fd44345f455438312ec4e18ae07fd58c",
+    "sha256": "sha256-EU6T9yQCdOLx98Io8o01rEsgxDFF/Xoy42LgPopD2/A="
+  },
+  "invidious": {
+    "rev": "cf12e9dec1a1425520acd250532c7a293e4f6b66",
+    "sha256": "sha256-LXCwVcjXx0JQ9yOyiZCqWYkMmD5iCEe4vcuCZTX+S/Y=",
+    "version": "unstable-2022-09-17"
+  },
+  "lsquic": {
+    "sha256": "sha256-hG8cUvhbCNeMOsKkaJlgGpzUrIx47E/WhmPIdI5F3qM=",
+    "version": "2.18.1"
+  },
+  "videojs": {
+    "sha256": "0m09pc9acpzhfwwvc9dayl60nn28skmmglgvmlp48dlkqgfbgc27"
+  }
+}

packages/invidious/videojs.nix

@@ -0,0 +1,18 @@
+{ stdenvNoCC, cacert, crystal, openssl, pkg-config, invidious }:
+
+let
+  versions = builtins.fromJSON (builtins.readFile ./versions.json);
+in
+stdenvNoCC.mkDerivation {
+  name = "videojs";
+
+  inherit (invidious) src;
+
+  builder = ./videojs.sh;
+
+  nativeBuildInputs = [ cacert crystal openssl pkg-config ];
+
+  outputHashAlgo = "sha256";
+  outputHashMode = "recursive";
+  outputHash = versions.videojs.sha256;
+}

packages/invidious/videojs.sh

@@ -0,0 +1,9 @@
+source $stdenv/setup
+
+unpackPhase
+cd source
+# this helper downloads the videojs files and checks their checksums
+# against videojs-dependencies.yml so it should be pure
+crystal run scripts/fetch-player-dependencies.cr -- --minified
+rm -f assets/videojs/.gitignore
+mv assets/videojs "$out"