diff --git a/modules/invidious.nix b/modules/invidious.nix index eae77fd..315f27f 100644 --- a/modules/invidious.nix +++ b/modules/invidious.nix @@ -2,17 +2,17 @@ let iv_port = "3001"; domain = "yt.mc-fucker.cool"; + db_user = "kemal"; + db = "invidious"; in { services.invidious = { - #package = pkgs.callPackage /root/invidious { - # lsquic = pkgs.callPackage /root/invidious/lsquic.nix {}; - #}; + package = pkgs.callPackage /etc/nixos/packages/invidious { lsquic = pkgs.callPackage /etc/nixos/packages/invidious/lsquic.nix {}; videojs = pkgs.callPackage /etc/nixos/packages/invidious/videojs.nix {}; }; enable = true; port = (lib.strings.toInt iv_port); database = { createLocally = false; - host = "postgres"; + host = "127.0.0.1"; passwordFile = "/etc/nixos/keys/invidious-dbpassword"; }; settings = { @@ -23,12 +23,24 @@ in }; }; }; - networking.firewall.allowedTCPPorts = [ config.services.invidious.port ]; systemd.services.invidious.serviceConfig = { Restart = "always"; RestartSec = "2s"; }; + services.postgresql = { + ensureDatabases = [ db ]; + ensureUsers = [ + { + name = db_user; + ensurePermissions = { + "DATABASE ${db}" = "ALL PRIVILEGES"; + }; + } + ]; + authentication = "host invidious kemal 127.0.0.1/32 trust"; + }; + services.nginx.virtualHosts."${domain}" = { forceSSL = true; enableACME = true; diff --git a/packages/invidious/default.nix b/packages/invidious/default.nix new file mode 100644 index 0000000..087683b --- /dev/null +++ b/packages/invidious/default.nix @@ -0,0 +1,113 @@ +{ lib, stdenv, crystal, fetchFromGitHub, librsvg, pkg-config, libxml2, openssl, shards, sqlite, lsquic, videojs, nixosTests }: +let + # All versions, revisions, and checksums are stored in ./versions.json. + # The update process is the following: + # * pick the latest commit + # * update .invidious.rev, .invidious.version, and .invidious.sha256 + # * prefetch the videojs dependencies with scripts/fetch-player-dependencies.cr + # and update .videojs.sha256 (they are normally fetched during build + # but nix's sandboxing does not allow that) + # * if shard.lock changed + # * recreate shards.nix by running crystal2nix + # * update lsquic and boringssl if necessarry, lsquic.cr depends on + # the same version of lsquic and lsquic requires the boringssl + # commit mentioned in its README + versions = builtins.fromJSON (builtins.readFile ./versions.json); +in +crystal.buildCrystalPackage rec { + pname = "invidious"; + inherit (versions.invidious) version; + + src = fetchFromGitHub { + owner = "iv-org"; + repo = pname; + fetchSubmodules = true; + inherit (versions.invidious) rev sha256; + }; + + postPatch = + let + # Replacing by the value (templates) of the variables ensures that building + # fails if upstream changes the way the metadata is formatted. + branchTemplate = ''{{ "#{`git branch | sed -n '/* /s///p'`.strip}" }}''; + commitTemplate = ''{{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }}''; + versionTemplate = ''{{ "#{`git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g`.strip}" }}''; + # This always uses the latest commit which invalidates the cache even if + # the assets were not changed + assetCommitTemplate = ''{{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit -- assets`.strip}" }}''; + in + '' + for d in ${videojs}/*; do ln -s "$d" assets/videojs; done + + # Use the version metadata from the derivation instead of using git at + # build-time + substituteInPlace src/invidious.cr \ + --replace ${lib.escapeShellArg branchTemplate} '"master"' \ + --replace ${lib.escapeShellArg commitTemplate} '"${lib.substring 0 7 versions.invidious.rev}"' \ + --replace ${lib.escapeShellArg versionTemplate} '"${lib.replaceChars ["-"] ["."] (lib.substring 9 10 version)}"' \ + --replace ${lib.escapeShellArg assetCommitTemplate} '"${lib.substring 0 7 versions.invidious.rev}"' + + # Patch the assets and locales paths to be absolute + substituteInPlace src/invidious.cr \ + --replace 'public_folder "assets"' 'public_folder "${placeholder "out"}/share/invidious/assets"' + substituteInPlace src/invidious/helpers/i18n.cr \ + --replace 'File.read("locales/' 'File.read("${placeholder "out"}/share/invidious/locales/' + + # Reference sql initialisation/migration scripts by absolute path + substituteInPlace src/invidious/database/base.cr \ + --replace 'config/sql' '${placeholder "out"}/share/invidious/config/sql' + + substituteInPlace src/invidious/user/captcha.cr \ + --replace 'Process.run(%(rsvg-convert' 'Process.run(%(${lib.getBin librsvg}/bin/rsvg-convert' + ''; + + nativeBuildInputs = [ pkg-config shards ]; + buildInputs = [ libxml2 openssl sqlite ]; + + format = "crystal"; + shardsFile = ./shards.nix; + crystalBinaries.invidious = { + src = "src/invidious.cr"; + options = [ "--release" "--progress" "--verbose" "--no-debug" "-Dskip_videojs_download" ]; + }; + + postConfigure = '' + # lib includes nix store paths which can’t be patched, so the links have to + # be dereferenced first. + cp -rL lib lib2 + rm -r lib + mv lib2 lib + chmod +w -R lib + cp ${lsquic}/lib/liblsquic.a lib/lsquic/src/lsquic/ext + ''; + + postInstall = '' + mkdir -p $out/share/invidious/config + + # Copy static parts + cp -r assets locales $out/share/invidious + cp -r config/sql $out/share/invidious/config + ''; + + # Invidious tries to open config/config.yml and connect to the database, even + # when running --help. This specifies a minimal configuration in an + # environment variable. Even though the database is bogus, --help still + # works. + installCheckPhase = '' + INVIDIOUS_CONFIG="database_url: sqlite3:///dev/null" $out/bin/invidious --help + ''; + + passthru = { + inherit lsquic; + tests = { inherit (nixosTests) invidious; }; + updateScript = ./update.sh; + }; + + meta = with lib; { + description = "An open source alternative front-end to YouTube"; + homepage = "https://invidious.io/"; + license = licenses.agpl3; + maintainers = with maintainers; [ infinisil sbruder ]; + broken = stdenv.isDarwin && stdenv.isAarch64; + }; +} diff --git a/packages/invidious/lsquic.nix b/packages/invidious/lsquic.nix new file mode 100644 index 0000000..ca04c97 --- /dev/null +++ b/packages/invidious/lsquic.nix @@ -0,0 +1,111 @@ +{ lib, boringssl, stdenv, fetchgit, fetchFromGitHub, fetchurl, cmake, zlib, perl, libevent }: +let + versions = builtins.fromJSON (builtins.readFile ./versions.json); + + fetchGitilesPatch = { name, url, sha256 }: + fetchurl { + url = "${url}%5E%21?format=TEXT"; + inherit name sha256; + downloadToTemp = true; + postFetch = '' + base64 -d < $downloadedFile > $out + ''; + }; + + # lsquic requires a specific boringssl version (noted in its README) + boringssl' = boringssl.overrideAttrs ({ preBuild, ... }: { + version = versions.boringssl.rev; + src = fetchgit { + url = "https://boringssl.googlesource.com/boringssl"; + inherit (versions.boringssl) rev sha256; + }; + + patches = [ + # Use /etc/ssl/certs/ca-certificates.crt instead of /etc/ssl/cert.pem + ./use-etc-ssl-certs.patch + + # because lsquic requires that specific boringssl version and that + # version does not yet include fixes for gcc11 build errors, they + # must be backported + (fetchGitilesPatch { + name = "fix-mismatch-between-header-and-implementation-of-bn_sqr_comba8.patch"; + url = "https://boringssl.googlesource.com/boringssl/+/139adff9b27eaf0bdaac664ec4c9a7db2fe3f920"; + sha256 = "05sp602dvh50v46jkzmh4sf4wqnq5bwy553596g2rhxg75bailjj"; + }) + (fetchGitilesPatch { + name = "use-an-unsized-helper-for-truncated-SHA-512-variants.patch"; + url = "https://boringssl.googlesource.com/boringssl/+/a24ab549e6ae246b391155d7bed3790ac0e07de2"; + sha256 = "0483jkpg4g64v23ln2blb74xnmzdjcn3r7w4zk7nfg8j3q5f9lxm"; + }) +/* + # the following patch is too complex, so we will modify the build flags + # of crypto/fipsmodule/CMakeFiles/fipsmodule.dir/bcm.c.o in preBuild + # and turn off -Werror=stringop-overflow + (fetchGitilesPatch { + name = "make-md32_common.h-single-included-and-use-an-unsized-helper-for-SHA-256.patch"; + url = "https://boringssl.googlesource.com/boringssl/+/597ffef971dd980b7de5e97a0c9b7ca26eec94bc"; + sha256 = "1y0bkkdf1ccd6crx326agp01q22clm4ai4p982y7r6dkmxmh52qr"; + }) +*/ + (fetchGitilesPatch { + name = "fix-array-parameter-warnings.patch"; + url = "https://boringssl.googlesource.com/boringssl/+/92c6fbfc4c44dc8462d260d836020d2b793e7804"; + sha256 = "0h4sl95i8b0dj0na4ngf50wg54raxyjxl1zzwdc810abglp10vnv"; + }) + ]; + + preBuild = '' + ${preBuild} + sed -e '/^build crypto\/fipsmodule\/CMakeFiles\/fipsmodule\.dir\/bcm\.c\.o:/,/^ *FLAGS =/ s/^ *FLAGS = -Werror/& -Wno-error=stringop-overflow/' \ + -i build.ninja + ''; + }); +in +stdenv.mkDerivation rec { + pname = "lsquic"; + version = versions.lsquic.version; + + src = fetchFromGitHub { + owner = "litespeedtech"; + repo = pname; + rev = "v${version}"; + inherit (versions.lsquic) sha256; + fetchSubmodules = true; + }; + + nativeBuildInputs = [ cmake perl ]; + buildInputs = [ boringssl' libevent zlib ]; + + cmakeFlags = [ + "-DBORINGSSL_DIR=${lib.getDev boringssl'}" + "-DBORINGSSL_LIB_crypto=${lib.getLib boringssl'}/lib/libcrypto.a" + "-DBORINGSSL_LIB_ssl=${lib.getLib boringssl'}/lib/libssl.a" + "-DZLIB_LIB=${zlib}/lib/libz.so" + ]; + + # adapted from lsquic.cr’s Dockerfile + # (https://github.com/iv-org/lsquic.cr/blob/master/docker/Dockerfile) + installPhase = '' + runHook preInstall + + mkdir combinedlib + cd combinedlib + ar -x ${lib.getLib boringssl'}/lib/libssl.a + ar -x ${lib.getLib boringssl'}/lib/libcrypto.a + ar -x ../src/liblsquic/liblsquic.a + ar rc liblsquic.a *.o + ranlib liblsquic.a + install -D liblsquic.a $out/lib/liblsquic.a + + runHook postInstall + ''; + + passthru.boringssl = boringssl'; + + meta = with lib; { + description = "A library for QUIC and HTTP/3 (version for Invidious)"; + homepage = "https://github.com/litespeedtech/lsquic"; + maintainers = with maintainers; [ infinisil sbruder ]; + license = with licenses; [ openssl isc mit bsd3 ]; # statically links against boringssl, so has to include its licenses + }; +} diff --git a/packages/invidious/shards.nix b/packages/invidious/shards.nix new file mode 100644 index 0000000..e5f297d --- /dev/null +++ b/packages/invidious/shards.nix @@ -0,0 +1,80 @@ +{ + athena-negotiation = { + owner = "athena-framework"; + repo = "negotiation"; + rev = "v0.1.1"; + sha256 = "1vkk59lqrxb0l8kyzs114i3c18zb2bdiah2xhazkk8q7x6fz4yzk"; + }; + backtracer = { + owner = "sija"; + repo = "backtracer.cr"; + rev = "v1.2.1"; + sha256 = "02r1l7rn2wsljkx495s5s7j04zgn73m2kx0hkzs7620camvlwbqq"; + }; + db = { + owner = "crystal-lang"; + repo = "crystal-db"; + rev = "v0.10.1"; + sha256 = "03c5h14z6h2mxnx949lihnyqjd19hcj38iasdwq9fp95h8cld376"; + }; + exception_page = { + owner = "crystal-loot"; + repo = "exception_page"; + rev = "v0.2.2"; + sha256 = "1c8askb9b7621jjz5pjj6b8pdbhw3r1l3dym6swg1saspf5j3jwi"; + }; + kemal = { + owner = "kemalcr"; + repo = "kemal"; + rev = "v1.1.2"; + sha256 = "1149q4qw0zrws5asqqr4snrdi67xsmisdcq58zcrbgqgsxgly9d0"; + }; + kilt = { + owner = "jeromegn"; + repo = "kilt"; + rev = "v0.6.1"; + sha256 = "0dpc15y9m8c5l9zdfif6jlf7zmkrlm9w4m2igi5xa22fdjwamwfp"; + }; + lsquic = { + owner = "iv-org"; + repo = "lsquic.cr"; + rev = "v2.18.1-2"; + sha256 = "0bljk0pwbjb813dfwrhgi00w2ai09k868xvak4hfzdkbmpc7id6y"; + }; + pg = { + owner = "will"; + repo = "crystal-pg"; + rev = "v0.24.0"; + sha256 = "07i5bqkv5j6y6f8v5cpqdxc5wzzrvgv3ds24znv4mzv6nc84csn4"; + }; + protodec = { + owner = "iv-org"; + repo = "protodec"; + rev = "v0.1.4"; + sha256 = "15azh9izxqgwpgkpicmivfdz31wkibnwy09rwhxsg0lyc4wf8xj9"; + }; + radix = { + owner = "luislavena"; + repo = "radix"; + rev = "v0.4.1"; + sha256 = "1l08cydkdidq9yyil1wl240hvk41iycv04jrg6nx5mkvzw4z1bzg"; + }; + spectator = { + owner = "icy-arctic-fox"; + repo = "spectator"; + rev = "v0.10.4"; + sha256 = "0rcxq2nbslvwrd8m9ajw6dzaw3hagxmkdy9s8p34cgnr4c9dijdq"; + }; + sqlite3 = { + owner = "crystal-lang"; + repo = "crystal-sqlite3"; + rev = "v0.18.0"; + sha256 = "03nnvpchhq9f9ywsm3pk2rrj4a3figw7xs96zdziwgr5znkz6x93"; + }; + ameba = { + owner = "crystal-ameba"; + repo = "ameba"; + rev = "v0.14.3"; + sha256 = "1cfr95xi6hsyxw1wlrh571hc775xhwmssk3k14i8b7dgbwfmm5x1"; + }; +} diff --git a/packages/invidious/update.sh b/packages/invidious/update.sh new file mode 100755 index 0000000..4810534 --- /dev/null +++ b/packages/invidious/update.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p curl crystal crystal2nix jq git moreutils nix nix-prefetch pkg-config +git_url='https://github.com/iv-org/invidious.git' +git_branch='master' +git_dir='/var/tmp/invidious.git' +pkg='invidious' + +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")" + +info() { + if [ -t 2 ]; then + set -- '\033[32m%s\033[39m\n' "$@" + else + set -- '%s\n' "$@" + fi + printf "$@" >&2 +} + +json_get() { + jq -r "$1" < 'versions.json' +} + +json_set() { + jq --arg x "$2" "$1 = \$x" < 'versions.json' | sponge 'versions.json' +} + +old_rev=$(json_get '.invidious.rev') +old_version=$(json_get '.invidious.version') +today=$(LANG=C date -u +'%Y-%m-%d') + +info "fetching $git_url..." +if [ ! -d "$git_dir" ]; then + git init --initial-branch="$git_branch" "$git_dir" + git -C "$git_dir" remote add origin "$git_url" +fi +git -C "$git_dir" fetch origin "$git_branch" + +# use latest commit before today, we should not call the version *today* +# because there might still be commits coming +# use the day of the latest commit we picked as version +new_rev=$(git -C "$git_dir" log -n 1 --format='format:%H' --before="${today}T00:00:00Z" "origin/$git_branch") +new_version="unstable-$(TZ=UTC git -C "$git_dir" log -n 1 --date='format-local:%Y-%m-%d' --format='%cd' "$new_rev")" +info "latest commit before $today: $new_rev" + +if [ "$new_rev" = "$old_rev" ]; then + info "$pkg is up-to-date." + exit +fi + +json_set '.invidious.version' "$new_version" +json_set '.invidious.rev' "$new_rev" +new_sha256=$(nix-prefetch -I 'nixpkgs=../../..' "$pkg") +json_set '.invidious.sha256' "$new_sha256" +commit_msg="$pkg: $old_version -> $new_version" + +# fetch video.js dependencies +info "Running scripts/fetch-player-dependencies.cr..." +git -C "$git_dir" reset --hard "$new_rev" +(cd "$git_dir" && crystal run scripts/fetch-player-dependencies.cr -- --minified) +rm -f "$git_dir/assets/videojs/.gitignore" +videojs_new_sha256=$(nix-hash --type sha256 --base32 "$git_dir/assets/videojs") +json_set '.videojs.sha256' "$videojs_new_sha256" + +if git -C "$git_dir" diff-tree --quiet "${old_rev}..${new_rev}" -- 'shard.lock'; then + info "shard.lock did not change since $old_rev." +else + info "Updating shards.nix..." + crystal2nix -- "$git_dir/shard.lock" # argv's index seems broken + + lsquic_old_version=$(json_get '.lsquic.version') + # lsquic.cr's version tracks lsquic's, so lsquic must be updated to the + # version in the shards file + lsquic_new_version=$(nix eval --raw -f 'shards.nix' lsquic.rev \ + | sed -e 's/^v//' -e 's/-[0-9]*$//') + if [ "$lsquic_old_version" != "$lsquic_new_version" ]; then + info "Updating lsquic to $lsquic_new_version..." + json_set '.lsquic.version' "$lsquic_new_version" + lsquic_new_sha256=$(nix-prefetch -I 'nixpkgs=../../..' "${pkg}.lsquic") + json_set '.lsquic.sha256' "$lsquic_new_sha256" + + info "Updating boringssl..." + # lsquic specifies the boringssl commit it requires in its README + boringssl_new_rev=$(curl -LSsf "https://github.com/litespeedtech/lsquic/raw/v${lsquic_new_version}/README.md" \ + | grep -Pom1 '(?<=^git checkout ).*') + json_set '.boringssl.rev' "$boringssl_new_rev" + boringssl_new_sha256=$(nix-prefetch -I 'nixpkgs=../../..' "${pkg}.lsquic.boringssl") + json_set '.boringssl.sha256' "$boringssl_new_sha256" + commit_msg="$commit_msg + +lsquic: $lsquic_old_version -> $lsquic_new_version" + fi +fi + +git commit --verbose --message "$commit_msg" -- versions.json shards.nix diff --git a/packages/invidious/use-etc-ssl-certs.patch b/packages/invidious/use-etc-ssl-certs.patch new file mode 100644 index 0000000..b60b0b1 --- /dev/null +++ b/packages/invidious/use-etc-ssl-certs.patch @@ -0,0 +1,13 @@ +diff --git a/crypto/x509/x509_def.c b/crypto/x509/x509_def.c +index d2bc3e5c1..329580075 100644 +--- a/crypto/x509/x509_def.c ++++ b/crypto/x509/x509_def.c +@@ -67,7 +67,7 @@ + + #define X509_CERT_AREA OPENSSLDIR + #define X509_CERT_DIR OPENSSLDIR "/certs" +-#define X509_CERT_FILE OPENSSLDIR "/cert.pem" ++#define X509_CERT_FILE "/etc/ssl/certs/ca-certificates.crt" + #define X509_PRIVATE_DIR OPENSSLDIR "/private" + #define X509_CERT_DIR_EVP "SSL_CERT_DIR" + #define X509_CERT_FILE_EVP "SSL_CERT_FILE" diff --git a/packages/invidious/versions.json b/packages/invidious/versions.json new file mode 100644 index 0000000..4609559 --- /dev/null +++ b/packages/invidious/versions.json @@ -0,0 +1,18 @@ +{ + "boringssl": { + "rev": "251b5169fd44345f455438312ec4e18ae07fd58c", + "sha256": "sha256-EU6T9yQCdOLx98Io8o01rEsgxDFF/Xoy42LgPopD2/A=" + }, + "invidious": { + "rev": "cf12e9dec1a1425520acd250532c7a293e4f6b66", + "sha256": "sha256-LXCwVcjXx0JQ9yOyiZCqWYkMmD5iCEe4vcuCZTX+S/Y=", + "version": "unstable-2022-09-17" + }, + "lsquic": { + "sha256": "sha256-hG8cUvhbCNeMOsKkaJlgGpzUrIx47E/WhmPIdI5F3qM=", + "version": "2.18.1" + }, + "videojs": { + "sha256": "0m09pc9acpzhfwwvc9dayl60nn28skmmglgvmlp48dlkqgfbgc27" + } +} diff --git a/packages/invidious/videojs.nix b/packages/invidious/videojs.nix new file mode 100644 index 0000000..e447079 --- /dev/null +++ b/packages/invidious/videojs.nix @@ -0,0 +1,18 @@ +{ stdenvNoCC, cacert, crystal, openssl, pkg-config, invidious }: + +let + versions = builtins.fromJSON (builtins.readFile ./versions.json); +in +stdenvNoCC.mkDerivation { + name = "videojs"; + + inherit (invidious) src; + + builder = ./videojs.sh; + + nativeBuildInputs = [ cacert crystal openssl pkg-config ]; + + outputHashAlgo = "sha256"; + outputHashMode = "recursive"; + outputHash = versions.videojs.sha256; +} diff --git a/packages/invidious/videojs.sh b/packages/invidious/videojs.sh new file mode 100644 index 0000000..31609db --- /dev/null +++ b/packages/invidious/videojs.sh @@ -0,0 +1,9 @@ +source $stdenv/setup + +unpackPhase +cd source +# this helper downloads the videojs files and checks their checksums +# against videojs-dependencies.yml so it should be pure +crystal run scripts/fetch-player-dependencies.cr -- --minified +rm -f assets/videojs/.gitignore +mv assets/videojs "$out"