From 2a92a84e8a83c5fe9b132cb46257736f70de2d33 Mon Sep 17 00:00:00 2001 From: Astro Date: Wed, 11 Nov 2020 23:15:25 +0100 Subject: [PATCH 1/3] nixbld: add hydra-binary-cache-store.patch that is rebased on hydra-restrictdist.patch the patch can only restrict downloads of .narinfo but not of nar/*.nar.xz --- nixbld-etc-nixos/configuration.nix | 1 + .../hydra-binary-cache-store.patch | 224 ++++++++++++++++++ 2 files changed, 225 insertions(+) create mode 100644 nixbld-etc-nixos/hydra-binary-cache-store.patch diff --git a/nixbld-etc-nixos/configuration.nix b/nixbld-etc-nixos/configuration.nix index 4e49e31..12cb651 100644 --- a/nixbld-etc-nixos/configuration.nix +++ b/nixbld-etc-nixos/configuration.nix @@ -436,6 +436,7 @@ in ./hydra-unbreak-sysbuild.patch ./hydra-restrictdist.patch ./hydra-giteastatus.patch + ./hydra-binary-cache-store.patch ]; hydraPath = oa.hydraPath + ":" + super.lib.makeBinPath [ super.jq ]; }); diff --git a/nixbld-etc-nixos/hydra-binary-cache-store.patch b/nixbld-etc-nixos/hydra-binary-cache-store.patch new file mode 100644 index 0000000..05217f3 --- /dev/null +++ b/nixbld-etc-nixos/hydra-binary-cache-store.patch @@ -0,0 +1,224 @@ +From 253a07ce8f1f010df19f5a2cd7045e978a4eca01 Mon Sep 17 00:00:00 2001 +From: Astro +Date: Thu, 15 Oct 2020 18:07:17 +0200 +Subject: [PATCH 1/2] Add support for local binary cache stores for binary + cache resources + +--- + src/lib/Hydra/Controller/Root.pm | 46 +++++++++++++++++++++++--------- + src/lib/Hydra/Helper/Nix.pm | 7 +++++ + 2 files changed, 41 insertions(+), 12 deletions(-) + +diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm +index 71869ba0..4276f95a 100644 +--- a/src/lib/Hydra/Controller/Root.pm ++++ b/src/lib/Hydra/Controller/Root.pm +@@ -15,6 +15,7 @@ use File::Basename; + use JSON; + use List::MoreUtils qw{any}; + use Net::Prometheus; ++use IO::Handle; + + # Put this controller at top-level. + __PACKAGE__->config->{namespace} = ''; +@@ -316,11 +317,7 @@ sub nar :Local :Args(1) { + + die if $path =~ /\//; + +- if (!isLocalStore) { +- notFound($c, "There is no binary cache here."); +- } +- +- else { ++ if (isLocalStore) { + $path = $Nix::Config::storeDir . "/$path"; + + gone($c, "Path " . $path . " is no longer available.") unless isValidPath($path); +@@ -329,13 +326,26 @@ sub nar :Local :Args(1) { + $c->stash->{current_view} = 'NixNAR'; + $c->stash->{storePath} = $path; + } ++ ++ elsif (isLocalBinaryCacheStore && getStoreUri =~ "^file:/+(.+)") { ++ $c->response->content_type('application/x-nix-archive'); ++ ++ $path = "/" . $1 . "/nar/$path"; ++ my $fh = new IO::Handle; ++ open $fh, "<", $path; ++ $c->response->body($fh); ++ } ++ ++ else { ++ notFound($c, "There is no binary cache here."); ++ } + } + + + sub nix_cache_info :Path('nix-cache-info') :Args(0) { + my ($self, $c) = @_; + +- if (!isLocalStore) { ++ if (!isLocalStore && !isLocalBinaryCacheStore) { + notFound($c, "There is no binary cache here."); + } + +@@ -356,14 +366,11 @@ sub nix_cache_info :Path('nix-cache-info') :Args(0) { + sub narinfo :LocalRegex('^([a-z0-9]+).narinfo$') :Args(0) { + my ($self, $c) = @_; + +- if (!isLocalStore) { +- notFound($c, "There is no binary cache here."); +- } ++ my $hash = $c->req->captures->[0]; + +- else { +- my $hash = $c->req->captures->[0]; ++ die if length($hash) != 32; + +- die if length($hash) != 32; ++ if (isLocalStore) { + my $path = queryPathFromHashPart($hash); + + if (!$path) { +@@ -379,6 +386,21 @@ sub narinfo :LocalRegex('^([a-z0-9]+).narinfo$') :Args(0) { + $c->stash->{storePath} = $path; + $c->forward('Hydra::View::NARInfo'); + } ++ ++ elsif (isLocalBinaryCacheStore && getStoreUri =~ "^file:/+(.+)") { ++ $c->response->content_type('application/x-nix-archive'); ++ ++ my $path = "/" . $1 . "/" . $hash . ".narinfo"; ++ notFound($c, "Redistribution restricted") if isRedistRestricted($path); ++ ++ my $fh = new IO::Handle; ++ open $fh, "<", $path; ++ $c->response->body($fh); ++ } ++ ++ else { ++ notFound($c, "There is no binary cache here."); ++ } + } + + +diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm +index fd7a3170..2b99afa5 100644 +--- a/src/lib/Hydra/Helper/Nix.pm ++++ b/src/lib/Hydra/Helper/Nix.pm +@@ -27,6 +27,7 @@ our @EXPORT = qw( + getStoreUri + readNixFile + isLocalStore ++ isLocalBinaryCacheStore + cancelBuilds restartBuilds); + + +@@ -481,4 +482,10 @@ sub isLocalStore { + } + + ++sub isLocalBinaryCacheStore { ++ my $uri = getStoreUri(); ++ return $uri =~ "^file:"; ++} ++ ++ + 1; +-- +2.29.0 + +From 2f8c19772712c507096ba9d39f46bc9dc980332d Mon Sep 17 00:00:00 2001 +From: Astro +Date: Tue, 20 Oct 2020 16:05:33 +0200 +Subject: [PATCH 2/2] sign narinfo on the fly + +--- + src/lib/Hydra/Controller/Root.pm | 43 ++++++++++++++++++++++++++------ + 1 file changed, 35 insertions(+), 8 deletions(-) + +diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm +index 4276f95a..21ff8a97 100644 +--- a/src/lib/Hydra/Controller/Root.pm ++++ b/src/lib/Hydra/Controller/Root.pm +@@ -10,12 +10,13 @@ use Hydra::View::TT; + use Digest::SHA1 qw(sha1_hex); + use Nix::Store; + use Nix::Config; ++use Nix::Utils; ++use Nix::Manifest; + use Encode; + use File::Basename; + use JSON; + use List::MoreUtils qw{any}; + use Net::Prometheus; +-use IO::Handle; + + # Put this controller at top-level. + __PACKAGE__->config->{namespace} = ''; +@@ -327,7 +328,7 @@ sub nar :Local :Args(1) { + $c->stash->{storePath} = $path; + } + +- elsif (isLocalBinaryCacheStore && getStoreUri =~ "^file:/+(.+)") { ++ elsif (isLocalBinaryCacheStore && getStoreUri =~ "^file:/+([^\?]+)") { + $c->response->content_type('application/x-nix-archive'); + + $path = "/" . $1 . "/nar/$path"; +@@ -381,21 +382,47 @@ sub narinfo :LocalRegex('^([a-z0-9]+).narinfo$') :Args(0) { + setCacheHeaders($c, 60 * 60); + return; + } +- notFound($c, "Redistribution restricted") if isRedistRestricted($path); + + $c->stash->{storePath} = $path; + $c->forward('Hydra::View::NARInfo'); + } + +- elsif (isLocalBinaryCacheStore && getStoreUri =~ "^file:/+(.+)") { ++ elsif (isLocalBinaryCacheStore && getStoreUri =~ "^file:/+([^\?]+)") { + $c->response->content_type('application/x-nix-archive'); ++ setCacheHeaders($c, 24 * 60 * 60); + + my $path = "/" . $1 . "/" . $hash . ".narinfo"; +- notFound($c, "Redistribution restricted") if isRedistRestricted($path); ++ if (!-f $path) { ++ return notFound($c, "NARInfo not found"); ++ } ++ my $content = readFile $path; ++ my %info; ++ foreach my $line (split "\n", $content) { ++ return undef unless $line =~ /^(.*): (.*)$/; ++ $info{$1} = $2; ++ } ++ notFound($c, "Redistribution restricted") if isRedistRestricted($info{StorePath}); + +- my $fh = new IO::Handle; +- open $fh, "<", $path; +- $c->response->body($fh); ++ my $secretKeyFile = $c->config->{binary_cache_secret_key_file}; ++ if (! defined $secretKeyFile && getStoreUri =~ "[\?\&]secret-key=([^\?\&]+)") { ++ $secretKeyFile = $1; ++ } ++ if (defined $secretKeyFile) { ++ # Optionally, sign the NAR info file ++ my @refs = map { $Nix::Config::storeDir . "/" . $_ } split(" ", $info{References}); ++ my $fingerprint = fingerprintPath($info{StorePath}, $info{NarHash}, $info{NarSize}, [ @refs ]); ++ my $secretKey = readFile $secretKeyFile; ++ my $sig = signString($secretKey, $fingerprint); ++ ++ $content =~ s/^Sig:.+\n//m; ++ $content .= "Sig: $sig\n"; ++ $c->response->body($content); ++ } ++ else { ++ my $fh = new IO::Handle; ++ open $fh, "<", $path; ++ $c->response->body($fh); ++ } + } + + else { +-- +2.29.0 + -- 2.42.0 From a16ab7d2de9a9982bcd932395604161ad6cb06f2 Mon Sep 17 00:00:00 2001 From: Astro Date: Thu, 26 Nov 2020 17:53:32 +0100 Subject: [PATCH 2/3] nixbld: add :raw for opening .nar.xz in hydra-binary-cache-store.patch --- nixbld-etc-nixos/hydra-binary-cache-store.patch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixbld-etc-nixos/hydra-binary-cache-store.patch b/nixbld-etc-nixos/hydra-binary-cache-store.patch index 05217f3..1570e5b 100644 --- a/nixbld-etc-nixos/hydra-binary-cache-store.patch +++ b/nixbld-etc-nixos/hydra-binary-cache-store.patch @@ -44,7 +44,7 @@ index 71869ba0..4276f95a 100644 + + $path = "/" . $1 . "/nar/$path"; + my $fh = new IO::Handle; -+ open $fh, "<", $path; ++ open $fh, "<:raw", $path; + $c->response->body($fh); + } + -- 2.42.0 From 9d0da9bf44a5c2abee48999d21f3ce370fd3a0fa Mon Sep 17 00:00:00 2001 From: Astro Date: Mon, 22 Feb 2021 00:17:24 +0100 Subject: [PATCH 3/3] nixbld: add some error handling to hydra-binary-cache-store.patch --- .../hydra-binary-cache-store.patch | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/nixbld-etc-nixos/hydra-binary-cache-store.patch b/nixbld-etc-nixos/hydra-binary-cache-store.patch index 1570e5b..a66ba66 100644 --- a/nixbld-etc-nixos/hydra-binary-cache-store.patch +++ b/nixbld-etc-nixos/hydra-binary-cache-store.patch @@ -222,3 +222,30 @@ index 4276f95a..21ff8a97 100644 -- 2.29.0 +From e20013bff3c92b21bab6e476489a1508df1cf348 Mon Sep 17 00:00:00 2001 +From: Astro +Date: Mon, 22 Feb 2021 00:12:12 +0100 +Subject: [PATCH] check if nar not found + +--- + src/lib/Hydra/Controller/Root.pm | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm +index 6be9f2bc..37464c34 100644 +--- a/src/lib/Hydra/Controller/Root.pm ++++ b/src/lib/Hydra/Controller/Root.pm +@@ -326,6 +326,10 @@ sub nar :Local :Args(1) { + $c->response->content_type('application/x-nix-archive'); + + $path = "/" . $1 . "/nar/$path"; ++ if (!-f $path) { ++ return notFound($c, "NAR not found"); ++ } ++ + my $fh = new IO::Handle; + open $fh, "<:raw", $path; + $c->response->body($fh); +-- +2.30.0 + -- 2.42.0