M-Labs
/
it-infra
8
0
Fork 4
Code Issues 8 Pull Requests Releases Wiki Activity

hydra: add patches to serve from local binary cache store #9

Closed
astro wants to merge 1 commits from astro/it-infra:hydra-local-binary-cache into master
pull from: astro/it-infra:hydra-local-binary-cache
merge into: M-Labs:master
Conversation 1 Commits 1 Files Changed 3
3 changed files with 207 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
nixbld-etc-nixos/configuration.nix
View File

@ -415,6 +415,8 @@ in
./hydra-conda.patch
./hydra-unbreak-sysbuild.patch
./hydra-restrictdist.patch
./hydra-local-binary-cache.patch
./hydra-sign-narinfo.patch
];
hydraPath = oa.hydraPath + ":" + super.lib.makeBinPath [ super.jq ];
});

128
nixbld-etc-nixos/hydra-local-binary-cache.patch Normal file
View File

@ -0,0 +1,128 @@
From 11787b9f8b37af064bceef8cd2fdec504bb0dad6 Mon Sep 17 00:00:00 2001
From: Astro <astro@spaceboyz.net>
Date: Thu, 15 Oct 2020 18:07:17 +0200
Subject: [PATCH 1/2] Add support for local binary cache stores for binary
cache resources
---
src/lib/Hydra/Controller/Root.pm | 44 +++++++++++++++++++++++---------
src/lib/Hydra/Helper/Nix.pm | 7 +++++
2 files changed, 39 insertions(+), 12 deletions(-)
diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm
index a9b0d558..2df4323a 100644
--- a/src/lib/Hydra/Controller/Root.pm
+++ b/src/lib/Hydra/Controller/Root.pm
@@ -15,6 +15,7 @@ use File::Basename;
use JSON;
use List::MoreUtils qw{any};
use Net::Prometheus;
+use IO::Handle;
# Put this controller at top-level.
__PACKAGE__->config->{namespace} = '';
@@ -311,11 +312,7 @@ sub nar :Local :Args(1) {
die if $path =~ /\//;
- if (!isLocalStore) {
- notFound($c, "There is no binary cache here.");
- }
-
- else {
+ if (isLocalStore) {
$path = $Nix::Config::storeDir . "/$path";
gone($c, "Path " . $path . " is no longer available.") unless isValidPath($path);
@@ -323,13 +320,26 @@ sub nar :Local :Args(1) {
$c->stash->{current_view} = 'NixNAR';
$c->stash->{storePath} = $path;
}
+
+ elsif (isLocalBinaryCacheStore && getStoreUri =~ "^file:/+(.+)") {
+ $c->response->content_type('application/x-nix-archive');
+
+ $path = "/" . $1 . "/nar/$path";
+ my $fh = new IO::Handle;
+ open $fh, "<", $path;
+ $c->response->body($fh);
+ }
+
+ else {
+ notFound($c, "There is no binary cache here.");
+ }
}
sub nix_cache_info :Path('nix-cache-info') :Args(0) {
my ($self, $c) = @_;
- if (!isLocalStore) {
+ if (!isLocalStore && !isLocalBinaryCacheStore) {
notFound($c, "There is no binary cache here.");
}
@@ -350,14 +360,11 @@ sub nix_cache_info :Path('nix-cache-info') :Args(0) {
sub narinfo :LocalRegex('^([a-z0-9]+).narinfo$') :Args(0) {
my ($self, $c) = @_;
- if (!isLocalStore) {
- notFound($c, "There is no binary cache here.");
- }
+ my $hash = $c->req->captures->[0];
- else {
- my $hash = $c->req->captures->[0];
+ die if length($hash) != 32;
- die if length($hash) != 32;
+ if (isLocalStore) {
my $path = queryPathFromHashPart($hash);
if (!$path) {
@@ -372,6 +379,19 @@ sub narinfo :LocalRegex('^([a-z0-9]+).narinfo$') :Args(0) {
$c->stash->{storePath} = $path;
$c->forward('Hydra::View::NARInfo');
}
+
+ elsif (isLocalBinaryCacheStore && getStoreUri =~ "^file:/+(.+)") {
+ $c->response->content_type('application/x-nix-archive');
+
+ my $path = "/" . $1 . "/" . $hash . ".narinfo";
+ my $fh = new IO::Handle;
+ open $fh, "<", $path;
+ $c->response->body($fh);
+ }
+
+ else {
+ notFound($c, "There is no binary cache here.");
+ }
}
diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm
index fd7a3170..2b99afa5 100644
--- a/src/lib/Hydra/Helper/Nix.pm
+++ b/src/lib/Hydra/Helper/Nix.pm
@@ -27,6 +27,7 @@ our @EXPORT = qw(
getStoreUri
readNixFile
isLocalStore
+ isLocalBinaryCacheStore
cancelBuilds restartBuilds);
@@ -481,4 +482,10 @@ sub isLocalStore {
}
+sub isLocalBinaryCacheStore {
+ my $uri = getStoreUri();
+ return $uri =~ "^file:";
+}
+
+
1;
--
2.28.0

77
nixbld-etc-nixos/hydra-sign-narinfo.patch Normal file
View File

@ -0,0 +1,77 @@
From c6185b4dc1fe87018d6129fc908791c0d2b37f82 Mon Sep 17 00:00:00 2001
From: Astro <astro@spaceboyz.net>
Date: Tue, 20 Oct 2020 16:05:33 +0200
Subject: [PATCH 2/2] sign narinfo on the fly
---
src/lib/Hydra/Controller/Root.pm | 32 ++++++++++++++++++++++++++------
1 file changed, 26 insertions(+), 6 deletions(-)
diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm
index 2df4323a..251800b2 100644
--- a/src/lib/Hydra/Controller/Root.pm
+++ b/src/lib/Hydra/Controller/Root.pm
@@ -10,12 +10,13 @@ use Hydra::View::TT;
use Digest::SHA1 qw(sha1_hex);
use Nix::Store;
use Nix::Config;
+use Nix::Utils;
+use Nix::Manifest;
use Encode;
use File::Basename;
use JSON;
use List::MoreUtils qw{any};
use Net::Prometheus;
-use IO::Handle;
# Put this controller at top-level.
__PACKAGE__->config->{namespace} = '';
@@ -321,7 +322,7 @@ sub nar :Local :Args(1) {
$c->stash->{storePath} = $path;
}
- elsif (isLocalBinaryCacheStore && getStoreUri =~ "^file:/+(.+)") {
+ elsif (isLocalBinaryCacheStore && getStoreUri =~ "^file:/+([^\?]+)") {
$c->response->content_type('application/x-nix-archive');
$path = "/" . $1 . "/nar/$path";
@@ -380,13 +381,32 @@ sub narinfo :LocalRegex('^([a-z0-9]+).narinfo$') :Args(0) {
$c->forward('Hydra::View::NARInfo');
}
- elsif (isLocalBinaryCacheStore && getStoreUri =~ "^file:/+(.+)") {
+ elsif (isLocalBinaryCacheStore && getStoreUri =~ "^file:/+([^\?]+)") {
$c->response->content_type('application/x-nix-archive');
+ setCacheHeaders($c, 24 * 60 * 60);
my $path = "/" . $1 . "/" . $hash . ".narinfo";
- my $fh = new IO::Handle;
- open $fh, "<", $path;
- $c->response->body($fh);
+ my $secretKeyFile = $c->config->{binary_cache_secret_key_file};
+ if (! defined $secretKeyFile && getStoreUri =~ "[\?\&]secret-key=([^\?\&]+)") {
+ $secretKeyFile = $1;
+ }
+ if (defined $secretKeyFile) {
+ # Optionally, sign the NAR info file
+ my $secretKey = readFile $secretKeyFile;
+ my $storePath = queryPathFromHashPart($hash);
+ my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($storePath, 1);
+ my $fingerprint = fingerprintPath($storePath, $narHash, $narSize, $refs);
+ my $sig = signString($secretKey, $fingerprint);
+
+ my $info = readFile $path;
+ $info .= "Sig: $sig\n";
+ $c->response->body($info);
+ }
+ else {
+ my $fh = new IO::Handle;
+ open $fh, "<", $path;
+ $c->response->body($fh);
+ }
}
else {
--
2.28.0