push sheeet

pkgs/build-support/fetchdocker/credentials.nix

@@ -0,0 +1,36 @@
+{ lib }:
+# We provide three paths to get the credentials into the builder's
+# environment:
+#
+# 1. Via impureEnvVars. This method is difficult for multi-user Nix
+#    installations (but works very well for single-user Nix
+#    installations!) because it requires setting the environment
+#    variables on the nix-daemon which is either complicated or unsafe
+#    (i.e: configuring via Nix means the secrets will be persisted
+#    into the store)
+#
+# 2. If the DOCKER_CREDENTIALS key with a path to a credentials file
+#    is added to the NIX_PATH (usually via the '-I ' argument to most
+#    Nix tools) then an attempt will be made to read credentials from
+#    it. The semantics are simple, the file should contain two lines
+#    for the username and password based authentication:
+#
+# $ cat ./credentials-file.txt
+# DOCKER_USER=myusername
+# DOCKER_PASS=mypassword
+#
+#    ... and a single line for the token based authentication:
+#
+# $ cat ./credentials-file.txt
+# DOCKER_TOKEN=mytoken
+#
+# 3. A credential file at /etc/nix-docker-credentials.txt with the
+#    same format as the file described in #2 can also be used to
+#    communicate credentials to the builder. This is necessary for
+#    situations (like Hydra) where you cannot customize the NIX_PATH
+#    given to the nix-build invocation to provide it with the
+#    DOCKER_CREDENTIALS path
+let
+  pathParts = (builtins.filter ({ prefix, path }: "DOCKER_CREDENTIALS" == prefix) builtins.nixPath);
+in
+lib.optionalString (pathParts != [ ]) ((builtins.head pathParts).path)

pkgs/build-support/fetchdocker/default.nix

@@ -0,0 +1,81 @@
+{
+  stdenv,
+  lib,
+  coreutils,
+  bash,
+  gnutar,
+  writeText,
+}:
+let
+  stripScheme = builtins.replaceStrings [ "https://" "http://" ] [ "" "" ];
+  stripNixStore = s: lib.removePrefix "${builtins.storeDir}/" s;
+in
+{
+  name,
+  registry ? "https://registry-1.docker.io/v2/",
+  repository ? "library",
+  imageName,
+  tag,
+  imageLayers,
+  imageConfig,
+  image ? "${stripScheme registry}/${repository}/${imageName}:${tag}",
+}:
+
+# Make sure there are *no* slashes in the repository or container
+# names since we use these to make the output derivation name for the
+# nix-store path.
+assert null == lib.findFirst (c: "/" == c) null (lib.stringToCharacters repository);
+assert null == lib.findFirst (c: "/" == c) null (lib.stringToCharacters imageName);
+
+let
+  # Abuse paths to collapse possible double slashes
+  repoTag0 = toString (/. + "/${stripScheme registry}/${repository}/${imageName}");
+  repoTag1 = lib.removePrefix "/" repoTag0;
+
+  layers = map stripNixStore imageLayers;
+
+  manifest = writeText "manifest.json" (
+    builtins.toJSON [
+      {
+        Config = stripNixStore imageConfig;
+        Layers = layers;
+        RepoTags = [ "${repoTag1}:${tag}" ];
+      }
+    ]
+  );
+
+  repositories = writeText "repositories" (
+    builtins.toJSON {
+      ${repoTag1} = {
+        ${tag} = lib.last layers;
+      };
+    }
+  );
+
+  imageFileStorePaths = writeText "imageFileStorePaths.txt" (
+    lib.concatStringsSep "\n" ((lib.unique imageLayers) ++ [ imageConfig ])
+  );
+in
+stdenv.mkDerivation {
+  builder = ./fetchdocker-builder.sh;
+  buildInputs = [ coreutils ];
+  preferLocalBuild = true;
+
+  inherit
+    name
+    imageName
+    repository
+    tag
+    ;
+  inherit
+    bash
+    gnutar
+    manifest
+    repositories
+    ;
+  inherit imageFileStorePaths;
+
+  passthru = {
+    inherit image;
+  };
+}

pkgs/build-support/fetchdocker/fetchDockerConfig.nix

@@ -0,0 +1,26 @@
+pkgargs@{
+  stdenv,
+  lib,
+  haskellPackages,
+  writeText,
+  gawk,
+}:
+let
+  generic-fetcher = import ./generic-fetcher.nix pkgargs;
+in
+
+args@{
+  repository ? "library",
+  imageName,
+  tag,
+  ...
+}:
+
+generic-fetcher (
+  {
+    fetcher = "hocker-config";
+    name = "${repository}_${imageName}_${tag}-config.json";
+    tag = "unused";
+  }
+  // args
+)

pkgs/build-support/fetchdocker/fetchDockerLayer.nix

@@ -0,0 +1,21 @@
+pkgargs@{
+  stdenv,
+  lib,
+  haskellPackages,
+  writeText,
+  gawk,
+}:
+let
+  generic-fetcher = import ./generic-fetcher.nix pkgargs;
+in
+
+args@{ layerDigest, ... }:
+
+generic-fetcher (
+  {
+    fetcher = "hocker-layer";
+    name = "docker-layer-${layerDigest}.tar.gz";
+    tag = "unused";
+  }
+  // args
+)

pkgs/build-support/fetchdocker/fetchdocker-builder.sh

@@ -0,0 +1,26 @@
+echo "exporting ${repository}/${imageName} (tag: ${tag}) into ${out}"
+mkdir -p "${out}"
+
+cat <<EOF > "${out}/compositeImage.sh"
+#! ${bash}/bin/bash
+#
+# Create a tar archive of a docker image's layers, docker image config
+# json, manifest.json, and repositories json; this streams directly to
+# stdout and is intended to be used in concert with docker load, i.e:
+#
+# ${out}/compositeImage.sh | docker load
+
+# The first character follow the 's' command for sed becomes the
+# delimiter sed will use; this makes the transformation regex easy to
+# read. We feed tar a file listing the files we want in the archive,
+# because the paths are absolute and docker load wants them flattened in
+# the archive, we need to transform all of the paths going in by
+# stripping everything *including* the last solidus so that we end up
+# with the basename of the path.
+${gnutar}/bin/tar \
+  --transform='s=.*/==' \
+  --transform="s=.*-manifest.json=manifest.json=" \
+  --transform="s=.*-repositories=repositories=" \
+  -c "${manifest}" "${repositories}" -T "${imageFileStorePaths}"
+EOF
+chmod +x "${out}/compositeImage.sh"

pkgs/build-support/fetchdocker/generic-fetcher.nix

@@ -0,0 +1,104 @@
+{
+  stdenv,
+  lib,
+  haskellPackages,
+  writeText,
+  gawk,
+}:
+let
+  awk = "${gawk}/bin/awk";
+  dockerCredentialsFile = import ./credentials.nix { inherit lib; };
+in
+{
+  fetcher,
+  name,
+  registry ? "https://registry-1.docker.io/v2/",
+  repository ? "library",
+  imageName,
+  sha256,
+  tag ? "",
+  layerDigest ? "",
+}:
+
+# There must be no slashes in the repository or container names since
+# we use these to make the output derivation name for the nix store
+# path
+assert null == lib.findFirst (c: "/" == c) null (lib.stringToCharacters repository);
+assert null == lib.findFirst (c: "/" == c) null (lib.stringToCharacters imageName);
+
+# Only allow hocker-config and hocker-layer as fetchers for now
+assert (
+  builtins.elem fetcher [
+    "hocker-config"
+    "hocker-layer"
+  ]
+);
+
+# If layerDigest is non-empty then it must not have a 'sha256:' prefix!
+assert (if layerDigest != "" then !lib.hasPrefix "sha256:" layerDigest else true);
+
+let
+  layerDigestFlag = lib.optionalString (layerDigest != "") "--layer ${layerDigest}";
+in
+stdenv.mkDerivation {
+  inherit name;
+  builder = writeText "${fetcher}-builder.sh" ''
+    echo "${fetcher} exporting to $out"
+
+    declare -A creds
+
+    # This is a hack for Hydra since we have no way of adding values
+    # to the NIX_PATH for Hydra jobsets!!
+    staticCredentialsFile="/etc/nix-docker-credentials.txt"
+    if [ ! -f "$dockerCredentialsFile" -a -f "$staticCredentialsFile" ]; then
+      echo "credentials file not set, falling back on static credentials file at: $staticCredentialsFile"
+      dockerCredentialsFile=$staticCredentialsFile
+    fi
+
+    if [ -f "$dockerCredentialsFile" ]; then
+      echo "using credentials from $dockerCredentialsFile"
+
+      CREDSFILE=$(cat "$dockerCredentialsFile")
+      creds[token]=$(${awk} -F'=' '/DOCKER_TOKEN/ {print $2}' <<< "$CREDSFILE" | head -n1)
+
+      # Prefer DOCKER_TOKEN over the username and password
+      # authentication method
+      if [ -z "''${creds[token]}" ]; then
+        creds[user]=$(${awk} -F'=' '/DOCKER_USER/  {print $2}' <<< "$CREDSFILE" | head -n1)
+        creds[pass]=$(${awk} -F'=' '/DOCKER_PASS/  {print $2}' <<< "$CREDSFILE" | head -n1)
+      fi
+    fi
+
+    # These variables will be filled in first by the impureEnvVars, if
+    # those variables are empty then they will default to the
+    # credentials that may have been read in from the 'DOCKER_CREDENTIALS'
+    DOCKER_USER="''${DOCKER_USER:-''${creds[user]}}"
+    DOCKER_PASS="''${DOCKER_PASS:-''${creds[pass]}}"
+    DOCKER_TOKEN="''${DOCKER_TOKEN:-''${creds[token]}}"
+
+    ${fetcher} --out="$out" \
+      ''${registry:+--registry "$registry"} \
+      ''${DOCKER_USER:+--username "$DOCKER_USER"} \
+      ''${DOCKER_PASS:+--password "$DOCKER_PASS"} \
+      ''${DOCKER_TOKEN:+--token "$DOCKER_TOKEN"} \
+      ${layerDigestFlag} \
+      "${repository}/${imageName}" \
+      "${tag}"
+  '';
+
+  buildInputs = [ haskellPackages.hocker ];
+
+  outputHashAlgo = "sha256";
+  outputHashMode = "flat";
+  outputHash = sha256;
+
+  preferLocalBuild = true;
+
+  impureEnvVars = [
+    "DOCKER_USER"
+    "DOCKER_PASS"
+    "DOCKER_TOKEN"
+  ];
+
+  inherit registry dockerCredentialsFile;
+}