Browse Source

[DEVOPS-1222] Build with iohk/nix and nix-tools.

avieth/libyaml_bound
Jean-Baptiste Giraudeau 3 years ago
parent
commit
f014f27510
No known key found for this signature in database
GPG Key ID: DDF3944950267CD1
  1. 12
      .buildkite/pipeline.yml
  2. 7
      .gitattributes
  3. 1
      .stack-to-nix.cache
  4. 8
      bors.toml
  5. 31
      ci/check-hydra.sh
  6. 20
      ci/check-stack2nix.sh
  7. 50
      default.nix
  8. 26
      iohk-monitoring.nix
  9. 16
      nix/.stack-pkgs.nix
  10. 115
      nix/.stack.nix/iohk-monitoring.nix
  11. 1
      nix/config.nix
  12. 14
      nix/iohk-common.nix
  13. 7
      nix/iohk-nix-src.json
  14. 70
      nix/pkgs.nix
  15. 5
      nix/regenerate.sh
  16. 9
      nix/update-iohk-nix.sh
  17. 71
      release.nix
  18. 10
      stack.yaml

12
.buildkite/pipeline.yml

@ -6,9 +6,11 @@ steps:
artifact_paths: "docs/IOHK-Monitoring.pdf ; docs/IOHK-Monitoring.log"
timeout: 5
- label: "nix build"
command: "nix-build release.nix"
- label: 'check-hydra'
command: 'ci/check-hydra.sh'
agents:
"system=x86_64-linux"
artifact_paths: "**/*-tests.log"
timeout: 15
system: x86_64-linux
- label: 'stack2nix'
command: 'ci/check-stack2nix.sh'
agents:
system: x86_64-linux

7
.gitattributes vendored

@ -0,0 +1,7 @@
# GitHub Linguist annotations.
# Hide nix/.stack.nix/*.nix
# That is stuff that is generated by nix-tools stack-to-nix
nix/.stack.nix/*.nix linguist-generated=true
.stack-to-nix.cache linguist-generated=true
nix/.stack-pkgs.nix linguist-generated=true

1
.stack-to-nix.cache generated

@ -0,0 +1 @@
https://git.haskell.org/packages/transformers.git 80557845cdc0e72bc05cec19cf7a1bf5495e9e69 . 1lnqvghdmvcndax9lm8mg99s65qfvx1w72infx5930br60m82ir1 transformers .stack.nix/transformers.nix

8
bors.toml

@ -0,0 +1,8 @@
status = [
"buildkite/iohk-monitoring-framework",
"ci/hydra:Cardano:iohk-monitoring:required",
]
timeout_sec = 7200
required_approvals = 1
block_labels = [ "WIP", "DO NOT MERGE" ]
delete_merged_branches = true

31
ci/check-hydra.sh

@ -0,0 +1,31 @@
#!/usr/bin/env nix-shell
#!nix-shell -p jq hydra -i bash -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/09195057114a0a8d112c847a9a8f52957420857d.tar.gz
echo '~~~ Evaluating release.nix'
command time --format '%e' -o eval-time.txt \
hydra-eval-jobs \
--option allowed-uris "https://github.com/NixOS https://github.com/input-output-hk" \
-I . release.nix \
--arg supportedSystems '["x86_64-linux"]' > eval.json
EVAL_EXIT_CODE="$?"
if [ "$EVAL_EXIT_CODE" != 0 ]
then
rm eval.json eval-time.txt
echo -e "\\e[31;1mERROR: Failed to evaluate release.nix\\e[0m"
exit 1
fi
EVAL_TIME=$(cat eval-time.txt)
jq . < eval.json
ERRORS=$(jq -r 'map_values(.error)|to_entries[]|select(.value)|@text "\(.key): \(.value)"' < eval.json)
NUM_ERRORS=$(jq -r '[ map_values(.error)|to_entries[]|select(.value) ] |length' < eval.json)
rm eval.json eval-time.txt
if [ "$NUM_ERRORS" != 0 ]
then
echo -e "\\e[31;1mERROR: evaluation completed in $EVAL_TIME seconds with $NUM_ERRORS errors\\e[0m"
echo "$ERRORS"
exit 1
else
echo -e "\\e[32;1mOK: evaluation completed in $EVAL_TIME seconds with no errors\\e[0m"
exit 0
fi

20
ci/check-stack2nix.sh

@ -0,0 +1,20 @@
#!/usr/bin/env nix-shell
#!nix-shell -i bash -p git bash nixStable
# Check and warn if stack-to-nix auto-generated code is out of date.
set -xe
fail_stack2nix_check() {
git diff -w --text > /tmp/stack2nix.patch
buildkite-agent artifact upload /tmp/stack2nix.patch --job "$BUILDKITE_JOB_ID"
echo "ERROR: you need to (run ./nix/regenerate.sh or apply the patch in the buildkite artifact) and commit the changes" >&2
exit 1
}
# Get relative path to script directory
scriptDir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
"${scriptDir}/../nix/regenerate.sh"
git diff -w --text --exit-code || fail_stack2nix_check

50
default.nix

@ -0,0 +1,50 @@
#
# The defaul.nix file. This will generate targets for all
# buildables (see release.nix for nomenclature, excluding
# the "build machine" last part, specific to release.nix), eg.:
#
# - nix build -f default.nix nix-tools.tests.iohk-monitoring # All `iohk-monitoring` tests
# - nix build -f default.nix nix-tools.tests.iohk-monitoring.tests
# - nix build -f default.nix nix-tools.exes.iohk-monitoring # All `iohk-monitoring` executables
# - nix build -f default.nix nix-tools.cexes.iohk-monitoring.example-simple
#
# Generated targets include anything from stack.yaml (via
# nix-tools:stack-to-nix and the nix/regenerate.sh script)
# or cabal.project (via nix-tools:plan-to-nix), including all
# version overrides specified there.
#
# Nix-tools stack-to-nix will generate the `nix/.stack-pkgs.nix`
# file which is imported from the `nix/pkgs.nix` where further
# customizations outside of the ones in stack.yaml/cabal.project
# can be specified as needed for nix/ci.
#
# Please run `nix/regenerate.sh` after modifying stack.yaml
# or relevant part of cabal configuration files.
# When switching to recent stackage or hackage package version,
# you might also need to update the iohk-nix common lib. You
# can do so by running the `nix/update-iohk-nix.sh` script.
#
# More information about iohk-nix and nix-tools is available at:
# https://github.com/input-output-hk/iohk-nix/blob/master/docs/nix-toolification.org#for-a-stackage-project
#
# We will need to import the iohk-nix common lib, which includes
# the nix-tools tooling.
let
commonLib = import ./nix/iohk-common.nix;
in
# This file needs to export a function that takes
# the arguments it is passed and forwards them to
# the default-nix template from iohk-nix. This is
# important so that the release.nix file can properly
# parameterize this file when targetting different
# hosts.
{ ... }@args:
# We will instantiate the default-nix template with the
# nix/pkgs.nix file...
commonLib.nix-tools.default-nix ./nix/pkgs.nix args
# ... and add additional non-haskell packages we want to build on CI:
// {
}

26
iohk-monitoring.nix

@ -1,26 +0,0 @@
{ mkDerivation, aeson, Cabal, array, async, auto-update, base, bytestring
, clock, containers, contravariant, directory, download, ekg, ekg-core
, filepath, katip, lens, mtl, process, QuickCheck, random
, safe-exceptions, semigroups, stdenv, stm, tasty, tasty-hunit
, tasty-quickcheck, template-haskell, text, time, time-units
, transformers, unix, unordered-containers, void, yaml
}:
mkDerivation {
pname = "iohk-monitoring";
version = "0.1.0.0";
src = ./.;
libraryHaskellDepends = [
aeson Cabal array async auto-update base bytestring clock containers
contravariant download directory ekg ekg-core filepath katip lens mtl
safe-exceptions stm template-haskell text time time-units
transformers unix unordered-containers yaml
];
testHaskellDepends = [
Cabal array base bytestring clock containers mtl process QuickCheck
random semigroups stm tasty tasty-hunit tasty-quickcheck text time
time-units transformers void
];
description = "logging, benchmarking and monitoring framework";
license = stdenv.lib.licenses.mit; #unfree;
hydraPlatforms = stdenv.lib.platforms.none;
}

16
nix/.stack-pkgs.nix generated

@ -0,0 +1,16 @@
{
overlay = hackage:
{
packages = {
"time-units" = (((hackage.time-units)."1.0.0").revisions).default;
"ekg" = (((hackage.ekg)."0.4.0.15").revisions).default;
"ekg-json" = (((hackage.ekg-json)."0.1.0.6").revisions).default;
"process" = (((hackage.process)."1.6.5.0").revisions).default;
"transformers" = (((hackage.transformers)."0.5.6.2").revisions).default;
} // { iohk-monitoring = ./.stack.nix/iohk-monitoring.nix; };
compiler.version = "8.6.3";
compiler.nix-name = "ghc863";
};
resolver = "lts-13.6";
compiler = "ghc-8.6.3";
}

115
nix/.stack.nix/iohk-monitoring.nix generated

@ -0,0 +1,115 @@
{ system, compiler, flags, pkgs, hsPkgs, pkgconfPkgs, ... }:
{
flags = {};
package = {
specVersion = "1.10";
identifier = { name = "iohk-monitoring"; version = "0.1.1.0"; };
license = "MIT";
copyright = "2018 IOHK";
maintainer = "";
author = "Alexander Diemand, Andreas Triantafyllos";
homepage = "";
url = "";
synopsis = "logging, benchmarking and monitoring framework";
description = "";
buildType = "Simple";
};
components = {
"library" = {
depends = [
(hsPkgs.base)
(hsPkgs.aeson)
(hsPkgs.array)
(hsPkgs.async)
(hsPkgs.attoparsec)
(hsPkgs.auto-update)
(hsPkgs.bytestring)
(hsPkgs.clock)
(hsPkgs.containers)
(hsPkgs.contravariant)
(hsPkgs.directory)
(hsPkgs.ekg)
(hsPkgs.ekg-core)
(hsPkgs.filepath)
(hsPkgs.katip)
(hsPkgs.lens)
(hsPkgs.mtl)
(hsPkgs.safe-exceptions)
(hsPkgs.scientific)
(hsPkgs.stm)
(hsPkgs.template-haskell)
(hsPkgs.text)
(hsPkgs.time)
(hsPkgs.time-units)
(hsPkgs.transformers)
(hsPkgs.unordered-containers)
(hsPkgs.vector)
(hsPkgs.yaml)
] ++ (if system.isWindows
then [ (hsPkgs.Win32) ]
else [ (hsPkgs.unix) ]);
};
exes = {
"example-simple" = {
depends = [
(hsPkgs.base)
(hsPkgs.iohk-monitoring)
(hsPkgs.async)
(hsPkgs.bytestring)
(hsPkgs.mtl)
] ++ (if system.isWindows
then [ (hsPkgs.Win32) ]
else [ (hsPkgs.unix) ]);
};
"example-complex" = {
depends = ([
(hsPkgs.base)
(hsPkgs.iohk-monitoring)
(hsPkgs.async)
(hsPkgs.bytestring)
(hsPkgs.mtl)
(hsPkgs.random)
(hsPkgs.text)
] ++ (if system.isWindows
then [ (hsPkgs.Win32) ]
else [
(hsPkgs.unix)
])) ++ (pkgs.lib).optional (system.isLinux) (hsPkgs.download);
};
};
tests = {
"tests" = {
depends = [
(hsPkgs.base)
(hsPkgs.iohk-monitoring)
(hsPkgs.aeson)
(hsPkgs.array)
(hsPkgs.async)
(hsPkgs.bytestring)
(hsPkgs.clock)
(hsPkgs.containers)
(hsPkgs.directory)
(hsPkgs.filepath)
(hsPkgs.mtl)
(hsPkgs.process)
(hsPkgs.QuickCheck)
(hsPkgs.random)
(hsPkgs.semigroups)
(hsPkgs.split)
(hsPkgs.stm)
(hsPkgs.tasty)
(hsPkgs.tasty-hunit)
(hsPkgs.tasty-quickcheck)
(hsPkgs.text)
(hsPkgs.time)
(hsPkgs.time-units)
(hsPkgs.transformers)
(hsPkgs.unordered-containers)
(hsPkgs.vector)
(hsPkgs.void)
(hsPkgs.yaml)
];
};
};
};
} // rec { src = (pkgs.lib).mkDefault .././../.; }

1
nix/config.nix

@ -0,0 +1 @@
{}

14
nix/iohk-common.nix

@ -0,0 +1,14 @@
# Imports the iohk-nix library.
# The version can be overridden for debugging purposes by setting
# NIX_PATH=iohk_nix=/path/to/iohk-nix
import (
let try = builtins.tryEval <iohk_nix>;
in if try.success
then builtins.trace "using host <iohk_nix>" try.value
else
let
spec = builtins.fromJSON (builtins.readFile ./iohk-nix-src.json);
in builtins.fetchTarball {
url = "${spec.url}/archive/${spec.rev}.tar.gz";
inherit (spec) sha256;
}) {}

7
nix/iohk-nix-src.json

@ -0,0 +1,7 @@
{
"url": "https://github.com/input-output-hk/iohk-nix",
"rev": "fdf62bbe2a97f3bbd7ece6d7d6df391ee2c09a11",
"date": "2019-02-12T11:24:55+01:00",
"sha256": "0wj18578xrm3ydnnnhc4bfgfwxh6vnvny8zq645dcjnpg60w61by",
"fetchSubmodules": false
}

70
nix/pkgs.nix

@ -0,0 +1,70 @@
{ pkgs ? import <nixpkgs> {}
, iohk-overlay ? {}
, iohk-module ? {}
, haskell
, hackage
, stackage
, ...
}:
let
# our packages
stack-pkgs = import ./.stack-pkgs.nix;
# packages which will require TH and thus
# will need -fexternal-interpreter treatment
# when cross compiling.
th-packages = [
"aeson"
"bifunctors"
"exceptions"
"free"
"invariant"
"iohk-monitoring"
"katip"
"lens"
"microlens-th"
"reflection"
"semigroupoids"
"tagged"
"th-abstraction"
"yaml"
];
# Build the packageset with module support.
# We can essentially override anything in the modules
# section.
#
# packages.cbors.patches = [ ./one.patch ];
# packages.cbors.flags.optimize-gmp = false;
#
compiler = (stack-pkgs.overlay hackage).compiler.nix-name;
pkgSet = haskell.mkNewPkgSet {
inherit pkgs;
pkg-def = stackage.${stack-pkgs.resolver};
pkg-def-overlays = [
stack-pkgs.overlay
iohk-overlay.${compiler}
];
modules = [
haskell.ghcHackagePatches.${compiler}
(iohk-module { nixpkgs = pkgs;
inherit th-packages; })
{
# Packages we wish to ignore version bounds of.
# This is similar to jailbreakCabal, however it
# does not require any messing with cabal files.
packages.katip.components.library.doExactConfig = true;
packages.iohk-monitoring.components.tests.tests.setupTestFlags = [
"--show-details=streaming"
];
}
];
};
in
pkgSet.config.hsPkgs // { _config = pkgSet.config; }

5
nix/regenerate.sh

@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -euo pipefail
exec $(nix-build `dirname $0`/iohk-common.nix -A nix-tools.regeneratePackages --no-out-link)

9
nix/update-iohk-nix.sh

@ -0,0 +1,9 @@
#!/usr/bin/env nix-shell
#!nix-shell -i bash -p nix-prefetch-git
set -euo pipefail
NIX_DIR=`dirname $0`
nix-prefetch-git https://github.com/input-output-hk/iohk-nix \
> $NIX_DIR/iohk-nix-src.json

71
release.nix

@ -1,24 +1,55 @@
let
addRealTimeTestLogs = drv: pkgs.haskell.lib.overrideCabal drv (attrs: {
testTarget = "--show-details=streaming";
});
commonLib = import ./nix/iohk-common.nix;
in
commonLib.nix-tools.release-nix {
package-set-path = ./.;
config = {
packageOverrides = pkgs: rec {
haskellPackages = pkgs.haskell.packages.ghc863.override {
overrides = haskellPackagesNew: haskellPackagesOld: rec {
iohk-monitoring = pkgs.haskell.lib.dontHaddock (haskellPackagesNew.callPackage ./iohk-monitoring.nix { });
};
};
};
};
# packages from our stack.yaml or plan file (via nix/pkgs.nix) we
# are interested in building on CI via nix-tools.
packages = [ "iohk-monitoring" ];
nixpkgs = builtins.fetchTarball {
url = "https://github.com/NixOS/nixpkgs/archive/6054276a8e6e877f8846c79b0779e3310c495a6b.tar.gz";
sha256 = "11mv8an4zikh2hybn11znqcbxazqq32byvvvaymy2xkpww2jnkxp";
};
# The set of jobs we consider crutial for each CI run.
# if a single one of these fails, the build will be marked
# as failed.
#
# The names can be looked up on hydra when in doubt.
#
# custom jobs will follow their name as set forth in
# other-packages.
#
# nix-tools packages are prefixed with `nix-tools` and
# follow the following naming convention:
#
# namespace optional cross compilation prefix build machine
# .-------. .-----------------. .--------------------------.
# nix-tools.{libs,exes,tests,benchmarks}.{x86_64-pc-mingw-,}$pkg.$component.{x86_64-linux,x86_64-darwin}
# '--------------------------' '-------------'
# component type cabal pkg and component*
#
# * note that for `libs`, $component is empty, as cabal only
# provides a single library for packages right now.
# * note that for `exes`, $component is also empty, because it
# it provides all exes under a single result directory.
# To specify a single executable component to build, use
# `cexes` as component type.
#
# Example:
#
# nix-tools.libs.iohk-monitoring.x86_64-darwin -- will build the iohk-monitoring library on and for macOS
# nix-tools.libs.x86_64-pc-mingw32-iohk-monitoring.x86_64-linux -- will build the iohk-monitoring library on linux for windows.
# nix-tools.tests.iohk-monitoring.tests.x86_64-linux -- will build and run the tests from the
# iohk-monitoring package on linux.
pkgs = import nixpkgs { inherit config; };
in
{ iohk-monitoring = addRealTimeTestLogs pkgs.haskellPackages.iohk-monitoring;
}
# The required jobs that must pass for ci not to fail:
required-name = "iohk-monitoring-required-checks";
required-targets = jobs: [
# targets are specified using above nomenclature:
jobs.nix-tools.tests.iohk-monitoring.tests.x86_64-linux
jobs.nix-tools.exes.iohk-monitoring.x86_64-linux
# Disabled due to: https://github.com/psibi/download/issues/17:
#jobs.nix-tools.exes.x86_64-pc-mingw32-iohk-monitoring.x86_64-linux
];
}

10
stack.yaml

@ -0,0 +1,10 @@
resolver: lts-13.6
compiler: ghc-8.6.3
packages:
- .
extra-deps:
- time-units-1.0.0
- ekg-0.4.0.15
- ekg-json-0.1.0.6
- process-1.6.5.0
- transformers-0.5.6.2
Loading…
Cancel
Save