Browse Source

Remove all legacy (cardano-sl) related code.

master
Jean-Baptiste Giraudeau 2 years ago
parent
commit
9ca7dc7023
No known key found for this signature in database GPG Key ID: 7CEF8C9CC2D9933B
  1. 77
      clusters/cardano.nix
  2. 12
      deployments/cardano-aws.nix
  3. 2
      globals-bench-txgen-simple.nix
  4. 5
      globals-defaults.nix
  5. 1
      globals-mainnet-candidate-2.nix
  6. 1
      globals-mainnet-candidate-3.nix
  7. 1
      globals-mainnet-candidate-4.nix
  8. 1
      globals-mainnet-candidate.nix
  9. 2
      globals-shelley-dev.nix
  10. 1
      globals-shelley-qa.nix
  11. 1
      globals-shelley-testnet.nix
  12. 95
      modules/base-legacy-service.nix
  13. 171
      modules/cardano-explorer-python.nix
  14. 89
      modules/common-cardano-legacy.nix
  15. 1383
      modules/grafana/cardano/cardano-application-dashboard.json
  16. 4703
      modules/grafana/cardano/cardano-application-grouped-dashboard.json
  17. 557
      modules/grafana/cardano/explorer-python-api.json
  18. 42
      modules/monitoring-cardano.nix
  19. 11
      nix/cardano.nix
  20. 2
      nix/packages.nix
  21. 60
      nix/sources.json
  22. 11
      physical/aws/security-groups/allow-legacy-peers.nix
  23. 4
      physical/aws/security-groups/allow-legacy-public.nix
  24. 80
      roles/byron-proxy.nix
  25. 113
      roles/explorer-legacy.nix
  26. 25
      roles/legacy-core.nix
  27. 10
      roles/legacy-relay.nix
  28. 19
      roles/sync-nonlegacy-chain-state.nix
  29. 4
      shell.nix
  30. 6
      topologies/bench-txgen-distrib-12.nix
  31. 6
      topologies/bench-txgen-distrib-3.nix
  32. 6
      topologies/bench-txgen-distrib-6.nix
  33. 6
      topologies/bench-txgen-distrib-9.nix
  34. 4
      topologies/mainnet-candidate-2.nix
  35. 4
      topologies/mainnet-candidate-3.nix
  36. 4
      topologies/mainnet-candidate-4.nix
  37. 4
      topologies/mainnet-candidate.nix
  38. 4
      topologies/mainnet.nix
  39. 6
      topologies/shelley-dev.nix
  40. 6
      topologies/shelley-qa.nix
  41. 7
      topologies/shelley-testnet.nix
  42. 6
      topologies/staging-shelley.nix
  43. 4
      topologies/staging.nix
  44. 6
      topologies/testnet.nix

77
clusters/cardano.nix

@ -15,19 +15,13 @@ with pkgs;
let
inherit (globals) topology byronProxyPort;
inherit (topology) legacyCoreNodes legacyRelayNodes byronProxies coreNodes relayNodes;
inherit (topology) coreNodes relayNodes;
privateRelayNodes = topology.privateRelayNodes or [];
inherit (lib) recursiveUpdate mapAttrs listToAttrs imap1 concatLists;
# for now, keys need to be generated for each core nodes with:
# for i in {1..2}; do cardano-cli --byron-legacy keygen --secret ./keys/$i.sk --no-password; done
cardanoNodes = listToAttrs (concatLists [
(map mkLegacyCoreNode legacyCoreNodes)
(map mkLegacyRelayNode legacyRelayNodes)
(map mkCoreNode coreNodes)
(map mkRelayNode (relayNodes ++ privateRelayNodes))
(map mkByronProxyNode byronProxies)
(map mkTestNode (topology.testNodes or []))
]);
@ -65,17 +59,7 @@ let
labels = { alias = "cardano-graphql-exporter"; };
}];
}
])) ++ (lib.optional globals.withLegacyExplorer (
# TODO: remove once explorer python api is deprecated
{
job_name = "explorer-python-api";
scrape_interval = "10s";
metrics_path = "/metrics/explorer-python-api";
static_configs = [{
targets = [ "explorer-ip" ];
labels = { alias = "explorer-python-api"; };
}];
})) ++ (lib.optional globals.withFaucet (
])) ++ (lib.optional globals.withFaucet (
{
job_name = "cardano-faucet";
scrape_interval = "10s";
@ -107,9 +91,7 @@ let
imports = [
(if globals.withHighCapacityExplorer then c5-4xlarge else xlarge)
cardano-ops.roles.explorer
]
# TODO: remove module when the new explorer is available
++ lib.optional (globals.withLegacyExplorer) cardano-ops.roles.explorer-legacy;
];
services.monitoring-exporters.extraPrometheusExportersPorts =
[ globals.cardanoNodePrometheusExporterPort ];
@ -196,59 +178,6 @@ let
} def;
};
mkByronProxyNode = def: {
inherit (def) name;
value = mkNode {
node = {
roles.isByronProxy = true;
inherit (def) org nodeId;
};
services.byron-proxy = {
inherit (def) producers;
};
deployment.ec2.region = def.region;
imports = [
medium
cardano-ops.roles.byron-proxy
];
services.cardano-node-legacy.staticRoutes = def.staticRoutes or [];
services.cardano-node-legacy.dynamicSubscribe = def.dynamicSubscribe or [];
services.monitoring-exporters.extraPrometheusExportersPorts = [ globals.byronProxyPrometheusExporterPort ];
} def;
};
mkLegacyCoreNode = def: {
inherit (def) name;
value = mkNode {
node = {
roles.isCardanoLegacyCore = true;
inherit (def) org nodeId;
};
deployment.ec2.region = def.region;
imports = [ medium cardano-ops.roles.legacy-core ];
# Temporary for legacy migration:
#imports = [ medium cardano-ops.roles.legacy-core
# cardano-ops.roles.sync-nonlegacy-chain-state ];
services.cardano-node-legacy.staticRoutes = def.staticRoutes;
} def;
};
mkLegacyRelayNode = def: {
inherit (def) name;
value = mkNode {
node = {
roles.isCardanoLegacyRelay = true;
inherit (def) org;
};
deployment.ec2.region = def.region;
imports = [ medium cardano-ops.roles.legacy-relay ];
services.cardano-node-legacy.staticRoutes = def.staticRoutes or [];
services.cardano-node-legacy.dynamicSubscribe = def.dynamicSubscribe or [];
} def;
};
# Load client with optimized NVME disks, for prometheus monitored clients syncs
mkTestNode = def: {
inherit (def) name;

12
deployments/cardano-aws.nix

@ -5,7 +5,7 @@ let
mapAttrs' mapAttrs nameValuePair recursiveUpdate unique optional any concatMap
getAttrs optionalString hasPrefix;
inherit (globals.topology) legacyCoreNodes legacyRelayNodes byronProxies coreNodes relayNodes;
inherit (globals.topology) coreNodes relayNodes;
privateRelayNodes = globals.topology.privateRelayNodes or [];
inherit (globals.ec2.credentials) accessKeyIds;
inherit (iohk-ops-lib.physical) aws;
@ -38,15 +38,7 @@ let
securityGroups = with aws.security-groups; [
{
nodes = getAttrs (map (n: n.name) (legacyCoreNodes ++ byronProxies)) nodes;
groups = [ (import ../physical/aws/security-groups/allow-legacy-peers.nix) ];
}
{
nodes = getAttrs (map (n: n.name) legacyRelayNodes) nodes;
groups = [ (import ../physical/aws/security-groups/allow-legacy-public.nix) ];
}
{
nodes = getAttrs (map (n: n.name) (coreNodes ++ byronProxies ++ privateRelayNodes)) nodes;
nodes = getAttrs (map (n: n.name) (coreNodes ++ privateRelayNodes)) nodes;
groups = [ (import ../physical/aws/security-groups/allow-peers.nix) ];
}
{

2
globals-bench-txgen-simple.nix

@ -110,7 +110,6 @@ in reportDeployment (rec {
withMonitoring = false;
withExplorer = true;
withLegacyExplorer = false;
environmentName = "bench-txgen-${benchmarkingParams.meta.topology}-${benchmarkingProfileName}";
@ -119,7 +118,6 @@ in reportDeployment (rec {
environmentConfig = rec {
relays = "relays.${pkgs.globals.domain}";
edgePort = pkgs.globals.cardanoNodePort;
confKey = abort "legacy nodes not supported by benchmarking environment";
genesisFile = ./keys/genesis.json;
private = true;
networkConfig = envConfigBase.networkConfig // {

5
globals-defaults.nix

@ -26,7 +26,6 @@ in {
withMonitoring = true;
withExplorer = true;
withLegacyExplorer = false;
withCardanoDBExtended = true;
withSubmitApi = false;
withFaucet = false;
@ -48,17 +47,13 @@ in {
deployerIp = requireEnv "DEPLOYER_IP";
cardanoNodePort = 3001;
cardanoNodeLegacyPort = 3000;
cardanoNodePrometheusExporterPort = 12798;
byronProxyPrometheusExporterPort = 12799;
cardanoExplorerPrometheusExporterPort = 8080;
cardanoExplorerPythonApiPrometheusExporterPort = 7001;
netdataExporterPort = 19999;
extraPrometheusExportersPorts = [
pkgs.globals.cardanoNodePrometheusExporterPort
pkgs.globals.byronProxyPrometheusExporterPort
pkgs.globals.cardanoExplorerPrometheusExporterPort
pkgs.globals.netdataExporterPort
];

1
globals-mainnet-candidate-2.nix

@ -7,7 +7,6 @@ pkgs: {
topology = import ./topologies/mainnet-candidate-2.nix pkgs;
withExplorer = true;
withLegacyExplorer = false;
withHighLoadRelays = true;
withSmash = true;
withSubmitApi = true;

1
globals-mainnet-candidate-3.nix

@ -7,7 +7,6 @@ pkgs: {
topology = import ./topologies/mainnet-candidate-3.nix pkgs;
withExplorer = true;
withLegacyExplorer = false;
withHighLoadRelays = true;
withSmash = true;
withSubmitApi = true;

1
globals-mainnet-candidate-4.nix

@ -7,7 +7,6 @@ pkgs: {
topology = import ./topologies/mainnet-candidate-4.nix pkgs;
withExplorer = true;
withLegacyExplorer = false;
withHighLoadRelays = true;
withSmash = true;
withSubmitApi = true;

1
globals-mainnet-candidate.nix

@ -7,7 +7,6 @@ pkgs: {
topology = import ./topologies/mainnet-candidate.nix pkgs;
withExplorer = true;
withLegacyExplorer = false;
withHighLoadRelays = true;
withSmash = true;
withSubmitApi = true;

2
globals-shelley-dev.nix

@ -1,7 +1,6 @@
pkgs: rec {
withMonitoring = false;
withLegacyExplorer = false;
environmentName = "shelley-dev";
@ -13,7 +12,6 @@ pkgs: rec {
"52.58.137.138"
];
edgePort = pkgs.globals.cardanoNodePort;
confKey = abort "legacy nodes not supported by shelley-dev environment";
genesisFile = ./keys/genesis.json;
genesisHash = builtins.replaceStrings ["\n"] [""] (builtins.readFile ./keys/GENHASH);
private = true;

1
globals-shelley-qa.nix

@ -8,7 +8,6 @@ pkgs: {
withFaucet = true;
withExplorer = true;
withLegacyExplorer = false;
withCardanoDBExtended = false;
withSubmitApi = true;
faucetHostname = "faucet";

1
globals-shelley-testnet.nix

@ -7,7 +7,6 @@ pkgs: {
topology = import ./topologies/shelley-testnet.nix pkgs;
withExplorer = true;
withLegacyExplorer = false;
withHighLoadRelays = true;
withSmash = true;

95
modules/base-legacy-service.nix

@ -1,95 +0,0 @@
pkgs: { name, nodes, config, options, resources, ... }:
with pkgs; with lib;
let
inherit (iohkNix.cardanoLib) cardanoConfig;
cfg = config.services.cardano-node-legacy;
stateDir = "/var/lib/cardano-node";
port = globals.cardanoNodeLegacyPort;
command = toString ([
cfg.executable
"--address ${cfg.publicIp}:${toString port}"
"--listen ${cfg.listenIp}:${toString port}"
(optionalString cfg.jsonLog "--json-log ${stateDir}/jsonLog.json")
(optionalString (config.services.monitoring-exporters.metrics) "--metrics +RTS -T -RTS --statsd-server 127.0.0.1:${toString config.services.monitoring-exporters.statsdPort}")
(optionalString (cfg.nodeType == "core") "--keyfile ${stateDir}/key.sk")
(optionalString (cfg.assetLockFile != null) "--asset-lock-file ${cfg.assetLockFile}")
"--log-config ${cardano-node-legacy-config}/log-configs/cluster.yaml"
"--logs-prefix /var/lib/cardano-node"
"--db-path ${stateDir}/node-db"
"--configuration-file ${cardanoConfig}/configuration.yaml"
"--configuration-key ${globals.environmentConfig.confKey}"
"--topology ${cfg.topologyYaml}"
"--node-id ${name}"
] ++ cfg.extraCommandArgs);
in {
imports = [
cardano-ops.modules.common-cardano-legacy
];
options = {
services.cardano-node-legacy = {
enable = mkEnableOption "cardano-node-legacy" // { default = true; };
extraCommandArgs = mkOption { type = types.listOf types.str; default = []; };
saveCoreDumps = mkOption {
type = types.bool;
default = true;
description = "automatically save coredumps when cardano-node segfaults";
};
executable = mkOption {
type = types.str;
description = "Executable to run as the daemon.";
default = "${cardano-node-legacy}/bin/cardano-node-simple";
};
autoStart = mkOption { type = types.bool; default = true; };
jsonLog = mkOption { type = types.bool; default = false; };
};
};
config = mkIf cfg.enable {
users = {
users.cardano-node = {
uid = 10014;
description = "cardano-node server user";
group = "cardano-node";
home = stateDir;
createHome = true;
extraGroups = [ "keys" ];
};
groups.cardano-node = {
gid = 123123;
};
};
systemd.services.cardano-node-legacy = {
description = "cardano node legacy service";
after = [ "network.target" ];
wantedBy = optionals cfg.autoStart [ "multi-user.target" ];
script = ''
[ -f /var/lib/keys/cardano-node ] && cp -f /var/lib/keys/cardano-node ${stateDir}/key.sk
${optionalString (cfg.saveCoreDumps) ''
# only a process with non-zero coresize can coredump (the default is 0)
ulimit -c unlimited
''}
exec ${command}
'';
serviceConfig = {
User = "cardano-node";
Group = "cardano-node";
# Allow a maximum of 5 retries separated by 30 seconds, in total capped by 200s
Restart = "always";
RestartSec = 30;
StartLimitInterval = 200;
StartLimitBurst = 5;
KillSignal = "SIGINT";
WorkingDirectory = stateDir;
PrivateTmp = true;
Type = "notify";
MemoryMax = "3.5G";
};
};
};
}

171
modules/cardano-explorer-python.nix

@ -1,171 +0,0 @@
pkgs: { config, ... }:
with pkgs;
let
inherit (lib) mkForce mkIf mkEnableOption mkOption types;
explorerPythonAPI = cardano-sl-pkgs.explorerPythonAPI;
cfg = config.services.explorer-python-api;
in {
options = {
services.explorer-python-api = {
enable = mkEnableOption "Explorer Python API";
epochSlots = mkOption {
description = "The number of slots per epoch. 0 < EPOCHSLOTS <= 21600";
type = types.ints.positive;
default = 21600;
};
addrMaxLen = mkOption {
description = "The maximum address length: 200 <= ADDRMAXLEN <= 8000";
type = types.ints.positive;
default = 200;
};
legacyProxyPort = mkOption {
description = "The legacy explorer port to proxy to; typically 8100 or 8101.";
type = types.ints.positive;
default = 8101;
};
pythonApiProxyPort = mkOption {
description = "The python explorer API port to proxy to; typically 7000.";
type = types.ints.positive;
default = 7000;
};
pythonApiMetricsPort = mkOption {
description = "The python explorer API prometheus metrics port; typically 7001.";
type = types.ints.positive;
default = 7001;
};
};
};
config = mkIf cfg.enable {
services.nginx = mkIf config.services.nginx.enable {
virtualHosts = {
"${globals.explorerHostName}.${globals.domain}" = mkForce {
locations = if (globals.initialPythonExplorerDBSyncDone) then {
# Pass to python API once the initial DB dump sync has completed
"/api/addresses/summary/".proxyPass = "http://127.0.0.1:${toString cfg.pythonApiProxyPort}";
} else {
# Otherwise use the main explorer API
"/api/addresses/summary/".proxyPass = "http://127.0.0.1:${toString cfg.legacyProxyPort}";
};
};
"explorer-ip" = {
locations = {
"/metrics/explorer-python-api" = {
proxyPass = "http://127.0.0.1:${toString cfg.pythonApiMetricsPort}/";
};
};
};
};
};
networking.firewall.allowedTCPPorts = [ cfg.pythonApiProxyPort cfg.pythonApiMetricsPort ];
users.users.explorer-python-api = {
home = "/var/empty";
isSystemUser = true;
};
systemd.services.explorer-python-api = {
wantedBy = [ "multi-user.target" ];
environment = {
DBSOCKPATH = config.services.cardano-postgres.postgresqlSocketPath;
prometheus_multiproc_dir = "/tmp/explorer-python-metrics";
ADDRMAXLEN = builtins.toString cfg.addrMaxLen;
EXPLORERURL = "http://localhost:${toString cfg.legacyProxyPort}";
};
preStart = "sleep 5";
script = "exec ${explorerPythonAPI}/bin/run-explorer-python-api";
serviceConfig = {
User = "explorer-python-api";
Restart = "always";
RestartSec = "30s";
};
};
systemd.services.explorer-python-dumper = {
wantedBy = [ "multi-user.target" ];
environment = {
DBSOCKPATH = config.services.cardano-postgres.postgresqlSocketPath;
EPOCHSLOTS = "${builtins.toString cfg.epochSlots}";
ADDRMAXLEN = "${builtins.toString cfg.addrMaxLen}";
EXPLORERURL = "http://localhost:${toString cfg.legacyProxyPort}";
};
preStart = "sleep 5";
script = "exec ${explorerPythonAPI}/bin/run-explorer-python-dumper";
serviceConfig = {
User = "explorer-python-api";
Restart = "always";
RestartSec = "30s";
};
};
services.cardano-postgres.enable = true;
services.postgresql = {
initialScript = pkgs.writeText "explorerPythonAPI-initScript" ''
create database explorer_python_api;
\connect explorer_python_api;
create schema scraper;
create table scraper.blocks (
cbeBlkHash text primary key
, cbeEpoch smallint
, cbeSlot smallint
, cbeBlkHeight integer
, cbeTimeIssued timestamp without time zone
, cbeTxNum integer
, cbeTotalSent bigint
, cbeSize integer
, cbeBlockLead text
, cbeFees bigint
, cbsPrevHash text
, cbsNextHash text
, cbsMerkleRoot text
);
create index i_blocks_cbeBlkHash on scraper.blocks (cbeBlkHash asc);
create table scraper.tx (
ctsId text primary key
, ctsTxTimeIssued timestamp without time zone
, ctsBlockTimeIssued timestamp without time zone
, ctsBlockHash text
, ctsTotalInput bigint
, ctsTotalOutput bigint
, ctsFees bigint
);
create index i_tx_ctsId on scraper.tx (ctsId asc);
create index i_tx_ctsTxTimeIssued on scraper.tx (ctsTxTimeIssued asc);
create table scraper.txinput (
ctsId text
, ctsIdIndex smallint
, ctsTxTimeIssued timestamp without time zone
, ctsInputAddr text
, ctsInput bigint
, constraint pk_txinput primary key (ctsId, ctsIdIndex)
);
create index i_txinput_ctsId on scraper.txinput (ctsId asc);
create index i_txinput_ctsIdIndex on scraper.txinput (ctsIdIndex asc);
create index i_txinput_ctsTxTimeIssued on scraper.txinput (ctsTxTimeIssued asc);
create index i_txinput_ctsInputAddr_ctsId on scraper.txinput (ctsInputAddr asc, ctsId asc);
create table scraper.txoutput (
ctsId text
, ctsIdIndex smallint
, ctsTxTimeIssued timestamp without time zone
, ctsOutputAddr text
, ctsOutput bigint
, constraint pk_txoutput primary key (ctsId, ctsIdIndex)
);
create index i_txoutput_ctsId on scraper.txoutput (ctsId asc);
create index i_txoutput_ctsIdIndex on scraper.txoutput (ctsIdIndex asc);
create index i_txoutput_ctsTxTimeIssued on scraper.txoutput (ctsTxTimeIssued asc);
create index i_txoutput_ctsOutputAddr_ctsId on scraper.txoutput (ctsOutputAddr asc, ctsId asc);
create user explorer_python_api;
grant all privileges on database explorer_python_api to explorer_python_api;
grant all privileges on schema scraper to explorer_python_api;
grant all privileges on all tables in schema scraper to explorer_python_api;
'';
identMap = ''
explorer-users explorer-python-api explorer_python_api
explorer-users root explorer_python_api
explorer-users postgres postgres
explorer-users cexplorer cexplorer
explorer-users root cexplorer
'';
authentication = ''
local all all ident map=explorer-users
'';
};
};
}

89
modules/common-cardano-legacy.nix

@ -1,89 +0,0 @@
pkgs: { name, nodes, config, options, resources, ... }:
with pkgs; with lib;
let
cfg = config.services.cardano-node-legacy;
port = globals.cardanoNodeLegacyPort;
hostName = name: "${name}.cardano";
cardanoNodes = filterAttrs
(_: node: node.config.services.cardano-node-legacy.enable
or node.config.services.byron-proxy.enable or false)
nodes;
cardanoHostList = lib.mapAttrsToList (nodeName: node: {
name = hostName nodeName;
ip = staticRouteIp nodeName;
}) cardanoNodes;
topology = {
nodes = mapAttrs (name: node: let nodeCfg = node.config.services.cardano-node-legacy; in {
type = nodeCfg.nodeType;
region = node.config.deployment.ec2.region;
host = hostName name;
port = port;
kademlia = false;
} // optionalAttrs (concatLists nodeCfg.staticRoutes != []) {
static-routes = nodeCfg.staticRoutes;
} // optionalAttrs (concatLists nodeCfg.dynamicSubscribe != []) {
dynamic-subscribe = map (map (h: {
"host" = if (nodes ? ${h}) then hostName h else h;
})) nodeCfg.dynamicSubscribe;
}) cardanoNodes;
};
nodeName = node: head (attrNames (filterAttrs (_: n: n == node) nodes));
staticRouteIp = getStaticRouteIp resources nodes;
in {
imports = [
cardano-ops.modules.common
];
options = {
services.cardano-node-legacy = {
listenIp = mkOption { type = types.str; default = getListenIp nodes.${name};};
publicIp = mkOption { type = types.str; default = staticRouteIp name;};
nodeType = mkOption { type = types.enum [ "core" "relay" "edge" ];};
topologyYaml = mkOption {
type = types.path;
default = writeText "topology.yaml" (builtins.toJSON topology);
};
staticRoutes = mkOption {
default = [];
type = types.listOf (types.listOf types.str);
description = ''Static routes to peers.'';
};
dynamicSubscribe = mkOption {
default = [];
type = types.listOf (types.listOf types.str);
description = ''Dnymic subscribe routes.'';
};
assetLockFile = mkOption { type = types.nullOr types.path; default = null; };
};
};
config = {
environment.systemPackages = [ pkgs.telnet ];
networking.firewall = {
allowedTCPPorts = [ port ];
# TODO: securing this depends on CSLA-27
# NOTE: this implicitly blocks DHCPCD, which uses port 68
allowedUDPPortRanges = [ { from = 1024; to = 65000; } ];
};
services.dnsmasq = {
enable = true;
servers = [ "127.0.0.1" ];
};
networking.extraHosts = ''
${concatStringsSep "\n" (map (host: "${host.ip} ${host.name}") cardanoHostList)}
'';
};
}

1383
modules/grafana/cardano/cardano-application-dashboard.json

File diff suppressed because it is too large

4703
modules/grafana/cardano/cardano-application-grouped-dashboard.json

File diff suppressed because it is too large

557
modules/grafana/cardano/explorer-python-api.json

@ -1,557 +0,0 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"id": 7,
"links": [],
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "prometheus",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 7,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "rate(address_summary_hist_count[1h])*3600",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Explorer Python API Hit Rate (Hits per hour)",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": "Address API Hits/hr",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "prometheus",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 0
},
"id": 6,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "/.*status=\"[^2][0-9][0-9].*/",
"yaxis": 2
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "address_summary_hist_sum / address_summary_hist_count",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Explorer Python API Avg Response Time (s)",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": "2XX Avg Response (s)",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": "Non-2XX Avg Response (s)",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"cards": {
"cardPadding": null,
"cardRound": null
},
"color": {
"cardColor": "#73BF69",
"colorScale": "sqrt",
"colorScheme": "interpolateOranges",
"exponent": 0.5,
"min": null,
"mode": "opacity"
},
"dataFormat": "tsbuckets",
"datasource": "prometheus",
"gridPos": {
"h": 11,
"w": 12,
"x": 0,
"y": 8
},
"heatmap": {},
"hideZeroBuckets": false,
"highlightCards": true,
"id": 2,
"interval": "",
"legend": {
"show": true
},
"links": [],
"options": {},
"reverseYBuckets": false,
"targets": [
{
"expr": "sum(increase(address_summary_hist_bucket{status=\"200\"}[2m])) by (le)",
"format": "heatmap",
"intervalFactor": 1,
"legendFormat": "{{le}}",
"refId": "A"
}
],
"timeFrom": null,
"timeShift": null,
"title": "Explorer Python API Heatmap, 2m Rolling Window, Opacity View",
"tooltip": {
"show": true,
"showHistogram": false
},
"type": "heatmap",
"xAxis": {
"show": true
},
"xBucketNumber": null,
"xBucketSize": null,
"yAxis": {
"decimals": null,
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true,
"splitFactor": null
},
"yBucketBound": "auto",
"yBucketNumber": null,
"yBucketSize": null
},
{
"cards": {
"cardPadding": null,
"cardRound": null
},
"color": {
"cardColor": "#73BF69",
"colorScale": "sqrt",
"colorScheme": "interpolateOranges",
"exponent": 0.5,
"min": null,
"mode": "spectrum"
},
"dataFormat": "tsbuckets",
"datasource": "prometheus",
"gridPos": {
"h": 11,
"w": 12,
"x": 12,
"y": 8
},
"heatmap": {},
"hideZeroBuckets": false,
"highlightCards": true,
"id": 3,
"legend": {
"show": true
},
"links": [],
"options": {},
"reverseYBuckets": false,
"targets": [
{
"expr": "sum(increase(address_summary_hist_bucket{status=\"200\"}[2m])) by (le)",
"format": "heatmap",
"intervalFactor": 1,
"legendFormat": "{{le}}",
"refId": "A"
}
],
"timeFrom": null,
"timeShift": null,
"title": "Explorer Python API Heatmap, 2m Rolling Window, Spectrum View",
"tooltip": {
"show": true,
"showHistogram": false
},
"type": "heatmap",
"xAxis": {
"show": true
},
"xBucketNumber": null,
"xBucketSize": null,
"yAxis": {
"decimals": null,
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true,
"splitFactor": null
},
"yBucketBound": "auto",
"yBucketNumber": null,
"yBucketSize": null
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "prometheus",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 19
},
"id": 9,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "address_summary_hist_count",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Explorer Python API Total Hit Count",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"decimals": null,
"format": "short",
"label": "Address API Hits",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "prometheus",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 19
},
"id": 11,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "block_height",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Python Block Height",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": "Python Block Height",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"refresh": "15m",
"schemaVersion": 20,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-1h",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
]
},
"timezone": "",
"title": "Explorer Python API",
"uid": "bSVCvsvZk",
"version": 6
}

42
modules/monitoring-cardano.nix

@ -47,18 +47,6 @@ in {
description = "{{$labels.alias}}: When a node's MemPoolSize grows larger than the system can handle, transactions will be dropped. The actual thresholds for that in mainnet are unknown, but [based on benchmarks done beforehand](https://input-output-rnd.slack.com/archives/C2VJ41WDP/p1506563332000201) transactions started getting dropped when the MemPoolSize was ~200 txs.";
};
}
{
alert = "cardano_node_block_divergence";
expr = "abs(max(cardano_byron_proxy_ChainDB_blockNum_int) - ignoring(alias,instance,job,role) group_right(instance) cardano_node_ChainDB_metrics_blockNum_int) > 2";
for = "5m";
labels = {
severity = "page";
};
annotations = {
summary = "{{$labels.alias}}: cardano-node block divergence with byron proxies detected for more than 5 minutes";
description = "{{$labels.alias}}: cardano-node block divergence with byron proxies detected for more than 5 minutes";
};
}
{
alert = "cardano_new_node_block_divergence";
expr = "abs(max(cardano_node_ChainDB_metrics_blockNum_int{alias!~\"bft-dr.*|rel-dr.*\"}) - ignoring(alias,instance,job,role) group_right(instance) cardano_node_ChainDB_metrics_blockNum_int{alias!~\"bft-dr.*|rel-dr.*\"}) > 2";
@ -147,18 +135,6 @@ in {
description = "{{$labels.alias}}: cardano-node KES expiration warning: less than 4 hours until KES expiration; calculated from opcert decoding";
};
}
{
alert = "byron_proxy_block_divergence";
expr = "abs(max(cardano_total_main_blocks) - ignoring(alias,instance,job,role) group_right(instance) cardano_byron_proxy_ChainDB_blockNum_int) > 2";
for = "5m";
labels = {
severity = "page";
};
annotations = {
summary = "{{$labels.alias}}: byron-proxy block divergence detected for more than 5 minutes";
description = "{{$labels.alias}}: byron-proxy block divergence detected for more than 5 minutes";
};
}
{
alert = "explorer_node_db_block_divergence";
expr = "abs(cardano_node_ChainDB_metrics_blockNum_int{alias=~\"explorer.*\"} - on() db_block_height{alias=~\"explorer.*\"}) > 5";
@ -186,8 +162,8 @@ in {
] ++ (builtins.concatMap ({region, regionLetter}: [
{
alert = "high_tcp_connections_${region}";
expr = "avg(node_netstat_Tcp_CurrEstab{alias=~\"e-${regionLetter}-.*|rel-${regionLetter}-.*\"}) " +
"- count(count(node_netstat_Tcp_CurrEstab{alias=~\"e-${regionLetter}-.*|rel-${regionLetter}-.*\"}) by (alias)) > ${tcpHigh}";
expr = "avg(node_netstat_Tcp_CurrEstab{alias=~\"rel-${regionLetter}-.*\"}) " +
"- count(count(node_netstat_Tcp_CurrEstab{alias=~\"rel-${regionLetter}-.*\"}) by (alias)) > ${tcpHigh}";
for = "5m";
labels = {
severity = "page";
@ -199,8 +175,8 @@ in {
}
{
alert = "critical_tcp_connections_${region}";
expr = "avg(node_netstat_Tcp_CurrEstab{alias=~\"e-${regionLetter}-.*|rel-${regionLetter}-.*\"}) " +
"- count(count(node_netstat_Tcp_CurrEstab{alias=~\"e-${regionLetter}-.*|rel-${regionLetter}-.*\"}) by (alias)) > ${tcpCrit}";
expr = "avg(node_netstat_Tcp_CurrEstab{alias=~\"rel-${regionLetter}-.*\"}) " +
"- count(count(node_netstat_Tcp_CurrEstab{alias=~\"rel-${regionLetter}-.*\"}) by (alias)) > ${tcpCrit}";
for = "15m";
labels = {
severity = "page";
@ -212,7 +188,7 @@ in {
}
{
alert = "high_egress_${region}";
expr = "avg(rate(node_network_transmit_bytes_total{alias=~\"e-${regionLetter}-.*|rel-${regionLetter}-.*\",device!~\"lo\"}[20s]) * 8) > ${MbpsHigh} * 1000 * 1000";
expr = "avg(rate(node_network_transmit_bytes_total{alias=~\"rel-${regionLetter}-.*\",device!~\"lo\"}[20s]) * 8) > ${MbpsHigh} * 1000 * 1000";
for = "5m";
labels = {
severity = "page";
@ -224,7 +200,7 @@ in {
}
{
alert = "critical_egress_${region}";
expr = "avg(rate(node_network_transmit_bytes_total{alias=~\"e-${regionLetter}-.*|rel-${regionLetter}-.*\",device!~\"lo\"}[20s]) * 8) > ${MbpsCrit} * 1000 * 1000";
expr = "avg(rate(node_network_transmit_bytes_total{alias=~\"rel-${regionLetter}-.*\",device!~\"lo\"}[20s]) * 8) > ${MbpsCrit} * 1000 * 1000";
for = "15m";
labels = {
severity = "page";
@ -235,10 +211,10 @@ in {
};
}])
[{ region = "eu-central-1"; regionLetter = "a"; }
{ region = "ap-northeast-1"; regionLetter = "b"; }
{ region = "us-east-2"; regionLetter = "b"; }
{ region = "ap-southeast-1"; regionLetter = "c"; }
{ region = "us-east-2"; regionLetter = "d"; }
{ region = "eu-west-2"; regionLetter = "d"; }
{ region = "us-west-1"; regionLetter = "e"; }
{ region = "eu-west-1"; regionLetter = "f"; }
{ region = "ap-northeast-1"; regionLetter = "f"; }
]);
}

11
nix/cardano.nix

@ -1,22 +1,13 @@
self: super:
let
cardano-sl-pkgs = import self.sourcePaths.cardano-sl {
gitrev = self.sourcePaths.cardano-sl.rev;
};
cardano-db-sync-pkgs = import self.sourcePaths.cardano-db-sync {
gitrev = self.sourcePaths.cardano-db-sync.rev;
};
cardano-byron-proxy-pkgs = import self.sourcePaths.cardano-byron-proxy {
gitrev = self.sourcePaths.cardano-byron-proxy.rev;
};
cardano-explorer-app-pkgs = import self.sourcePaths.cardano-explorer-app {};
cardano-rest-pkgs = import (self.sourcePaths.cardano-rest + "/nix") {};
cardanoNodePkgs = import (self.sourcePaths.cardano-node + "/nix") { gitrev = self.sourcePaths.cardano-node.rev; };
in rec {
inherit cardano-sl-pkgs cardano-db-sync-pkgs cardano-byron-proxy-pkgs cardano-explorer-app-pkgs
cardano-rest-pkgs cardanoNodePkgs;
inherit cardano-db-sync-pkgs cardano-explorer-app-pkgs cardano-rest-pkgs cardanoNodePkgs;
inherit (cardanoNodePkgs.cardanoNodeHaskellPackages.cardano-cli.components.exes) cardano-cli;
inherit (cardanoNodePkgs.cardanoNodeHaskellPackages.cardano-node.components.exes) cardano-node;
cardano-node-legacy = cardano-sl-pkgs.nix-tools.cexes.cardano-sl-node.cardano-node-simple;
cardano-node-legacy-config = cardano-sl-pkgs.cardanoConfig; # FIXME: use iohk-nix
}

2
nix/packages.nix

@ -1,6 +1,4 @@
self: super: {
# TODO: remove when 20.03
inherit (import self.sourcePaths.nixpkgs-postgresql12 {}) postgresql_12;
pp = v: __trace (__toJSON v) v;
leftPad = number: width: self.lib.fixedWidthString width "0" (toString number);

60
nix/sources.json

@ -23,18 +23,6 @@
"url": "https://github.com/input-output-hk/cardano-benchmarking/archive/5d67c4764f27aab974bfa126178bc60827c19561.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"cardano-byron-proxy": {
"branch": "tags/1.0.5",
"description": "A bydirectional proxy between the Shelly and Byron versions of the Cardano network",
"homepage": null,
"owner": "input-output-hk",
"repo": "cardano-byron-proxy",
"rev": "5538821647728516d5702cbfd9fb3000fcf41082",
"sha256": "10s3ivzv6hb7mb32imi2dga02vlaibsx8z1m2h7ns8cfkj4yng30",
"type": "tarball",
"url": "https://github.com/input-output-hk/cardano-byron-proxy/archive/5538821647728516d5702cbfd9fb3000fcf41082.tar.gz",