CAD-1073 bench: anomaly detection
Kosyrev Serge
3 years ago
committed by
Kosyrev Serge
parent
d6890f06fe
commit
c8e9a1930a
@ -0,0 +1,266 @@
|
||||
#!/usr/bin/env bash
|
||||
# shellcheck disable=1091,2016
|
||||
|
||||
analysis_list=()
|
||||
|
||||
analysis_list+=(analysis_cleanup)
|
||||
analysis_cleanup() {
|
||||
local dir=${1:-.}
|
||||
|
||||
rm -f "$dir"/analysis.json
|
||||
rm -rf "$dir"/analysis
|
||||
mkdir -p "$dir"/analysis
|
||||
}
|
||||
|
||||
analysis_list+=(analysis_block_arrivals)
|
||||
analysis_block_arrivals() {
|
||||
local dir=${1:-.}
|
||||
cat "$dir"/logs/block-arrivals.gauge
|
||||
json_file_append "$dir"/analysis.json '
|
||||
{ block_arrivals: $arrivals
|
||||
}' --rawfile arrivals "$dir"/logs/block-arrivals.gauge <<<0
|
||||
}
|
||||
|
||||
analysis_list+=(analysis_unpack)
|
||||
analysis_unpack() {
|
||||
local dir=${1:-.}
|
||||
|
||||
tar x -C "$dir"/analysis -af "$dir"/logs/logs-explorer.tar.xz
|
||||
tar x -C "$dir"/analysis -af "$dir"/logs/logs-nodes.tar.xz
|
||||
}
|
||||
|
||||
analysis_list+=(analysis_log_inventory)
|
||||
analysis_log_inventory()
|
||||
{
|
||||
local dir=${1:-.}; shift
|
||||
local machines=("$@")
|
||||
|
||||
collect_jsonlog_inventory "$dir"/analysis "${machines[@]}" \
|
||||
> "$dir"/analysis/log-inventory.json
|
||||
|
||||
json_file_append "$dir"/analysis.json \
|
||||
'{ final_log_timestamp: ($logs | max_by(.latest) | .latest)
|
||||
, first_node_log_timestamp: ($logs
|
||||
| map (select(.name != "explorer" and
|
||||
.name != "generator"))
|
||||
| min_by(.earliest) | .earliest)
|
||||
, final_node_log_timestamp: ($logs
|
||||
| map (select(.name != "explorer" and
|
||||
.name != "generator"))
|
||||
| max_by(.latest) | .latest)
|
||||
, logs: $logs
|
||||
}' --slurpfile logs "$dir"/analysis/log-inventory.json <<<0
|
||||
}
|
||||
|
||||
analysis_list+=(analysis_timetoblock)
|
||||
analysis_timetoblock() {
|
||||
local dir=${1:-.}
|
||||
dir=$(realpath "$dir")
|
||||
|
||||
pushd "$dir"/analysis >/dev/null || return 1
|
||||
|
||||
"$dir"/tools/analyse.sh \
|
||||
logs-explorer/generator \
|
||||
logs-explorer/node \
|
||||
"$dir"/analysis
|
||||
|
||||
cp -f analysis/*.{csv,json} .
|
||||
|
||||
popd >/dev/null || return 1
|
||||
|
||||
json_file_prepend "$dir"/analysis.json \
|
||||
'{ tx_stats: $txstats[0]
|
||||
}' --slurpfile txstats "$dir"/analysis/tx-stats.json <<<0
|
||||
}
|
||||
|
||||
analysis_list+=(analysis_submission_threads)
|
||||
analysis_submission_threads() {
|
||||
local dir=${1:-.} sub_tids tnum
|
||||
|
||||
sub_tids="$("$dir"/tools/generator-logs.sh log-tids \
|
||||
"$dir"/analysis/logs-explorer/generator.json || true)"
|
||||
json_file_append "$dir"/analysis.json \
|
||||
'{ submission_tids: '"$(jq --slurp <<<$sub_tids)"' }' <<<0
|
||||
|
||||
for tnum in $(seq 0 $(($(echo "$sub_tids" | wc -w) - 1)))
|
||||
do "$dir"/tools/generator-logs.sh tid-trace "${tnum}" \
|
||||
"$dir"/analysis/logs-explorer/generator.json \
|
||||
> "$dir"/analysis/generator.submission-thread-trace."${tnum}".json
|
||||
done
|
||||
}
|
||||
|
||||
analysis_list+=(analysis_from_benchmarking)
|
||||
analysis_from_benchmarking() {
|
||||
local dir=${1:-.}
|
||||
local analysis aname files
|
||||
|
||||
files=($(ls -- "$dir"/analysis/logs-node-*/node-*.json 2>/dev/null || true))
|
||||
if test ${#files[*]} -gt 0
|
||||
then for analysis in $(ls -- "$dir"/tools/node.*.sh 2>/dev/null || true)
|
||||
do aname=$(sed 's_^.*/node\.\(.*\)\.sh$_\1_' <<<$analysis)
|
||||
echo -n " $aname.node"
|
||||
"$dir"/tools/node."$aname".sh "${files[@]}" \
|
||||
> "$dir"/analysis/node."$aname".json
|
||||
test -x "$dir"/tools/tocsv."$aname".sh &&
|
||||
"$dir"/tools/tocsv."$aname".sh \
|
||||
< "$dir"/analysis/node."$aname".json \
|
||||
> "$dir"/analysis/node."$aname".csv; done; fi
|
||||
|
||||
files=($(ls -- "$dir"/analysis/logs-explorer/node-*.json 2>/dev/null || true))
|
||||
if test ${#files[*]} -gt 0
|
||||
then for analysis in $(ls -- "$dir"/tools/explorer.*.sh 2>/dev/null || true)
|
||||
do aname=$(sed 's_^.*/explorer\.\(.*\)\.sh$_\1_' <<<$analysis)
|
||||
echo -n " $aname.explorer"
|
||||
"$dir"/tools/explorer."$aname".sh "${files[@]}" \
|
||||
> "$dir"/analysis/explorer."$aname".json
|
||||
test -x "$dir"/tools/tocsv."$aname".sh &&
|
||||
"$dir"/tools/tocsv."$aname".sh \
|
||||
< "$dir"/analysis/explorer."$aname".json \
|
||||
> "$dir"/analysis/explorer."$aname".csv; done; fi
|
||||
|
||||
files=($(ls -- "$dir"/analysis/logs-explorer/generator*json 2>/dev/null || true))
|
||||
if test ${#files[*]} -gt 0
|
||||
then for analysis in $(ls -- "$dir"/tools/generator.*.sh 2>/dev/null || true)
|
||||
do aname=$(sed 's_^.*/generator\.\(.*\)\.sh$_\1_' <<<$analysis)
|
||||
echo -n " $aname.generator"
|
||||
"$dir"/tools/generator."$aname".sh "${files[@]}" \
|
||||
> "$dir"/analysis/generator."$aname".json
|
||||
test -x "$dir"/tools/tocsv."$aname".sh &&
|
||||
"$dir"/tools/tocsv."$aname".sh \
|
||||
< "$dir"/analysis/generator."$aname".json \
|
||||
> "$dir"/analysis/generator."$aname".csv; done; fi
|
||||
}
|
||||
|
||||
analysis_list+=(analysis_TraceForgeInvalidBlock)
|
||||
analysis_TraceForgeInvalidBlock() {
|
||||
local dir=${1:-.} msg
|
||||
|
||||
msg=$(echo ${FUNCNAME[0]} | cut -d_ -f2)
|
||||
files=($(ls -- "$dir"/analysis/logs-node-*/node-*.json 2>/dev/null || true))
|
||||
if test ${#files[*]} -eq 0
|
||||
then return; fi
|
||||
|
||||
grep --quiet --no-filename -F "\"$msg\"" "${files[@]}" || true |
|
||||
jq 'def katip_timestamp_to_iso8601:
|
||||
.[:-4] + "Z";
|
||||
.
|
||||
| map
|
||||
( (.at | katip_timestamp_to_iso8601)
|
||||
as $date_iso
|
||||
| { date_iso: $date_iso
|
||||
, timestamp: $date_iso | fromdateiso8601
|
||||
, reason: .data.reason
|
||||
, slot: .data.slot
|
||||
}
|
||||
)
|
||||
| sort_by (.timestamp)
|
||||
| .[]
|
||||
' --slurp --compact-output > "$dir"/analysis/node."$msg".json
|
||||
}
|
||||
|
||||
analysis_list+=(analysis_message_types)
|
||||
analysis_message_types() {
|
||||
local dir=${1:-.} mach tnum sub_tids; shift
|
||||
local machines=("$@")
|
||||
|
||||
for mach in ${machines[*]}
|
||||
do echo -n .$mach >&2
|
||||
local types key
|
||||
"$dir"/tools/msgtypes.sh \
|
||||
"$dir/analysis/logs-$mach"/node-*.json |
|
||||
while read -r ty
|
||||
test -n "$ty"
|
||||
do key=$(jq .kind <<<$ty -r | sed 's_.*\.__g')
|
||||
jq '{ key: .kind, value: $count }' <<<$ty \
|
||||
--argjson count "$(grep -Fh "$key\"" \
|
||||
"$dir/analysis/logs-$mach"/node-*.json |
|
||||
wc -l)"
|
||||
done |
|
||||
jq '{ "\($name)": from_entries }
|
||||
' --slurp --arg name "$mach"
|
||||
# jq '{ "\($name)": $types }
|
||||
# ' --arg name "$mach" --null-input \
|
||||
# --argjson types "$("$dir"/tools/msgtypes.sh \
|
||||
# "$dir/analysis/logs-$mach"/node-*.json |
|
||||
# jq . --slurp)"
|
||||
done | analysis_append "$dir" \
|
||||
'{ message_types: add
|
||||
}' --slurp
|
||||
}
|
||||
|
||||
analysis_list+=(analysis_repackage_db)
|
||||
analysis_repackage_db() {
|
||||
local dir=${1:-.}
|
||||
|
||||
tar x -C "$dir"/analysis -af "$dir"/logs/db-analysis.tar.xz \
|
||||
--wildcards '*.csv' '*.txt'
|
||||
}
|
||||
|
||||
# TODO: broken
|
||||
# analysis_list+=(analysis_tx_losses)
|
||||
analysis_tx_losses() {
|
||||
local dir=${1:-.}
|
||||
dir=$(realpath "$dir")
|
||||
|
||||
pushd "$dir"/analysis >/dev/null || return 1
|
||||
if jqtest '(.tx_stats.tx_missing != 0)' "$dir"/analysis.json
|
||||
then echo -n " missing-txs"
|
||||
. "$dir"/tools/lib-loganalysis.sh
|
||||
op_analyse_losses; fi
|
||||
popd >/dev/null || return 1
|
||||
}
|
||||
|
||||
analysis_list+=(analysis_derived)
|
||||
analysis_derived() {
|
||||
local dir=${1:-.}
|
||||
local f="$dir"/analysis/node.TraceMempoolRejectedTx.json
|
||||
|
||||
analysis_append "$dir" \
|
||||
'{ tx_stats:
|
||||
($analysis.tx_stats
|
||||
+ { tx_rejected: $rejected
|
||||
, tx_utxo_invalid: $utxo_invalid
|
||||
, tx_missing_input: $missing_input })}
|
||||
' --argjson rejected "$(wc -l <$f)" \
|
||||
--argjson utxo_invalid "$(grep -F "(UTxOValidationUTxOError " $f | wc -l)" \
|
||||
--argjson missing_input "$(grep -F "(UTxOMissingInput " $f | wc -l)" \
|
||||
<<<0
|
||||
}
|
||||
|
||||
analysis_list+=(analysis_sanity)
|
||||
analysis_sanity() {
|
||||
local dir=${1:-.} tag errors
|
||||
tag=$(run_tag "$dir")
|
||||
|
||||
errors="$(sanity_check_run "$dir")"
|
||||
if test "$errors" != "[]"
|
||||
then echo
|
||||
oprint "sanity check failed for tag: $tag"
|
||||
echo "$errors" >&2
|
||||
mark_run_broken "$dir" "$errors"
|
||||
return 1; fi
|
||||
}
|
||||
|
||||
###
|
||||
### Aux
|
||||
###
|
||||
jsonlog_inventory() {
|
||||
local name=$1; shift
|
||||
local args fs=("$@")
|
||||
|
||||
args=(--arg name "$name"
|
||||
--argjson earliest "$(head -n1 ${fs[0]})"
|
||||
--argjson latest "$(tail -n1 ${fs[-1]})"
|
||||
--argjson files "$(echo ${fs[*]} | shell_list_to_json)"
|
||||
)
|
||||
jq 'def katip_timestamp_to_iso8601:
|
||||
.[:-4] + "Z";
|
||||
.
|
||||
| { name: $name
|
||||
, earliest: ($earliest.at
|
||||
| katip_timestamp_to_iso8601 | fromdateiso8601)
|
||||
, latest: ( $latest.at
|
||||
| katip_timestamp_to_iso8601 | fromdateiso8601)
|
||||
, files: $files
|
||||
}' "${args[@]}" <<<0
|
||||
}
|
||||
@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env bash
|
||||
# shellcheck disable=1091,2016
|
||||
|
||||
|
||||
collect_jsonlog_inventory() {
|
||||
local dir=$1; shift
|
||||
local constituents=("$@")
|
||||
|
||||
for mach in ${constituents[*]}
|
||||
do jsons=($(ls -- "$dir"/logs-"$mach"/node-*.json))
|
||||
jsonlog_inventory "$mach" "${jsons[@]}"; done
|
||||
jsonlog_inventory "generator" "$dir"/logs-explorer/generator.json
|
||||
}
|
||||
|
||||
analysis_append() {
|
||||
local dir=$1 expr=$2; shift 2
|
||||
json_file_append "$dir"/analysis.json '
|
||||
$meta[0] as $meta
|
||||
| $analysis[0] as $analysis
|
||||
| '"$expr
|
||||
" --slurpfile meta "$dir/meta.json" \
|
||||
--slurpfile analysis "$dir/analysis.json" \
|
||||
"$@"
|
||||
}
|
||||
|
||||
analysis_prepend() {
|
||||
local dir=$1 expr=$2; shift 2
|
||||
json_file_prepend "$dir"/analysis.json '
|
||||
$meta[0] as $meta
|
||||
| $analysis[0] as $analysis
|
||||
| '"$expr
|
||||
" --slurpfile meta "$dir/meta.json" \
|
||||
--slurpfile analysis "$dir/analysis.json" \
|
||||
"$@"
|
||||
}
|
||||
|
||||
###
|
||||
###
|
||||
|
||||
analyse_run() {
|
||||
while test $# -ge 1
|
||||
do case "$1" in
|
||||
--list ) echo ${analysis_list[*]}; return;;
|
||||
* ) break;; esac; shift; done
|
||||
|
||||
local dir=${1:-.} tag meta
|
||||
dir=$(realpath "$dir")
|
||||
|
||||
if test ! -d "$dir"
|
||||
then fail "run directory doesn't exist: $dir"; fi
|
||||
if test ! -f "$dir/meta.json"
|
||||
then fail "run directory doesn't has no metafile: $dir"; fi
|
||||
run_fetch_benchmarking "$dir/tools"
|
||||
|
||||
machines=($(jq '.machine_info | keys | join(" ")
|
||||
' --raw-output <"$dir/deployment-explorer.json"))
|
||||
meta=$(jq .meta "$dir/meta.json")
|
||||
tag=$(jq .tag <<<$meta --raw-output)
|
||||
|
||||
echo "--( processing logs in: $(basename "$dir")"
|
||||
|
||||
for a in "${analysis_list[@]}"
|
||||
do echo -n " $a" | sed 's/analysis_//'
|
||||
$a "$dir" "${machines[@]}"; done
|
||||
|
||||
patch_run "$dir"
|
||||
|
||||
rm -rf "$dir"/analysis/{analysis,logs-node-*,logs-explorer,startup}
|
||||
|
||||
oprint "analysed tag: ${tag}"
|
||||
}
|
||||
|
||||
runs_in() {
|
||||
local dir=${1:-.}
|
||||
dir=$(realpath $dir)
|
||||
find "$dir" -maxdepth 2 -mindepth 2 -name meta.json -type f | cut -d/ -f$(($(tr -cd / <<<$dir | wc -c) + 2))
|
||||
}
|
||||
|
||||
mass_analyse() {
|
||||
local parallel=
|
||||
while test $# -ge 1
|
||||
do case "$1" in
|
||||
--parallel ) parallel=t;;
|
||||
* ) break;; esac; shift; done
|
||||
|
||||
local dir=${1:-.} runs
|
||||
runs=($(runs_in "$dir"))
|
||||
|
||||
oprint "analysing runs: ${runs[*]}"
|
||||
|
||||
for run in "${runs[@]}"
|
||||
do if test -n "$parallel"
|
||||
then analyse_run "$dir/$run" &
|
||||
else analyse_run "$dir/$run"; fi; done
|
||||
}
|
||||
@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env bash
|
||||
# shellcheck disable=1091,2016
|
||||
|
||||
tag_format_timetoblock_header="tx id,tx time,block time,block no,delta t"
|
||||
patch_run() {
|
||||
local dir=${1:-.}
|
||||
dir=$(realpath "$dir")
|
||||
|
||||
if test "$(head -n1 "$dir"/analysis/timetoblock.csv)" \
|
||||
!= "${tag_format_timetoblock_header}"
|
||||
then echo "---| patching $dir/analysis/timetoblock.csv"
|
||||
sed -i "1 s_^_${tag_format_timetoblock_header}\n_; s_;_,_g" \
|
||||
"$dir"/analysis/timetoblock.csv
|
||||
fi
|
||||
|
||||
if test "$(head -n1 "$dir"/analysis/00-results-table.sql.csv)" \
|
||||
== "DROP TABLE"
|
||||
then echo "---| patching $dir/analysis/00-results-table.sql.csv"
|
||||
tail -n+3 "$dir"/analysis/00-results-table.sql.csv \
|
||||
> "$dir"/analysis/00-results-table.sql.csv.fixed
|
||||
mv "$dir"/analysis/00-results-table.sql.csv.fixed \
|
||||
"$dir"/analysis/00-results-table.sql.csv;
|
||||
fi
|
||||
}
|
||||
|
||||
run_report_name() {
|
||||
local metafile meta prof suffix=
|
||||
dir=${1:-.}
|
||||
metafile="$dir"/meta.json
|
||||
meta=$(jq .meta "$metafile" --raw-output)
|
||||
prof=$(jq .profile <<<$meta --raw-output)
|
||||
date=$(date +'%Y'-'%m'-'%d'-'%H.%M' --date=@"$(jq .timestamp <<<$meta)")
|
||||
|
||||
test -n "$meta" -a -n "$prof" ||
|
||||
fail "Bad run meta.json format: $metafile"
|
||||
|
||||
if is_run_broken "$dir"
|
||||
then suffix='broken'; fi
|
||||
|
||||
echo "$date.$prof${suffix:+.$suffix}"
|
||||
}
|
||||
|
||||
package_run() {
|
||||
local tag report_name package
|
||||
dir=${1:-.}
|
||||
report_name=$(run_report_name "$dir")
|
||||
|
||||
if is_run_broken "$dir"
|
||||
then resultroot=$(realpath ../bench-results-bad)
|
||||
else resultroot=$(realpath ../bench-results); fi
|
||||
|
||||
package=${resultroot}/$report_name.tar.xz
|
||||
|
||||
oprint "Packaging $tag as: $package"
|
||||
ln -sf "./runs/$tag" "$report_name"
|
||||
tar cf "$package" "$report_name" --xz --dereference
|
||||
rm -f "$report_name"
|
||||
}
|
||||
@ -0,0 +1,201 @@
|
||||
#!/usr/bin/env bash
|
||||
# shellcheck disable=1091,2016
|
||||
|
||||
sanity_check_list=()
|
||||
|
||||
sanity_check_list+=(sanity_check_start_log_spread)
|
||||
sanity_check_start_log_spread() {
|
||||
local dir=$1 t=${2:-${default_tolerances}}
|
||||
sanity_check "$t" "$dir" '
|
||||
$analysis.logs
|
||||
| map
|
||||
( (.earliest - $meta.timestamp | fabs)
|
||||
as $delta
|
||||
| select ($delta > $allowed.start_log_spread_s)
|
||||
| . +
|
||||
{ delta: $delta
|
||||
, start: $meta.timestamp })
|
||||
' '.
|
||||
| map
|
||||
({ kind: "start-log-spread"
|
||||
} + .)
|
||||
| .[]'
|
||||
}
|
||||
sanity_check_list+=(sanity_check_last_log_spread)
|
||||
sanity_check_last_log_spread() {
|
||||
local dir=$1 t=${2:-${default_tolerances}}
|
||||
sanity_check "$t" "$dir" '
|
||||
$analysis.logs
|
||||
| map ## Generator always finishes a bit early, and
|
||||
## we have it analysed to death by other means..
|
||||
(select (.name != "generator"))
|
||||
| map
|
||||
( (.latest - $analysis.final_log_timestamp | fabs)
|
||||
as $delta
|
||||
| select ($delta > $allowed.last_log_spread_s)
|
||||
| . +
|
||||
{ delta: $delta
|
||||
, final_log_timestamp: $analysis.final_log_timestamp })
|
||||
' '.
|
||||
| map
|
||||
({ kind: "latest-log-spread"
|
||||
} + .)
|
||||
| .[]'
|
||||
}
|
||||
sanity_check_list+=(sanity_check_not_even_started)
|
||||
sanity_check_not_even_started() {
|
||||
local dir=$1 t=${2:-${default_tolerances}}
|
||||
sanity_check "$t" "$dir" '
|
||||
$blocks
|
||||
| length == 0
|
||||
' '.
|
||||
| { kind: "blockchain-not-even-started"
|
||||
}' --slurpfile blocks "$dir"/analysis/explorer.MsgBlock.json
|
||||
}
|
||||
sanity_check_list+=(sanity_check_silence_since_last_block)
|
||||
sanity_check_silence_since_last_block() {
|
||||
local dir=$1 t=${2:-${default_tolerances}}
|
||||
sanity_check "$t" "$dir" '
|
||||
$blocks[-1] // { timestamp: $analysis.first_node_log_timestamp }
|
||||
| ($analysis.final_node_log_timestamp - .timestamp)
|
||||
as $delta
|
||||
| if $delta >= $allowed.silence_since_last_block_s
|
||||
then $delta else empty end
|
||||
' '.
|
||||
| { kind: "blockchain-stopped"
|
||||
, silence_since_last_block_s: .
|
||||
, allowance: $allowed.silence_since_last_block_s
|
||||
}' --slurpfile blocks "$dir"/analysis/explorer.MsgBlock.json
|
||||
}
|
||||
sanity_check_list+=(sanity_check_no_txs_in_blocks)
|
||||
sanity_check_no_txs_in_blocks() {
|
||||
local dir=$1 t=${2:-${default_tolerances}}
|
||||
sanity_check "$t" "$dir" '
|
||||
$txstats.tx_seen_in_blocks == 0' '
|
||||
{ kind: "no-txs-in-blocks"
|
||||
}'
|
||||
}
|
||||
sanity_check_list+=(sanity_check_announced_less_txs_than_specified)
|
||||
sanity_check_announced_less_txs_than_specified() {
|
||||
local dir=$1 t=${2:-${default_tolerances}}
|
||||
sanity_check "$t" "$dir" '
|
||||
## Guard against old logs, where tx_annced is 0:
|
||||
$txstats.tx_annced >= $txstats.tx_sent and
|
||||
$prof.generator.tx_count > $txstats.tx_annced' '
|
||||
{ kind: "announced-less-txs-than-specified"
|
||||
, required: $prof.generator.tx_count
|
||||
, sent: $txstats.tx_sent
|
||||
}'
|
||||
}
|
||||
sanity_check_list+=(sanity_check_sent_less_txs_than_specified)
|
||||
sanity_check_sent_less_txs_than_specified() {
|
||||
local dir=$1 t=${2:-${default_tolerances}}
|
||||
sanity_check "$t" "$dir" '
|
||||
$prof.generator.tx_count > $txstats.tx_sent' '
|
||||
{ kind: "sent-less-txs-than-specified"
|
||||
, required: $prof.generator.tx_count
|
||||
, sent: $txstats.tx_sent
|
||||
}'
|
||||
}
|
||||
sanity_check_list+=(sanity_check_tx_loss_over_threshold)
|
||||
sanity_check_tx_loss_over_threshold() {
|
||||
local dir=$1 t=${2:-${default_tolerances}}
|
||||
sanity_check "$t" "$dir" '
|
||||
$txstats.tx_sent * (1.0 - $allowed.tx_loss_ratio)
|
||||
> $txstats.tx_seen_in_blocks' '
|
||||
{ kind: "txs-loss-over-threshold"
|
||||
, sent: $txstats.tx_sent
|
||||
, threshold: ($txstats.tx_sent * (1.0 - $allowed.tx_loss_ratio))
|
||||
, received: $txstats.tx_seen_in_blocks
|
||||
}'
|
||||
}
|
||||
sanity_check_list+=(sanity_check_chain_density)
|
||||
sanity_check_chain_density() {
|
||||
local dir=$1 t=${2:-${default_tolerances}}
|
||||
sanity_check "$t" "$dir" '
|
||||
($blocks | length)
|
||||
as $block_count
|
||||
| ($analysis.final_node_log_timestamp
|
||||
- $analysis.first_node_log_timestamp)
|
||||
as $cluster_lifetime_s
|
||||
| ($cluster_lifetime_s * 1000 / $genesis.slot_duration | floor)
|
||||
as $cluster_lifetime_slots
|
||||
| ($block_count / ($cluster_lifetime_slots))
|
||||
as $chain_density
|
||||
| ($cluster_lifetime_slots - $block_count)
|
||||
as $missed_slots
|
||||
| if $chain_density < $allowed.minimum_chain_density or
|
||||
$missed_slots > $allowed.maximum_missed_slots
|
||||
then { lifetime_s: $cluster_lifetime_s
|
||||
, lifetime_slots: $cluster_lifetime_slots
|
||||
, block_count: $block_count
|
||||
, missed_slots: $missed_slots
|
||||
, chain_density: $chain_density
|
||||
} else empty end' '
|
||||
{ kind: "insufficient_overall_chain_density"
|
||||
, lifetime_s: .lifetime_s
|
||||
, lifetime_slots: .lifetime_slots
|
||||
, block_count: .block_count
|
||||
, missed_slots: .missed_slots
|
||||
, chain_density: .chain_density
|
||||
}' --slurpfile blocks "$dir"/analysis/explorer.MsgBlock.json
|
||||
}
|
||||
# sanity_check_list+=(sanity_check_)
|
||||
# sanity_check_() {
|
||||
# local t=$1 dir=$2
|
||||
# }
|
||||
|
||||
default_tolerances='
|
||||
{ "tx_loss_ratio": 0.0
|
||||
, "start_log_spread_s": 60
|
||||
, "last_log_spread_s": 60
|
||||
, "silence_since_last_block_s": 40
|
||||
, "cluster_startup_overhead_s": 60
|
||||
, "minimum_chain_density": 0.9
|
||||
, "maximum_missed_slots": 5
|
||||
}'
|
||||
|
||||
sanity_check_run() {
|
||||
local dir=${1:-.} metafile meta prof tolerances t
|
||||
|
||||
for check in ${sanity_check_list[*]}
|
||||
do $check "$dir" "${default_tolerances}"
|
||||
done | jq --slurp '
|
||||
if length != 0
|
||||
then . +
|
||||
[{ kind: "tolerances" }
|
||||
+ $tolerances] else . end
|
||||
' --argjson tolerances "$default_tolerances"
|
||||
}
|
||||
|
||||
sanity_check() {
|
||||
local tolerances=$1 dir=$2 test=$3 err=$4; shift 4
|
||||
sanity_checker "$tolerances" "$dir" \
|
||||
" ($test)"' as $test
|
||||
| if $test != {} and $test != [] and $test != "" and $test
|
||||
then ($test | '"$err"') else empty end
|
||||
' "$@"
|
||||
}
|
||||
|
||||
sanity_checker() {
|
||||
local tolerances=$1 dir=$2 expr=$3; shift 3
|
||||
|
||||
jq ' $meta[0].meta as $meta
|
||||
| $analysis[0] as $analysis
|
||||
| $txstats[0] as $txstats
|
||||
| ($meta.profile_content
|
||||
## TODO: backward compat
|
||||
// $meta.generator_params)
|
||||
as $prof
|
||||
| ($prof.genesis
|
||||
## TODO: backward compat
|
||||
// $prof.genesis_params)
|
||||
as $genesis
|
||||
| $prof.generator as $generator
|
||||
| '"$expr"'
|
||||
' --slurpfile meta "$dir/meta.json" \
|
||||
--slurpfile analysis "$dir/analysis.json" \
|
||||
--slurpfile txstats "$dir/analysis/tx-stats.json" \
|
||||
--argjson allowed "$tolerances" \
|
||||
"$@" <<<0
|
||||
}
|
||||
@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env bash
|
||||
# shellcheck disable=1091,2016
|
||||
|
||||
sheet_list=()
|
||||
|
||||
sheet_list+=(sheet_message_types_summary)
|
||||
sheet_message_types_summary() {
|
||||
local dir=${1:-.} name
|
||||
name=$(echo ${FUNCNAME[0]} | cut -d_ -f2-)
|
||||
|
||||
mkdir -p "$dir"/report
|
||||
|
||||
jq ' .message_types
|
||||
| to_entries
|
||||
| map ( .key as $mach
|
||||
| .value
|
||||
| to_entries
|
||||
| map([ $mach, .key, .value | tostring]))
|
||||
| add
|
||||
| .[]
|
||||
| join(",")' < "$dir"/analysis.json --raw-output \
|
||||
> "$dir"/report/"$name".csv
|
||||
|
||||
sed -i '1inode, message, occurences' "$dir"/report/"$name".csv
|
||||
}
|
||||
@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env bash
|
||||
# shellcheck disable=1091,2016
|
||||
|
||||
run_tag() {
|
||||
jq --raw-output .meta.tag "$(realpath "${1:-.}")/meta.json"
|
||||
}
|
||||
|
||||
cluster_last_meta_tag() {
|
||||
local meta=./last-meta.json tag dir meta2
|
||||
jq . "${meta}" >/dev/null || fail "malformed run metadata: ${meta}"
|
||||
|
||||
tag=$(jq --raw-output .meta.tag "${meta}")
|
||||
test -n "${tag}" || fail "bad tag in run metadata: ${meta}"
|
||||
|
||||
dir="./runs/${tag}"
|
||||
test -d "${dir}" ||
|
||||
fail "bad tag in run metadata: ${meta} -- ${dir} is not a directory"
|
||||
meta2=${dir}/meta.json
|
||||
jq --exit-status . "${meta2}" >/dev/null ||
|
||||
fail "bad tag in run metadata: ${meta} -- ${meta2} is not valid JSON"
|
||||
|
||||
test "$(realpath ./last-meta.json)" = "$(realpath "${meta2}")" ||
|
||||
fail "bad tag in run metadata: ${meta} -- ${meta2} is different from ${meta}"
|
||||
echo "${tag}"
|
||||
}
|
||||
|
||||
fetch_tag() {
|
||||
local tag
|
||||
tag=${1:-$(cluster_last_meta_tag)}
|
||||
|
||||
fetch_run "./runs/${tag}"
|
||||
}
|
||||
|
||||
analyse_tag() {
|
||||
local tag
|
||||
tag=${1:-$(cluster_last_meta_tag)}
|
||||
|
||||
analyse_run "${tagroot}/${tag}" || true
|
||||
}
|
||||
|
||||
sanity_check_tag() {
|
||||
local tag
|
||||
tag=${1:-$(cluster_last_meta_tag)}
|
||||
|
||||
sanity_check_run "${tagroot}/${tag}"
|
||||
}
|
||||
|
||||
tag_report_name() {
|
||||
local tag
|
||||
tag=${1:-$(cluster_last_meta_tag)}
|
||||
|
||||
run_report_name "${tagroot}/${tag}"
|
||||
}
|
||||
|
||||
|
||||
package_tag() {
|
||||
local tag
|
||||
tag=${1:-$(cluster_last_meta_tag)}
|
||||
|
||||
package_run "${tagroot}/${tag}"
|
||||
}
|
||||
@ -1,147 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# shellcheck disable=1091
|
||||
|
||||
tmjq() {
|
||||
jq .meta "${tagroot}/$1/"meta.json --raw-output
|
||||
}
|
||||
|
||||
tag_report_name() {
|
||||
local tag metafile meta prof suffix=
|
||||
tag=${1:-$(cluster_last_meta_tag)}
|
||||
metafile=${tagroot}/$tag/meta.json
|
||||
meta=$(jq .meta "$metafile" --raw-output)
|
||||
prof=$(jq .profile --raw-output <<<$meta)
|
||||
date=$(date +'%Y'-'%m'-'%d'-'%H.%M' --date=@"$(jq .timestamp <<<$meta)")
|
||||
|
||||
test -n "$meta" -a -n "$prof" ||
|
||||
fail "Bad tag meta.json format: $metafile"
|
||||
|
||||
if is_run_broken "$tag"
|
||||
then suffix='broken'; fi
|
||||
|
||||
echo "$date.$prof${suffix:+.$suffix}"
|
||||
}
|
||||
|
||||
package_tag() {
|
||||
local tag package report_name
|
||||
tag=${1:-$(cluster_last_meta_tag)}
|
||||
report_name=$(tag_report_name "$tag")
|
||||
|
||||
if is_run_broken "$tag"
|
||||
then resultroot=$(realpath ../bench-results-bad)
|
||||
else resultroot=$(realpath ../bench-results); fi
|
||||
|
||||
package=${resultroot}/$report_name.tar.xz
|
||||
|
||||
oprint "Packaging $tag as: $package"
|
||||
ln -sf "./runs/$tag" "$report_name"
|
||||
tar cf "$package" "$report_name" --xz --dereference
|
||||
rm -f "$report_name"
|
||||
}
|
||||
|
||||
analyse_tag() {
|
||||
local tag dir meta
|
||||
tag=${1:-$(cluster_last_meta_tag)}
|
||||
dir="${tagroot}/${tag}"
|
||||
|
||||
pushd "${dir}" >/dev/null || return 1
|
||||
rm -rf 'analysis'
|
||||
mkdir 'analysis'
|
||||
cd 'analysis'
|
||||
meta=$(tmjq "$tag" .)
|
||||
|
||||
oprint "running log analyses: "
|
||||
tar xaf '../logs/logs-explorer.tar.xz'
|
||||
tar xaf '../logs/logs-nodes.tar.xz'
|
||||
|
||||
echo " timetoblock.csv"
|
||||
../tools/analyse.sh \
|
||||
'logs-explorer/generator' \
|
||||
'logs-explorer/node' \
|
||||
'last-run/analysis/'
|
||||
cp analysis/timetoblock.csv .
|
||||
|
||||
local blocks
|
||||
echo -n "--( running log analyses: blocksizes"
|
||||
blocks="$(../tools/blocksizes.sh logs-explorer/node-*.json |
|
||||
jq . --slurp)"
|
||||
|
||||
declare -A msgtys
|
||||
local mach msgtys=() producers tnum sub_tids
|
||||
producers=($(jq '.machine_info | keys | join(" ")
|
||||
' --raw-output <'../deployment-explorer.json'))
|
||||
|
||||
for mach in explorer ${producers[*]}
|
||||
do echo -n " msgtys:${mach}"
|
||||
msgtys[${mach}]="$(../tools/msgtypes.sh logs-explorer/node-*.json |
|
||||
jq . --slurp)"; done
|
||||
## NOTE: This is a bit too costly, and we know the generator pretty well.
|
||||
# echo -n " msgtys:generator"
|
||||
# msgtys_generator="$(../tools/msgtypes.sh logs-explorer/generator.json |
|
||||
# jq . --slurp)"
|
||||
msgtys_generator='[]'
|
||||
|
||||
echo -n " node-to-node-submission-tids"
|
||||
sub_tids="$(../tools/generator-logs.sh log-tids \
|
||||
logs-explorer/generator.json || true)"
|
||||
for tnum in $(seq 0 $(($(echo "$sub_tids" | wc -w) - 1)))
|
||||
do echo -n " node-to-node-submission:${tnum}"
|
||||
../tools/generator-logs.sh tid-trace "${tnum}" \
|
||||
logs-explorer/generator.json \
|
||||
> generator.submission-thread-trace."${tnum}".json; done
|
||||
|
||||
for p in ${producers[*]}
|
||||
do echo -n " added-to-current-chain:$p"
|
||||
../tools/added-to-current-chain.sh logs-node-*/node-*.json \
|
||||
> $p.added-to-current-chain.csv; done
|
||||
|
||||
jq '{ tx_stats: $txstats[0]
|
||||
, submission_tids: '"$(jq --slurp <<<$sub_tids)"'
|
||||
, MsgBlock: '"${blocks}"'
|
||||
, message_kinds:
|
||||
({ generator: '"${msgtys_generator}"'
|
||||
}'"$(for mach in ${!msgtys[*]}
|
||||
do echo " + { \"$mach\": $(jq --slurp <<<${msgtys[$mach]}) }"
|
||||
done)"')
|
||||
}' --null-input \
|
||||
--slurpfile txstats 'analysis/tx-stats.json' \
|
||||
> ../analysis.json
|
||||
|
||||
echo -n " adding db-analysis"
|
||||
tar xaf '../logs/db-analysis.tar.xz' --wildcards '*.csv' '*.txt'
|
||||
|
||||
if jqtest '(.tx_stats.tx_missing != 0)' ../analysis.json
|
||||
then echo " missing-txs"
|
||||
. ../tools/lib-loganalysis.sh
|
||||
op_analyse_losses
|
||||
else echo
|
||||
fi
|
||||
patch_local_tag "$tag"
|
||||
|
||||
rm -rf analysis/ logs-node-*/ logs-explorer/ startup/
|
||||
|
||||
popd >/dev/null
|
||||
|
||||
oprint "analysed tag: ${tag}"
|
||||
}
|
||||
|
||||
tag_format_timetoblock_header="tx id,tx time,block time,block no,delta t"
|
||||
patch_local_tag() {
|
||||
local tag=${1?missing tag} target
|
||||
target=${tagroot}/${tag}
|
||||
cd "${target}" >/dev/null || return 1
|
||||
|
||||
if test "$(head -n1 analysis/timetoblock.csv)" != "${tag_format_timetoblock_header}"
|
||||
then echo "---| patching ${tag}/analysis/timetoblock.csv"
|
||||
sed -i "1 s_^_${tag_format_timetoblock_header}\n_; s_;_,_g" \
|
||||
'analysis/timetoblock.csv'
|
||||
fi
|
||||
|
||||
if test "$(head -n1 analysis/00-results-table.sql.csv)" == "DROP TABLE"
|
||||
then echo "---| patching ${tag}/analysis/00-results-table.sql.csv"
|
||||
tail -n+3 analysis/00-results-table.sql.csv > analysis/00-results-table.sql.csv.fixed
|
||||
mv analysis/00-results-table.sql.csv.fixed analysis/00-results-table.sql.csv;
|
||||
fi
|
||||
|
||||
cd - >/dev/null || return 1
|
||||
}
|
||||
Loading…
Reference in New Issue