Commit 6be2df98 authored by Jan Eitzinger's avatar Jan Eitzinger
Browse files

Add system benchmark script. Add GB unit in extract Results

parent ec8ab103
#!/usr/bin/env perl
# =======================================================================================
#
# Author: Jan Eitzinger (je), jan.eitzinger@fau.de
# Copyright (c) 2021 RRZE, University Erlangen-Nuremberg
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# =======================================================================================
use strict;
use warnings;
use utf8;
use Data::Dumper qw(Dumper);
my ($DIR, $CMD, $PREFIX) = @ARGV;
if (not defined $DIR) {
die "Usage: $0 <DATA-DIR> <EXE> <PREFIX>\n";
}
my %RES;
my %INFO;
my %RESULT;
my %PLOT;
my %DPLOT;
my @testcases = ('Init', 'Sum', 'Copy', 'Update', 'Triad', 'Daxpy', 'STriad', 'SDaxpy');
my $numDomainsPerSocket;
my $numDomainsPerNode;
my $numCoresPerMemoryDomain;
my $coresPerSocket;
my $SMT;
# Step 0 : Build benchmark
`rm $DIR/raw/*`;
`rm $DIR/*`;
`mkdir $DIR/raw`;
`make distclean`;
`make > $DIR/buildoutput.txt`;
# Step 1 : Extract system information
`likwid-topology -g > $DIR/topology.txt`;
my $topo = `likwid-topology -O`;
#my $topo = do {
# local $/ = undef;
# open my $fh, "<", './topo.txt'
# or die "could not open tile: $!";
# <$fh>;
#};
foreach my $ln (split("\n", $topo)) {
if ( $ln =~ /^CPU name:/ ) {
my @fields = split(",", $ln);
$INFO{processor} = $fields[1];
}
if ( $ln =~ /^CPU type/ ) {
my @fields = split(",", $ln);
$INFO{family} = $fields[1];
$INFO{family} =~ s/[\(\)]//g;
}
if ( $ln =~ /^Sockets:/ ) {
my @fields = split(",", $ln);
$INFO{numSockets} = $fields[1];
}
if ( $ln =~ /^Cores per socket:/ ) {
my @fields = split(",", $ln);
$INFO{numCoresPerSocket} = $fields[1];
}
if ( $ln =~ /^Threads per core:/ ) {
my @fields = split(",", $ln);
$SMT = $fields[1];
$INFO{numThreadsPerCore} = $SMT;
}
if ( $ln =~ /^NUMA domains:/ ) {
my @fields = split(",", $ln);
$numDomainsPerNode = $fields[1];
}
}
$INFO{family} =~ s/[ ]//g;
my $cinfo = `make info`;
# my $cinfo = do {
# local $/ = undef;
# open my $fh, "<", './toolchain.txt'
# or die "could not open tile: $!";
# <$fh>;
# };
my @tmp = split("\n", $cinfo);
$INFO{flags} = $tmp[0];
my @tools = split(" ", $tmp[1]);
$INFO{toolchain} = $tools[0]. ' ' . $tools[1];
$INFO{version} = $tmp[1];
$numDomainsPerSocket = $numDomainsPerNode / $INFO{numSockets};
$numDomainsPerNode = 2 * $numDomainsPerSocket;
$numCoresPerMemoryDomain = $INFO{numCoresPerSocket} / $numDomainsPerSocket;
my $TAG = $PREFIX."-S$INFO{numSockets}-M$numDomainsPerSocket-C$INFO{numCoresPerSocket}";
$INFO{numDomainsPerSocket} = $numDomainsPerSocket;
# Step 2 : Execute benchmark
foreach my $numdomains ( 1 ... $numDomainsPerNode ) {
foreach my $numcores ( 1 ... $numCoresPerMemoryDomain ) {
my $exp = join('@',map("E:M$_:$numcores:1:$SMT", 0 ... $numdomains-1));
print "node-M$numdomains-$numcores\n";
`likwid-pin -q -C $exp $CMD > $DIR/raw/node-M$numdomains-$numcores.txt`;
}
}
# Step 3 : Generate output
extractResults();
$RESULT{core} = findMaxResult(1,1);
$RESULT{domain} = findMaxResult(1, $numCoresPerMemoryDomain);
$RESULT{socket} = findMaxResult($numDomainsPerSocket, $numCoresPerMemoryDomain);
$RESULT{node} = findMaxResult($numDomainsPerNode, $numCoresPerMemoryDomain);
$RESULT{scaling} = sprintf "#nt";
foreach my $test ( @testcases ) {
$RESULT{scaling} .= sprintf "\t%s", $test;
$PLOT{$test} = '';
}
$RESULT{scaling} .= sprintf "\n";
foreach my $key (sort {$a <=> $b} keys %{$RES{1}}) {
$RESULT{scaling} .= sprintf "%d", $key;
foreach my $test ( @testcases ) {
$RESULT{scaling} .= sprintf "\t%.2f", $RES{1}{$key}{$test};
$PLOT{$test} .= "$key $RES{1}{$key}{$test}\n";
}
$RESULT{scaling} .= sprintf "\n";
}
foreach my $nm ( 1 ... $numDomainsPerNode ) {
my $tag = "s$nm";
$DPLOT{meta} .=<<"END";
\@ $tag hidden false
\@ $tag type xy
\@ $tag symbol 1
\@ $tag symbol size 1.000000
\@ $tag symbol color $nm
\@ $tag symbol pattern 1
\@ $tag symbol fill color $nm
\@ $tag symbol fill pattern 1
\@ $tag symbol linewidth 1.0
\@ $tag symbol linestyle 1
\@ $tag symbol char 65
\@ $tag symbol char font 0
\@ $tag symbol skip 0
\@ $tag line type 1
\@ $tag line linestyle 1
\@ $tag line linewidth 4.0
\@ $tag line color $nm
\@ $tag line pattern 1
\@ $tag baseline type 0
\@ $tag baseline off
\@ $tag dropline off
\@ $tag fill type 0
\@ $tag fill rule 0
\@ $tag fill color 1
\@ $tag fill pattern 1
\@ $tag avalue off
\@ $tag avalue type 2
\@ $tag avalue char size 1.000000
\@ $tag avalue font 0
\@ $tag avalue color 1
\@ $tag avalue rot 0
\@ $tag avalue format general
\@ $tag avalue prec 3
\@ $tag avalue prepend ""
\@ $tag avalue append ""
\@ $tag avalue offset 0.000000 , 0.000000
\@ $tag errorbar on
\@ $tag errorbar place both
\@ $tag errorbar color $nm
\@ $tag errorbar pattern 1
\@ $tag errorbar size 1.000000
\@ $tag errorbar linewidth 1.0
\@ $tag errorbar linestyle 1
\@ $tag errorbar riser linewidth 1.0
\@ $tag errorbar riser linestyle 1
\@ $tag errorbar riser clip off
\@ $tag errorbar riser clip length 0.100000
\@ $tag legend "$nm"
END
}
$RESULT{node} =~ /([0-9.]+)/;
my $ymax = $1 * 1.2;
$DPLOT{world} = "1, 0, $numCoresPerMemoryDomain, $ymax";
foreach my $test ( @testcases ) {
my %dplottmp;
$RESULT{'domain'.$test} = sprintf "#nm";
foreach my $nm ( 1 ... $numDomainsPerNode ) {
$RESULT{'domain'.$test} .= sprintf "\t%d", $nm;
$dplottmp{$nm} = '';
}
$RESULT{'domain'.$test} .= sprintf "\n";
foreach my $nt ( 1 ... $numCoresPerMemoryDomain ) {
$RESULT{'domain'.$test} .= sprintf "%d", $nt;
foreach my $nm ( 1 ... $numDomainsPerNode ) {
$RESULT{'domain'.$test} .= sprintf "\t%.2f", $RES{$nm}{$nt}{$test};
$dplottmp{$nm} .= "$nt $RES{$nm}{$nt}{$test}\n";
}
$RESULT{'domain'.$test} .= sprintf "\n";
}
$DPLOT{series} = '';
foreach my $nm ( 1 ... $numDomainsPerNode ) {
$DPLOT{series} .= "\@target G0.S$nm\n\@type xy\n";
$DPLOT{series} .= $dplottmp{$nm};
$DPLOT{series} .= "&\n";
}
generateDomainScalingPlot("$DIR/$TAG-domains$test.agr");
}
$RESULT{domain} =~ /([0-9]+\.)/;
$ymax = $1 * 1.2;
$RESULT{scalingPlot} = "figures/$TAG-scaling.png";
$RESULT{domainInitPlot} = "figures/$TAG-domainsInit.png";
$RESULT{domainSumPlot} = "figures/$TAG-domainsSum.png";
$RESULT{domainCopyPlot} = "figures/$TAG-domainsCopy.png";
$RESULT{domainTriadPlot} = "figures/$TAG-domainsTriad.png";
$PLOT{world} = "1, 0, $numCoresPerMemoryDomain, $ymax";
generateMarkdown("$DIR/$TAG.md");
generateScalingPlot("$DIR/$TAG-scaling.agr");
open(my $fh, '>', "$DIR/max.dat") or die "Could not open file $!";
$RESULT{core} =~ /([0-9.]+)/;
print $fh "CORE $1\n";
$RESULT{socket} =~ /([0-9.]+)/;
print $fh "SOCKET $1\n";
$RESULT{node} =~ /([0-9.]+)/;
print $fh "NODE $1\n";
close $fh;
`tar -czf $TAG.tgz $DIR`;
# Helper routines
sub findMaxResult {
my $maxDomains = shift;
my $maxCores = shift;
my %param;
my $max = 0;
foreach my $numDomains ( 1 ... $maxDomains ) {
foreach my $numCoresPerDomain ( 1 ... $maxCores ) {
foreach my $testcase (keys %{$RES{$numDomains}{$numCoresPerDomain}}) {
if ( $RES{$numDomains}{$numCoresPerDomain}{$testcase} > $max ){
$max = sprintf "\t%.2f",$RES{$numDomains}{$numCoresPerDomain}{$testcase};
$param{testcase} = $testcase;
$param{cores} = $numCoresPerDomain;
}
}
}
}
if ($maxCores == 1) {
return "$max ($param{testcase})";
} else {
return "$max ($param{testcase} with $param{cores} cores)";
}
}
sub extractResults {
while( defined( my $file = glob($DIR . '/raw/*' ) ) ) {
print "Process $file\n";
my $nt = 1;
my $nm = 1;
open(my $fh, "<","$file");
if ($file =~ /.*-M([0-9]+)-([0-9]+)\.txt/) {
$nm = $1;
$nt = $2;
}
# $RES{$nm}{$nt} = {};
while ( <$fh> ) {
my $cnt = split(/[ ]+/, $_);
if ( $cnt == 6 ) {
my @fields = split(/[ ]+/, $_);
if ( $fields[1] =~ /[0-9]+/ ) {
$fields[0] =~ s/://;
$RES{$nm}{$nt}{$fields[0]} = $fields[1] * 0.001;
}
}
}
close $fh or die "can't close file $!";
}
}
sub generateScalingPlot {
my $filename = shift;
open(my $fh, '>', $filename) or die "Could not open file '$filename' $!";
print $fh <<"END";
# Grace project file
#
\@version 50122
\@page size 792, 612
\@page scroll 5%
\@page inout 5%
\@link page off
\@map font 0 to "Times-Roman", "Times-Roman"
\@map font 1 to "Times-Italic", "Times-Italic"
\@map font 2 to "Times-Bold", "Times-Bold"
\@map font 3 to "Times-BoldItalic", "Times-BoldItalic"
\@map font 4 to "Helvetica", "Helvetica"
\@map font 5 to "Helvetica-Oblique", "Helvetica-Oblique"
\@map font 6 to "Helvetica-Bold", "Helvetica-Bold"
\@map font 7 to "Helvetica-BoldOblique", "Helvetica-BoldOblique"
\@map font 8 to "Courier", "Courier"
\@map font 9 to "Courier-Oblique", "Courier-Oblique"
\@map font 10 to "Courier-Bold", "Courier-Bold"
\@map font 11 to "Courier-BoldOblique", "Courier-BoldOblique"
\@map font 12 to "Symbol", "Symbol"
\@map font 13 to "ZapfDingbats", "ZapfDingbats"
\@map color 0 to (255, 255, 255), "white"
\@map color 1 to (0, 0, 0), "black"
\@map color 2 to (255, 0, 0), "red"
\@map color 3 to (0, 255, 0), "green"
\@map color 4 to (0, 0, 255), "blue"
\@map color 5 to (255, 255, 0), "yellow"
\@map color 6 to (188, 143, 143), "brown"
\@map color 7 to (220, 220, 220), "grey"
\@map color 8 to (148, 0, 211), "violet"
\@map color 9 to (0, 255, 255), "cyan"
\@map color 10 to (255, 0, 255), "magenta"
\@map color 11 to (255, 165, 0), "orange"
\@map color 12 to (114, 33, 188), "indigo"
\@map color 13 to (103, 7, 72), "maroon"
\@map color 14 to (64, 224, 208), "turquoise"
\@map color 15 to (0, 139, 0), "green4"
\@reference date 0
\@date wrap off
\@date wrap year 1950
\@default linewidth 1.0
\@default linestyle 1
\@default color 1
\@default pattern 1
\@default font 0
\@default char size 1.000000
\@default symbol size 1.000000
\@default sformat "%.8g"
\@background color 0
\@page background fill on
\@timestamp off
\@timestamp 0.03, 0.03
\@timestamp color 1
\@timestamp rot 0
\@timestamp font 0
\@timestamp char size 1.000000
\@timestamp def "Tue Dec 15 13:24:48 2020"
\@r0 off
\@link r0 to g0
\@r0 type above
\@r0 linestyle 1
\@r0 linewidth 1.0
\@r0 color 1
\@r0 line 0, 0, 0, 0
\@r1 off
\@link r1 to g0
\@r1 type above
\@r1 linestyle 1
\@r1 linewidth 1.0
\@r1 color 1
\@r1 line 0, 0, 0, 0
\@r2 off
\@link r2 to g0
\@r2 type above
\@r2 linestyle 1
\@r2 linewidth 1.0
\@r2 color 1
\@r2 line 0, 0, 0, 0
\@r3 off
\@link r3 to g0
\@r3 type above
\@r3 linestyle 1
\@r3 linewidth 1.0
\@r3 color 1
\@r3 line 0, 0, 0, 0
\@r4 off
\@link r4 to g0
\@r4 type above
\@r4 linestyle 1
\@r4 linewidth 1.0
\@r4 color 1
\@r4 line 0, 0, 0, 0
\@g0 on
\@g0 hidden false
\@g0 type XY
\@g0 stacked false
\@g0 bar hgap 0.000000
\@g0 fixedpoint off
\@g0 fixedpoint type 0
\@g0 fixedpoint xy 0.000000, 0.000000
\@g0 fixedpoint format general general
\@g0 fixedpoint prec 6, 6
\@with g0
\@ world $PLOT{world}
\@ stack world 0, 1, 0, 1
\@ znorm 1
\@ view 0.150000, 0.150000, 1.150000, 0.850000
\@ title ""
\@ title font 0
\@ title size 1.500000
\@ title color 1
\@ subtitle ""
\@ subtitle font 0
\@ subtitle size 1.000000
\@ subtitle color 1
\@ xaxes scale Normal
\@ yaxes scale Normal
\@ xaxes invert off
\@ yaxes invert off
\@ xaxis on
\@ xaxis type zero false
\@ xaxis offset 0.000000 , 0.000000
\@ xaxis bar on
\@ xaxis bar color 1
\@ xaxis bar linestyle 1
\@ xaxis bar linewidth 1.0
\@ xaxis label "number of cores"
\@ xaxis label layout para
\@ xaxis label place auto
\@ xaxis label char size 1.500000
\@ xaxis label font 0
\@ xaxis label color 1
\@ xaxis label place normal
\@ xaxis tick on
\@ xaxis tick major 1
\@ xaxis tick minor ticks 0
\@ xaxis tick default 6
\@ xaxis tick place rounded true
\@ xaxis tick in
\@ xaxis tick major size 1.000000
\@ xaxis tick major color 1
\@ xaxis tick major linewidth 1.0
\@ xaxis tick major linestyle 1
\@ xaxis tick major grid off
\@ xaxis tick minor color 1
\@ xaxis tick minor linewidth 1.0
\@ xaxis tick minor linestyle 1
\@ xaxis tick minor grid off
\@ xaxis tick minor size 0.500000
\@ xaxis ticklabel on
\@ xaxis ticklabel format general
\@ xaxis ticklabel prec 5
\@ xaxis ticklabel formula ""
\@ xaxis ticklabel append ""
\@ xaxis ticklabel prepend ""
\@ xaxis ticklabel angle 0
\@ xaxis ticklabel skip 0
\@ xaxis ticklabel stagger 0
\@ xaxis ticklabel place normal
\@ xaxis ticklabel offset auto
\@ xaxis ticklabel offset 0.000000 , 0.010000
\@ xaxis ticklabel start type auto
\@ xaxis ticklabel start 0.000000
\@ xaxis ticklabel stop type auto
\@ xaxis ticklabel stop 0.000000
\@ xaxis ticklabel char size 1.250000
\@ xaxis ticklabel font 0
\@ xaxis ticklabel color 1
\@ xaxis tick place normal
\@ xaxis tick spec type none
\@ yaxis on
\@ yaxis type zero false
\@ yaxis offset 0.000000 , 0.000000
\@ yaxis bar on
\@ yaxis bar color 1
\@ yaxis bar linestyle 1
\@ yaxis bar linewidth 1.0
\@ yaxis label "Memory bandwidth [GB/s]"
\@ yaxis label layout para
\@ yaxis label place auto
\@ yaxis label char size 1.500000
\@ yaxis label font 0
\@ yaxis label color 1
\@ yaxis label place normal
\@ yaxis tick on
\@ yaxis tick major 10
\@ yaxis tick minor ticks 1
\@ yaxis tick default 6
\@ yaxis tick place rounded true
\@ yaxis tick in
\@ yaxis tick major size 1.000000
\@ yaxis tick major color 1
\@ yaxis tick major linewidth 1.0
\@ yaxis tick major linestyle 1
\@ yaxis tick major grid on
\@ yaxis tick minor color 7
\@ yaxis tick minor linewidth 1.0
\@ yaxis tick minor linestyle 1
\@ yaxis tick minor grid on
\@ yaxis tick minor size 0.500000
\@ yaxis ticklabel on
\@ yaxis ticklabel format general
\@ yaxis ticklabel prec 5
\@ yaxis ticklabel formula ""
\@ yaxis ticklabel append ""
\@ yaxis ticklabel prepend ""
\@ yaxis ticklabel angle 0
\@ yaxis ticklabel skip 0
\@ yaxis ticklabel stagger 0
\@ yaxis ticklabel place normal
\@ yaxis ticklabel offset auto
\@ yaxis ticklabel offset 0.000000 , 0.010000
\@ yaxis ticklabel start type auto
\@ yaxis ticklabel start 0.000000
\@ yaxis ticklabel stop type auto
\@ yaxis ticklabel stop 0.000000
\@ yaxis ticklabel char size 1.250000
\@ yaxis ticklabel font 0
\@ yaxis ticklabel color 1
\@ yaxis tick place both
\@ yaxis tick spec type none
\@ altxaxis off
\@ altyaxis off
\@ legend on
\@ legend loctype view
\@ legend 0.93, 0.45
\@ legend box color 1
\@ legend box pattern 1
\@ legend box linewidth 1.0
\@ legend box linestyle 1
\@ legend box fill color 0
\@ legend box fill pattern 1
\@ legend font 0
\@ legend char size 1.000000
\@ legend color 1
\@ legend length 4
\@ legend vgap 1
\@ legend hgap 1
\@ legend invert false
\@ frame type 0
\@ frame linestyle 1
\@ frame linewidth 1.0
\@ frame color 1
\@ frame pattern 1
\@ frame background color 0
\@ frame background pattern 0
\@ s0 hidden false
\@ s0 type xy
\@ s0 symbol 1
\@ s0 symbol size 1.000000
\@ s0 symbol color 1
\@ s0 symbol pattern 1
\@ s0 symbol fill color 1
\@ s0 symbol fill pattern 1
\@ s0 symbol linewidth 1.0
\@ s0 symbol linestyle 1
\@ s0 symbol char 65
\@ s0 symbol char font 0
\@ s0 symbol skip 0
\@ s0 line type 1
\@ s0 line linestyle 1
\@ s0 line linewidth 4.0
\@ s0 line color 1
\@ s0 line pattern 1
\@ s0 baseline type 0
\@ s0 baseline off
\@ s0 dropline off
\@ s0 fill type 0
\@ s0 fill rule 0
\@ s0 fill color 1
\@ s0 fill pattern 1
\@ s0 avalue off
\@ s0 avalue type 2
\@ s0 avalue char size 1.000000
\@ s0 avalue font 0
\@ s0 avalue color 1
\@ s0 avalue rot 0
\@ s0 avalue format general
\@ s0 avalue prec 3
\@ s0 avalue prepend ""
\@ s0 avalue append ""
\@ s0 avalue offset 0.000000 , 0.000000
\@ s0 errorbar on
\@ s0 errorbar place both
\@ s0 errorbar color 1
\@ s0 errorbar pattern 1
\@ s0 errorbar size 1.000000
\@ s0 errorbar linewidth 1.0
\@ s0 errorbar linestyle 1
\@ s0 errorbar riser linewidth 1.0
\@ s0 errorbar riser linestyle 1
\@ s0 errorbar riser clip off
\@ s0 errorbar riser clip length 0.100000
\@ s0 legend "Init"
\@ s1 hidden false
\@ s1 type xy
\@ s1 symbol 1
\@ s1 symbol size 1.000000
\@ s1 symbol color 2
\@ s1 symbol pattern 1
\@ s1 symbol fill color 2
\@ s1 symbol fill pattern 1