Commit e523e438 authored by Carlos Galindo's avatar Carlos Galindo
Browse files

run.sh: analyze_results can be specific and won't fail for partial benchmarks

Partial benchmarks: only timing or only slices.
parent 8ea04ef3
Loading
Loading
Loading
Loading
+94 −54
Original line number Diff line number Diff line
@@ -80,13 +80,41 @@ EOF

analyze_results() {
	TMPDIR=/tmp/e-knife-csv
	PWD=$(pwd)
	pwd=$(pwd)

	rm -rf $TMPDIR
	mkdir -p $TMPDIR

  if [[ $# -eq 1 ]]; then
    cp -r "$1" $TMPDIR
  else
    cp -r benchmark_result* $TMPDIR
  fi
	cd $TMPDIR || return

  find . -type d -empty -delete
	has_slices=1
	for i in benchmark_result*; do
	  if [[ -d $i/Slices ]]; then
	    has_slices=0;
	    break;
    fi
  done
  has_times_slices=1
  for i in benchmark_result*; do
    if [[ -d $i/Times/Slices ]]; then
      has_times_slices=0;
      break;
    fi
  done
	## Cache analysis: should output nothing, the same slices should be produced regardless of caching.
	if [[ $has_slices -eq 0 ]]; then
    for i in */Slices/*/cached_*.erl; do
      diff -q "$i" "${i//cached/computed}";
    done
  fi
  ## Timing analysis
  if [[ $has_times_slices -eq 0 ]]; then
    # Type of slicing algorithm
    # The two categories were inverted when running benchmarks
    sed -i 's/^/t,/' -- */Times/Slices/*_subsumption_*/*.txt
@@ -123,13 +151,15 @@ SELECT
FROM bench
GROUP BY algorithm, stack;
EOF
  fi

	## Analysis of precision
	if [[ $has_slices -eq 0 ]]; then
    # Get node count of slices with astCounter
	erlc -o $TMPDIR "$PWD"/e-Knife/src/test/resources/astCounter.erl
    erlc -W0 -o $TMPDIR "$pwd/e-Knife/src/test/resources/astCounter.erl"
    mkdir -p Results
    find . -maxdepth 2 -type d -name Slices -exec \
	  erl -pa "$PWD" -noshell -eval "astCounter:enterDeepestDir(\"{}/\", \"tabular\"), init:stop()" \;
      erl -pa "$pwd" -noshell -eval "astCounter:enterDeepestDir(\"{}/\", \"tabular\"), init:stop()" \;
    # Remove redundant information
    sed -i -e 's/^[^0-9]\+//' -e 's/[_;]/,/g' -e 's/,bench/,/' Results/tabular/*.txt
    # Shorten algorithm labels (swap them, they are switched in the code)
@@ -145,36 +175,46 @@ EOF
    echo "Size results"
    sqlite3 << EOF
CREATE TABLE bench(stack, algorithm, file, cache, sc, size, PRIMARY KEY(stack, algorithm, file, cache, sc));
CREATE TABLE ratio(stack, file, cache, sc, ratio, PRIMARY KEY(stack, file, cache, sc));
CREATE TABLE ratio(stack, file, sc, ratio, PRIMARY KEY(stack, file, sc));
.mode csv
.import size-results.csv bench
INSERT INTO ratio SELECT a.stack, a.file, a.cache, a.sc, (a.size - b.size) / CAST(a.size AS REAL) AS ratio
  FROM bench AS a JOIN bench AS b USING (stack, file, cache, sc) WHERE a.algorithm = 't' AND b.algorithm = 's';
INSERT INTO ratio SELECT a.stack, a.file, a.sc, (a.size - b.size) / CAST(a.size AS REAL) AS ratio
  FROM bench AS a JOIN bench AS b USING (stack, file, cache, sc) WHERE a.algorithm = 't' AND b.algorithm = 's' AND a.cache = 'Y';
.headers on
SELECT COUNT(CASE WHEN ratio < 0 THEN ratio END) as negative, COUNT(CASE WHEN ratio > 0 THEN ratio END) as positive, COUNT(CASE WHEN ratio = 0 THEN ratio END) as zero FROM ratio;
SELECT * FROM ratio WHERE ratio < 0;
SELECT stack, AVG(ratio) AS ratio FROM ratio GROUP BY stack;
SELECT stack, AVG(size) AS size FROM bench GROUP BY stack;
EOF
  fi
  # Remove temporary files
  rm -f time-results.csv size-results.csv
}

if [[ $# -lt 1 ]]; then
	echo "Usage: use at least one of the following commands."
	echo -e "\t$0 compile\n\t\tCompile the Java sources, to be run before [background_]run."
	echo -e "\t$0 run <STACK_SIZE>\n\t\tRun the benchmark with the given stack size."
	echo -e "\t$0 background_run <STACK_SIZE>\n\t\tLike run, but in the background."
  echo -e "\t$0 follow_logs <STACK_SIZE>\n\t\tFollow the execution of a given background task."
  echo -e "\t$0 define_erl_server\n\t\tCreates and launches a systemd service to host the Erlang server."
	echo -e "\t$0 destroy_erl_server\n\t\tStops the Erlang server."
	echo -e "\t$0 test_erl_server\n\t\tTest the connection between Java and Erlang (requires define_erl_server)."
	echo -e "\t$0 analyze_results\n\t\tProduce a table of average times with SQLite, based on previous runs."
	echo "Usage: use exactly one of the following commands."
	echo -e "\tcompile\n\t\tCompile the Java sources, to be run before [background_]run."
	echo -e "\trun <STACK_SIZE>\n\t\tRun the benchmark with the given stack size."
	echo -e "\tbackground_run <STACK_SIZE>\n\t\tLike run, but in the background."
  echo -e "\tfollow_logs <STACK_SIZE>\n\t\tFollow the execution of a given background task."
  echo -e "\tdefine_erl_server\n\t\tCreates and launches a systemd service to host the Erlang server."
	echo -e "\tdestroy_erl_server\n\t\tStops the Erlang server."
	echo -e "\ttest_erl_server\n\t\tTest the connection between Java and Erlang (requires define_erl_server)."
	echo -e "\tanalyze_results [benchmark_result_folder(s)]\n\t\tProduce a table of average times with SQLite, based on all or selected runs."
	exit 1
fi

case $1 in
	compile|define_erl_server|destroy_erl_server|test_erl_server|analyze_results)
	compile|define_erl_server|destroy_erl_server|test_erl_server)
		$1
		;;
  analyze_results)
    if [[ $# -eq 2 ]]; then
      $1 "$2"
    else
      $1
    fi
    ;;
	run|background_run|follow_logs)
		if [[ $# -lt 2 ]]; then
			echo "You have provided no stack size! Use $0 $1 <STACK_SIZE>"