Commit 819bd44e authored by David Peter's avatar David Peter
Browse files

Update quant section

parent 98ccd279
No preview for this file type
......@@ -67,7 +67,7 @@ Finally, we look at the resource consumption of the multi-exit model. Of course,
\begin{center}
\begin{tabular}{lccc}
\toprule
Exit & \# Parameters & \# \glspl{madds} & \# \glspl{madds} \\
Exit & Parameters & \glspl{madds} & \glspl{madds} \\
& & & Cumulative \\
\midrule
Exit 1 & \SI{30.73}{\kilo\nothing} & \SI{1.59}{\mega\nothing} & \SI{5.35}{\mega\nothing} \\
......
%% Cell type:code id: tags:
``` python
import os
import torch
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
from matplotlib import pyplot
import json
import numpy as np
from glob import glob
from collections import ChainMap
%matplotlib inline
import yaml
import math
from pathlib import Path
from tensorboard.backend.event_processing.event_accumulator import EventAccumulator
import pytorch_lightning.utilities.argparse_utils
import math
```
%% Cell type:code id: tags:
``` python
out_path = '/Users/david/Git/mastersthesis/results/plots/'
os.chdir('/Users/david/Git/kws-nas/models/')
os.getcwd()
```
%% Output
'/Users/david/Git/kws-nas/models'
%% Cell type:code id: tags:
``` python
# Return average of values in python list x
def avg(x):
return sum(x) / len(x)
```
%% Cell type:code id: tags:
``` python
# Load results from models
def load_model_results(path):
if not os.path.exists(path):
raise FileNotFoundError
path = os.path.join(path, '**/learned_net/')
path = os.path.normpath(path)
model_results = {}
for p in glob(path, recursive=True):
model_name = p.split(os.sep)[-2]
if model_name[-1] == '_':
continue
# Get net.config
net_config_path = os.path.join(p, 'net.config')
with open(net_config_path, 'r') as net_config:
net_config = json.load(net_config)
# Get run.config
run_config_path = os.path.join(p, 'run.config')
with open(run_config_path, 'r') as run_config:
run_config = json.load(run_config)
# Get net_info.txt
net_info_path = os.path.join(p, 'logs', 'net_info.txt')
with open(net_info_path, 'r') as net_info:
net_info = json.load(net_info)
# Get outputs of different runs
output = {}
run_output_paths = glob(os.path.join(p, 'output_run*'))
for run_output_path in run_output_paths:
with open(run_output_path, 'r') as run_output:
run_output = json.load(run_output)
if not output:
output = {k:[float(v)] for k, v in run_output.items()}
else:
for k, v in run_output.items():
output[k].append(float(v))
# Insert results into dict which is later returned
model_results[model_name] = ChainMap(run_config, net_info, net_config, output)
return model_results
```
%% Cell type:code id: tags:
``` python
# Load results from models
def load_model_results_dyn(path):
if not os.path.exists(path):
raise FileNotFoundError
path = os.path.normpath(path)
model_name = path.split(os.sep)[-1]
# Get act_summary
act_summary_path = os.path.join(path, 'act_summary.json')
with open(act_summary_path, 'r') as f:
act_summary = json.load(f)
# Get weights_summary
weights_summary_path = os.path.join(path, 'weights_summary.json')
with open(weights_summary_path, 'r') as f:
weights_summary = json.load(f)
# Get hparams
hparams_path = os.path.join(path, 'hparams.yaml')
with open(hparams_path, 'r') as f:
hparams = yaml.load(f, Loader=yaml.BaseLoader)
# Insert results into dict which is later returned
model_results = {'act_summary': act_summary, 'weights_summary': weights_summary}
return model_results
```
%% Cell type:code id: tags:
``` python
def load_tf_results(path):
path = Path(path)
exp_data = []
exp_hparams = []
exp_results = []
exp_folder_list = [f for f in path.glob(path.parts[-1] + '*')]
for exp_folder in exp_folder_list:
version_folder_list = [f for f in exp_folder.glob('*') if str(f.parts[-1])[0] != '.']
version_data = []
version_results = []
for version_folder in version_folder_list:
events_file_list = [f for f in version_folder.glob('events*')]
hparams_file = version_folder / 'hparams.yaml'
events_data = []
for events_file in events_file_list:
event_acc = EventAccumulator(str(events_file))
event_acc.Reload()
events_data.append(event_acc)
version_data.append(events_data)
version_results.append(load_model_results_dyn(str(version_folder)))
with open(hparams_file, 'r') as f:
hparams = yaml.load(f, Loader=yaml.FullLoader)
exp_hparams.append(hparams)
exp_data.append(version_data)
exp_results.append(version_results)
assert len(exp_data) == len(exp_hparams) == len(exp_results)
return exp_data, exp_hparams, exp_results
```
%% Cell type:code id: tags:
``` python
model_results = load_model_results("pless_train_quantized")
model_results_v0 = load_model_results("pless_train_quantized_v0")
```
%% Cell type:code id: tags:
``` python
# Fixed bit width
weight = []
act = []
test_acc = []
weight_mem = []
act_mem = []
for name, result in model_results.items():
if result['activation_bit_width'] is None or result['weight_bit_width'] is None:
continue
if result['activation_bit_width'] == 32 or result['weight_bit_width'] == 32:
continue
if result['activation_bit_width'] is None:
result['activation_bit_width'] = 0
if result['weight_bit_width'] is None:
result['weight_bit_width'] = 0
if result['activation_bit_width'] == 1: # Use 1 bit values from first run
tmp_result = model_results_v0[f'pless_train_quantized_{result["weight_bit_width"]}W1A']
if result["weight_bit_width"] == 0:
tmp_result = model_results_v0[f'pless_train_quantized_1A']
else:
tmp_result = model_results_v0[f'pless_train_quantized_{result["weight_bit_width"]}W1A']
test_acc.append(avg(tmp_result['test_acc']))
else:
test_acc.append(avg(result['test_acc']))
act.append(result['activation_bit_width'])
weight.append(result['weight_bit_width'])
if result['activation_bit_width'] == 0:
act.append(32)
else:
act.append(result['activation_bit_width'])
if result['weight_bit_width'] == 0:
weight.append(32)
else:
weight.append(result['weight_bit_width'])
weight_mem.append(result['weight_memory'])
act_mem.append(result['act_memory'])
# Dynamic bit width
dyn_quant_data, dyn_quant_hparams, dyn_quant_results = load_tf_results('dyn_quant_combined_v3')
for exp_num in range(len(dyn_quant_results)):
version_acts = []
version_weights = []
version_act_mem = []
version_weight_mem = []
version_test_accs = []
for version_num in range(len(dyn_quant_results[exp_num])):
# Get test accuracies from tensorflow events file
try:
ta = dyn_quant_data[exp_num][version_num][0].Scalars('test_acc')[-1][2] * 100
except:
ta = dyn_quant_data[exp_num][version_num][1].Scalars('test_acc')[-1][2] * 100
try:
ab = dyn_quant_data[exp_num][version_num][0].Scalars('avg_activation_bit_width')[-1][