@article{soufleri2021network,
  title = {Network Compression via Mixed Precision Quantization Using a Multi-Layer Perceptron for the Bit-Width Allocation},
  author = {Soufleri, Efstathia and Roy, Kaushik},
  journal = {IEEE Access},
  volume = {9},
  pages = {135059--135068},
  year = {2021},
  publisher = {IEEE},
  doi = {10.1109/ACCESS.2021.3116418},
  file = {soufleri2021network.pdf}
}

Deep Neural Networks (DNNs) are a powerful tool for solving complex tasks in many application domains. The high performance of DNNs demands significant computational resources, which might not always be available. Network quantization with mixed-precision across the layers can alleviate this high demand. However, determining layer-wise optimal bit-widths is non-trivial, as the search space is exponential. This article proposes a novel technique for allocating layer-wise bit-widths for a DNN using a multi-layer perceptron (MLP). The Kullback-Leibler (KL) divergence of the softmax outputs between the quantized and full precision network is used as the metric to quantify the quantization quality. We explore the relationship between the KL-divergence and the network size, and from our experiments observe that more aggressive quantization leads to higher divergence, and vice versa. The MLP is trained with layer-wise bit-widths as labels and their corresponding KL-divergence as the input. The MLP training set, i.e. the pairs of the layer-wise bit-widths and their corresponding KL-divergence, is collected using a Monte Carlo sampling of the exponential search space. We introduce a penalty term in the loss to ensure that the MLP learns to predict bit-widths resulting in the smallest network size. We show that the layer-wise bit-width predictions from the trained MLP result in reduced network size without degrading accuracy while achieving better or comparable results with SOTA work but with less computational overhead. Our method achieves up to 6x, 4x, 4x compression on VGG16, ResNet50, and GoogLeNet respectively, with no accuracy drop compared to the original full precision pretrained model, on the ImageNet dataset.

@inproceedings{kosta2022hyperx,
  title = {HyperX: A Hybrid RRAM-SRAM partitioned system for error recovery in memristive Xbars},
  author = {Kosta, Adarsh and Soufleri, Efstathia and Chakraborty, Indranil and Agrawal, Amogh and Ankit, Aayush and Roy, Kaushik},
  booktitle = {2022 Design, Automation \& Test in Europe Conference \& Exhibition (DATE)},
  pages = {88--91},
  year = {2022},
  organization = {IEEE},
  doi = {10.23919/DATE54114.2022.9774549}
}

Memristive crossbars based on Non-volatile Memory (NVM) technologies such as RRAM, have recently shown great promise for accelerating Deep Neural Networks (DNNs). They achieve this by performing efficient Matrix-Vector-Multiplications (MVMs) while offering dense on-chip storage and minimal off-chip data movement. However, their analog nature of computing introduces functional errors due to non-ideal RRAM devices, significantly degrading the application accuracy. Further, RRAMs suffer from low endurance and high write costs, hindering on-chip trainability. To alleviate these limitations, we propose HyperX, a hybrid RRAM-SRAM system that leverages the complementary benefits of NVM and CMOS technologies. Our proposed system consists of a fixed RRAM block offering area and energy-efficient MVMs and an SRAM block enabling on-chip training to recover the accuracy drop due to the RRAM non-idealities. The improvements are reported in terms of energy and product of latency and area (ms×mm2) , termed as area-normalized latency. Our experiments on CIFAR datasets using ResNet-20 show up to 2.88 × and 10.1 × improvements in inference energy and area-normalized latency, respectively. In addition, for a transfer learning task from ImageNet to CIFAR datasets using ResNet-18, we observe up to 1.58 × and 4.48 × improvements in energy and area-normalized latency, respectively. These improvements are with respect to an all-SRAM baseline.

@inproceedings{panda2019evaluating,
  title = {Evaluating the Stability of Recurrent Neural Models during Training with Eigenvalue Spectra Analysis},
  author = {Panda, Priyadarshini and Soufleri, Efstathia and Roy, Kaushik},
  booktitle = {2019 International Joint Conference on Neural Networks (IJCNN)},
  pages = {1--8},
  year = {2019},
  organization = {IEEE},
  doi = {10.1109/IJCNN.2019.8852181},
  file = {panda2019evaluating.pdf}
}

We analyze the stability of recurrent networks, specifically, reservoir computing models during training by evaluating the eigenvalue spectra of the reservoir dynamics. To circumvent the instability arising in examining a closed loop reservoir system with feedback, we propose to break the closed loop system. Essentially, we unroll the reservoir dynamics over time while incorporating the feedback effects that preserve the overall temporal integrity of the system. We evaluate our methodology for fixed point and time varying targets with least squares regression and FORCE training, respectively. Our analysis establishes eigenvalue spectra (which is, shrinking of spectral circle as training progresses) as a valid and effective metric to gauge the convergence of training as well as the convergence of the chaotic activity of the reservoir toward stable states.