%% LaTeX2e file `references.bib'
%% generated by the `filecontents' environment
%% from source `template' on 2024/08/08.
%%
@book{goodfellow2016deep,
  title={Deep learning},
  author={Goodfellow, Ian and Bengio, Yoshua and Courville, Aaron and Bengio, Yoshua},
  volume={1},
  year={2016},
  publisher={MIT Press}
}

@article{power2022grokking,
  title={Grokking: Generalization beyond overfitting on small algorithmic datasets},
  author={Power, Alethea and Burda, Yuri and Edwards, Harri and Babuschkin, Igor and Misra, Vedant},
  journal={arXiv preprint arXiv:2201.02177},
  year={2022}
}

@article{vaswani2017attention,
  title={Attention is all you need},
  author={Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, {\L}ukasz and Polosukhin, Illia},
  journal={Advances in neural information processing systems},
  volume={30},
  year={2017}
}

@article{kingma2014adam,
  title={Adam: A method for stochastic optimization},
  author={Kingma, Diederik P and Ba, Jimmy},
  journal={arXiv preprint arXiv:1412.6980},
  year={2014}
}

@article{ba2016layer,
  title={Layer normalization},
  author={Ba, Jimmy Lei and Kiros, Jamie Ryan and Hinton, Geoffrey E},
  journal={arXiv preprint arXiv:1607.06450},
  year={2016}
}

@article{loshchilov2017adamw,
  title={Decoupled weight decay regularization},
  author={Loshchilov, Ilya and Hutter, Frank},
  journal={arXiv preprint arXiv:1711.05101},
  year={2017}
}

@article{radford2019language,
  title={Language Models are Unsupervised Multitask Learners},
  author={Radford, Alec and Wu, Jeff and Child, Rewon and Luan, David and Amodei, Dario and Sutskever, Ilya},
  year={2019}
}

@article{bahdanau2014neural,
  title={Neural machine translation by jointly learning to align and translate},
  author={Bahdanau, Dzmitry and Cho, Kyunghyun and Bengio, Yoshua},
  journal={arXiv preprint arXiv:1409.0473},
  year={2014}
}

@article{paszke2019pytorch,
  title={Pytorch: An imperative style, high-performance deep learning library},
  author={Paszke, Adam and Gross, Sam and Massa, Francisco and Lerer, Adam and Bradbury, James and Chanan, Gregory and Killeen, Trevor and Lin, Zeming and Gimelshein, Natalia and Antiga, Luca and others},
  journal={Advances in neural information processing systems},
  volume={32},
  year={2019}
}

@Article{Shea2024WhyLS,
 author = {Betty Shea and Mark Schmidt},
 booktitle = {arXiv.org},
 journal = {ArXiv},
 title = {Why Line Search when you can Plane Search? SO-Friendly Neural Networks allow Per-Iteration Optimization of Learning and Momentum Rates for Every Layer},
 volume = {abs/2406.17954},
 year = {2024}
}


@Article{Bahdanau2014NeuralMT,
 author = {Dzmitry Bahdanau and Kyunghyun Cho and Yoshua Bengio},
 booktitle = {International Conference on Learning Representations},
 journal = {CoRR},
 title = {Neural Machine Translation by Jointly Learning to Align and Translate},
 volume = {abs/1409.0473},
 year = {2014}
}


@Article{Gupta2018LISAER,
 author = {Pankaj Gupta and Hinrich Schütze},
 booktitle = {BlackboxNLP@EMNLP},
 pages = {154-164},
 title = {LISA: Explaining Recurrent Neural Network Judgments via Layer-wIse Semantic Accumulation and Example to Pattern Transformation},
 year = {2018}
}


@Article{Ko2022NotAL,
 author = {Yunyong Ko and Dongwon Lee and Sang-Wook Kim},
 booktitle = {The Web Conference},
 journal = {Proceedings of the ACM Web Conference 2022},
 title = {Not All Layers Are Equal: A Layer-Wise Adaptive Approach Toward Large-Scale DNN Training},
 year = {2022}
}


@Article{Ko2022NotAL,
 author = {Yunyong Ko and Dongwon Lee and Sang-Wook Kim},
 booktitle = {The Web Conference},
 journal = {Proceedings of the ACM Web Conference 2022},
 title = {Not All Layers Are Equal: A Layer-Wise Adaptive Approach Toward Large-Scale DNN Training},
 year = {2022}
}


@Article{Hu2021LoRALA,
 author = {J. E. Hu and Yelong Shen and Phillip Wallis and Zeyuan Allen-Zhu and Yuanzhi Li and Shean Wang and Weizhu Chen},
 booktitle = {International Conference on Learning Representations},
 journal = {ArXiv},
 title = {LoRA: Low-Rank Adaptation of Large Language Models},
 volume = {abs/2106.09685},
 year = {2021}
}


@Article{Shea2024WhyLS,
 author = {Betty Shea and Mark Schmidt},
 booktitle = {arXiv.org},
 journal = {ArXiv},
 title = {Why Line Search when you can Plane Search? SO-Friendly Neural Networks allow Per-Iteration Optimization of Learning and Momentum Rates for Every Layer},
 volume = {abs/2406.17954},
 year = {2024}
}


@Article{Ko2022NotAL,
 author = {Yunyong Ko and Dongwon Lee and Sang-Wook Kim},
 booktitle = {The Web Conference},
 journal = {Proceedings of the ACM Web Conference 2022},
 title = {Not All Layers Are Equal: A Layer-Wise Adaptive Approach Toward Large-Scale DNN Training},
 year = {2022}
}


@Article{Hu2021LoRALA,
 author = {J. E. Hu and Yelong Shen and Phillip Wallis and Zeyuan Allen-Zhu and Yuanzhi Li and Shean Wang and Weizhu Chen},
 booktitle = {International Conference on Learning Representations},
 journal = {ArXiv},
 title = {LoRA: Low-Rank Adaptation of Large Language Models},
 volume = {abs/2106.09685},
 year = {2021}
}


@Article{Ko2022NotAL,
 author = {Yunyong Ko and Dongwon Lee and Sang-Wook Kim},
 booktitle = {The Web Conference},
 journal = {Proceedings of the ACM Web Conference 2022},
 title = {Not All Layers Are Equal: A Layer-Wise Adaptive Approach Toward Large-Scale DNN Training},
 year = {2022}
}


@Article{Bahamou2023LayerwiseAS,
 author = {Achraf Bahamou and D. Goldfarb},
 booktitle = {arXiv.org},
 journal = {ArXiv},
 title = {Layer-wise Adaptive Step-Sizes for Stochastic First-Order Methods for Deep Learning},
 volume = {abs/2305.13664},
 year = {2023}
}


@Article{Bahamou2023LayerwiseAS,
 author = {Achraf Bahamou and D. Goldfarb},
 booktitle = {arXiv.org},
 journal = {ArXiv},
 title = {Layer-wise Adaptive Step-Sizes for Stochastic First-Order Methods for Deep Learning},
 volume = {abs/2305.13664},
 year = {2023}
}


@Article{Kingma2014AdamAM,
 author = {Diederik P. Kingma and Jimmy Ba},
 booktitle = {International Conference on Learning Representations},
 journal = {CoRR},
 title = {Adam: A Method for Stochastic Optimization},
 volume = {abs/1412.6980},
 year = {2014}
}


@Article{Gitman2018ConvergenceAO,
 author = {Igor Gitman and D. Dilipkumar and Ben Parr},
 booktitle = {arXiv.org},
 journal = {ArXiv},
 title = {Convergence Analysis of Gradient Descent Algorithms with Proportional Updates},
 volume = {abs/1801.03137},
 year = {2018}
}

