diff --git a/examples/imagenette.py b/examples/imagenette.py index 6c3347c..332354a 100644 --- a/examples/imagenette.py +++ b/examples/imagenette.py @@ -28,7 +28,7 @@ except ImportError: FFCV = False -app = typer.Typer(context_settings={"help_option_names": ["-h", "--help"]}) +app = typer.Typer(context_settings={"help_option_names": ["-h", "--help"]}, pretty_exceptions_show_locals=False) imagenette_stats = ([0.465,0.458,0.429],[0.285,0.280,0.301]) imagewoof_stats = ([0.496,0.461,0.399],[0.257,0.249,0.258]) @@ -187,9 +187,9 @@ def get_aug_transforms(flip:bool=True, flip_vert:bool=False, max_rotate:float=10 if len(xtra_tfms) == 0: xtra_tfms = None - return *aug_transforms(do_flip=flip, flip_vert=flip_vert, max_rotate=max_rotate, min_zoom=min_zoom, - max_zoom=max_zoom, max_lighting=max_lighting, max_warp=max_warp, - p_affine=prob_affine, p_lighting=prob_lighting, xtra_tfms=xtra_tfms), + return aug_transforms(do_flip=flip, flip_vert=flip_vert, max_rotate=max_rotate, min_zoom=min_zoom, + max_zoom=max_zoom, max_lighting=max_lighting, max_warp=max_warp, + p_affine=prob_affine, p_lighting=prob_lighting, xtra_tfms=xtra_tfms) def get_fastai_dls(size:int, bs:int, imagenette:bool=False, max_workers:int=16, center_crop:bool=True, diff --git a/fastxtend/__init__.py b/fastxtend/__init__.py index bbab024..1276d02 100644 --- a/fastxtend/__init__.py +++ b/fastxtend/__init__.py @@ -1 +1 @@ -__version__ = "0.1.4" +__version__ = "0.1.5" diff --git a/fastxtend/text/huggingface.py b/fastxtend/text/huggingface.py index 0681c25..136c43c 100644 --- a/fastxtend/text/huggingface.py +++ b/fastxtend/text/huggingface.py @@ -59,9 +59,9 @@ class HuggingFaceCallback(Callback): run_valid = True "Applies `HuggingFaceWrapper` and handles using model's built in loss or fastai `Learner` loss" def __init__(self, - labels:str|None='labels', # Input batch labels key. Set to None if dataset doesn't contain labels + labels:str|None='labels', # Input batch labels key. Set to None if input doesn't contain labels loss:str='loss', # Model output loss key - logits:str='logits', # Model output logits key, + logits:str='logits', # Model output logits key unwrap:bool=True, # After training completes, unwrap the Transformers model ): self._label_key, self._loss_key = labels, loss @@ -104,7 +104,7 @@ def after_fit(self): # %% ../../nbs/text.huggingface.ipynb 11 class HuggingFaceLoader(_DataLoader): - "A minimal compatibility wrapper between a Hugging Face Dataloader and `Learner`" + "A minimal compatibility DataLoader between a Hugging Face and `Learner`" def __init__(self, dataset:Dataset, # dataset from which to load the data batch_size:int, # Batch size diff --git a/fastxtend/vision/all.py b/fastxtend/vision/all.py index 9191935..4e1993c 100644 --- a/fastxtend/vision/all.py +++ b/fastxtend/vision/all.py @@ -1,13 +1,14 @@ -from .augment import * -from .augment.all import * from ..basics import * from ..callback.all import * from .data import * from ..data.all import * from ..losses import * from ..metrics import * -from .models.all import * from ..multiloss import * from ..optimizer.all import * from ..schedulers import * -from ..utils import * \ No newline at end of file +from ..utils import * + +from .augment import * +from .augment.all import * +from .models.all import * \ No newline at end of file diff --git a/install.sh b/install.sh index e2d3b30..2731fa6 100644 --- a/install.sh +++ b/install.sh @@ -1,5 +1,5 @@ -conda create -n fastxtend python=3.10 "pytorch>=2.0.1" torchvision torchaudio \ -pytorch-cuda=11.8 cuda fastai nbdev pkg-config libjpeg-turbo opencv tqdm \ -terminaltables psutil numpy numba librosa=0.9.2 timm kornia rich typer \ -jupyterlab ipywidgets wandb \ --c pytorch -c nvidia/label/cuda-11.8.0 -c fastai -c huggingface -c conda-forge \ No newline at end of file +conda create -n fastxtend python=3.11 "pytorch>=2.1" torchvision torchaudio \ +pytorch-cuda=12.1 fastai nbdev pkg-config libjpeg-turbo opencv tqdm psutil \ +terminaltables numpy "numba>=0.57" librosa timm kornia rich typer wandb \ +"transformers>=4.34" "tokenizers>=0.14" "datasets>=2.14" ipykernel ipywidgets \ +"matplotlib<3.8" -c pytorch -c nvidia -c fastai -c huggingface -c conda-forge \ No newline at end of file diff --git a/nbs/_quarto.yml b/nbs/_quarto.yml index ba0e66b..79bdd93 100644 --- a/nbs/_quarto.yml +++ b/nbs/_quarto.yml @@ -12,12 +12,12 @@ format: toc-depth: 4 highlight-style: custom.theme html-math-method: katex - fontsize: 1.1em + fontsize: 1.1rem grid: sidebar-width: 275px - body-width: 1000px + body-width: 1025px margin-width: 275px - gutter-width: 2em + gutter-width: 3rem website: favicon: favicon.ico diff --git a/nbs/custom.scss b/nbs/custom.scss index 3d14b69..a40d758 100644 --- a/nbs/custom.scss +++ b/nbs/custom.scss @@ -274,4 +274,9 @@ div.description { padding-top: 5px; font-size: 135%; padding-bottom: 1rem; +} + +// table +.table { + font-size: 0.95rem !important; } \ No newline at end of file diff --git a/nbs/ffcv.tutorial.ipynb b/nbs/ffcv.tutorial.ipynb index 6792521..f2e595f 100644 --- a/nbs/ffcv.tutorial.ipynb +++ b/nbs/ffcv.tutorial.ipynb @@ -63,17 +63,17 @@ "The easiest way to install fastai, fastxtend, and FFCV is to use [Conda](https://docs.conda.io/en/latest/) or [Miniconda](https://docs.conda.io/en/latest/miniconda.html)[^solver] on Linux (or WSL):\n", "\n", "```bash\n", - "conda create -n fastxtend python=3.10 \"pytorch>=2.0.0\" \\\n", - "torchvision pytorch-cuda=11.8 cuda fastai pkg-config \\\n", + "conda create -n fastxtend python=3.11 \"pytorch>=2.1\" \\\n", + "torchvision pytorch-cuda=12.1 fastai pkg-config \\\n", "libjpeg-turbo opencv tqdm terminaltables psutil numpy \\\n", - "numba timm kornia -c pytorch -c nvidia/label/cuda-11.8.0 \\\n", + "\"numba>=0.57\" timm kornia -c pytorch -c nvidia \\\n", "-c fastai -c huggingface -c conda-forge\n", "\n", "# Switch to the newly created conda environment\n", "conda activate fastxtend\n", "```\n", "\n", - "replacing `pytorch-cuda=11.8` and `nvidia/label/cuda-11.8.0`[^cuda] with your prefered [supported version of Cuda](https://pytorch.org/get-started/locally). In rare[^rare] cases, you may need to add the `compilers` package to the conda install.\n", + "replacing `pytorch-cuda=12.1`[^cuda] with your prefered [supported version of Cuda](https://pytorch.org/get-started/locally). In rare[^rare] cases, you may need to add the `compilers` package to the conda install.\n", "\n", "And then install fastxtend via pip:\n", "\n", @@ -85,12 +85,13 @@ "Or to install with all of fastxtend's features:\n", "\n", "```bash\n", - "conda create -n fastxtend python=3.10 \"pytorch>=2.0.0\" \\\n", - "torchvision torchaudio pytorch-cuda=11.8 cuda fastai \\\n", - "nbdev pkg-config libjpeg-turbo opencv tqdm terminaltables \\\n", - "psutil numpy numba librosa=0.9.2 timm kornia rich typer \\\n", - "wandb -c pytorch -c nvidia/label/cuda-11.8.0 -c fastai \\\n", - "-c huggingface -c conda-forge\n", + "conda create -n fastxtend python=3.11 \"pytorch>=2.1\" \\\n", + "torchvision torchaudio pytorch-cuda=12.1 fastai nbdev \\\n", + "pkg-config libjpeg-turbo opencv tqdm psutil terminaltables \\\n", + "numpy \"numba>=0.57\" librosa timm kornia rich typer wandb \\\n", + "\"transformers>=4.34\" \"tokenizers>=0.14\" \"datasets>=2.14\" \\\n", + "ipykernel ipywidgets \"matplotlib<3.8\" -c pytorch -c nvidia \\\n", + "-c fastai -c huggingface -c conda-forge\n", "\n", "# Switch to the newly created conda environment\n", "conda activate fastxtend\n", @@ -113,7 +114,7 @@ "\n", "[^solver]: Miniconda with the faster [libmamba solver](https://www.anaconda.com/blog/a-faster-conda-for-a-growing-community) is recommended.\n", "\n", - "[^cuda]: I find that specifying the Cuda version label results in better enviroment solving.\n", + "[^cuda]: If you want to include a full Cuda install, I find that specifying the Cuda version label `-c nvidia/label/cuda-12.1.0` usually results in better enviroment solving. Sometimes the label doesn't have all the packages, so you'll need to add them manually to the conda install packages `nvidia::missing_pacakge`.\n", "\n", "[^rare]: The FFCV Linux [installation guide states](https://github.com/libffcv/ffcv#linux) the `compilers` package is rarely needed.\n", "\n", diff --git a/nbs/sidebar.yml b/nbs/sidebar.yml index 14a0e3b..033a885 100644 --- a/nbs/sidebar.yml +++ b/nbs/sidebar.yml @@ -7,7 +7,6 @@ website: - callback.channelslast.ipynb - callback.cutmixup.ipynb - callback.ema.ipynb - - callback.gradaccum.ipynb - callback.lr_finder.ipynb - text: Mixed Precision file: callback.amp.ipynb diff --git a/nbs/text.huggingface.ipynb b/nbs/text.huggingface.ipynb index 393d92c..a6060c1 100644 --- a/nbs/text.huggingface.ipynb +++ b/nbs/text.huggingface.ipynb @@ -62,7 +62,7 @@ "fastxtend provides basic compatibility for training Hugging Face [Transformers](https://huggingface.co/docs/transformers/index) models using the `fastai.learner.Learner`.\n", "\n", ":::{.callout-tip collapse=\"true\"}\n", - "#### Tip: Use blurr For Integrated Transformers with fastai\n", + "#### Tip: Use blurr For a Complete Transformers Integration\n", "\n", "[blurr](https://ohmeow.github.io/blurr) provides a complete Hugging Face Transformers integration with fastai, including working fastai datablocks, dataloaders, and other fastai methods.\n", "\n", @@ -74,6 +74,13 @@ "Jump to the [example](#example) section for a full tutorial.\n", "\n", "```python\n", + "from fastai.text.all import *\n", + "from fastxtend.text.all import *\n", + "\n", + "# load a task specific AutoModel\n", + "hf_model = AutoModel.from_pretrained(\"model-name\")\n", + "\n", + "# setup dataset and then dataloaders\n", "train_dataset = dataset['train'].with_format('torch')\n", "train_dataloader = HuggingFaceLoader(\n", " train_dataset, batch_size=batch_size,\n", @@ -81,16 +88,17 @@ " drop_last=True, num_workers=num_cpus()\n", ")\n", "\n", - "# defining the valid_dataloader cut for brevity\n", + "# valid_dataloader definition cut for brevity\n", "dls = DataLoaders(train_dataloader, valid_dataloader)\n", "\n", - "hf_model = GPTForCausalLM(...)\n", "learn = Learner(dls, hf_model, loss_func=HuggingFaceLoss(), ...,\n", " cbs=HuggingFaceCallback()).to_bf16()\n", "\n", + "# save the model after training using Transformers\n", + "learn.model.save_pretrained(\"trained-model-name\")\n", "```\n", "\n", - "To train with a different loss, pass in a loss to `Learner` as normal, and `HuggingFaceCallback` will ignore the Hugging Face model's built in loss calculation." + "To train with a different loss, pass in a PyTorch compatible loss to `Learner` as normal, and `HuggingFaceCallback` will use it instead of the model's built in loss." ] }, { @@ -159,9 +167,9 @@ " run_valid = True\n", " \"Applies `HuggingFaceWrapper` and handles using model's built in loss or fastai `Learner` loss\"\n", " def __init__(self,\n", - " labels:str|None='labels', # Input batch labels key. Set to None if dataset doesn't contain labels\n", + " labels:str|None='labels', # Input batch labels key. Set to None if input doesn't contain labels\n", " loss:str='loss', # Model output loss key\n", - " logits:str='logits', # Model output logits key,\n", + " logits:str='logits', # Model output logits key\n", " unwrap:bool=True, # After training completes, unwrap the Transformers model\n", " ):\n", " self._label_key, self._loss_key = labels, loss\n", @@ -207,13 +215,13 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "HuggingFaceCallback automatically wraps a Transformer model with the `HuggingFaceWrapper` for compatability with `fastai.learner.Learner`.\n", + "HuggingFaceCallback automatically wraps a Transformer model with the `HuggingFaceWrapper` for compatibility with `fastai.learner.Learner`.\n", "\n", "If `HuggingFaceLoss` is passed to `Learner`, then HuggingFaceCallback will use the Hugging Face model's built in loss.\n", "\n", "If any other loss function is passed to `Learner`, HuggingFaceCallback will prevent the built-in loss from being calculated and will use the `Learner` loss function instead.\n", "\n", - "If `labels=None`, then HuggingFaceCallback will not attempt to assign a fastai label from the Hugging Face input batch. The default fastai and fastxtend metrics will not work without targets.\n", + "If `labels=None`, then HuggingFaceCallback will not attempt to assign a fastai target from the Hugging Face input batch. The default fastai and fastxtend metrics will not work without labels.\n", "\n", "After training, the HuggingFaceCallback will automatically unwrap model. Set `unwrap=False` to keep the model wrapped in HuggingFaceWrapper." ] @@ -226,7 +234,7 @@ "source": [ "#|export\n", "class HuggingFaceLoader(_DataLoader):\n", - " \"A minimal compatibility wrapper between a Hugging Face Dataloader and `Learner`\"\n", + " \"A minimal compatibility DataLoader between a Hugging Face and `Learner`\"\n", " def __init__(self,\n", " dataset:Dataset, # dataset from which to load the data\n", " batch_size:int, # Batch size\n", @@ -268,7 +276,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Hugging Face Datasets, and thus DataLoaders, return dictionary objects while the `fastai.learner.Learner` expects tuples. HuggingFaceLoader is a PyTorch DataLoader which wraps the Hugging Face dictionary in a tuple for `Learner` compatibility. It is otherwise identical to the PyTorch DataLoader." + "Hugging Face Datasets, and thus DataLoaders, return dictionary objects while the `fastai.learner.Learner` expects tuples. HuggingFaceLoader is a PyTorch DataLoader which wraps the Hugging Face batch dictionary in a tuple for `Learner` compatibility. It is otherwise identical to a PyTorch DataLoader." ] }, { @@ -287,6 +295,7 @@ "#|hide\n", "#|cuda\n", "import os\n", + "import tempfile\n", "\n", "from datasets import concatenate_datasets, load_dataset\n", "from transformers import AutoModelForSequenceClassification, AutoTokenizer\n", @@ -399,7 +408,7 @@ "\n", "We need to use fastxtend's `HuggingFaceLoader` instead of the PyTorch `DataLoader`. HuggingFaceLoader is a simple wrapper around a PyTorch `DataLoader` which returns Transformer's dictionary batches in tuples as the `fastai.learner.Learner` expects. It is otherwise identical to the PyTorch DataLoader.\n", "\n", - "After creating the train and valid HuggingFaceLoader, we need to wrap them in `fastai.data.DataLoaders`." + "After creating the train and valid HuggingFaceLoader, we need to wrap them in `fastai.data.core.DataLoaders`." ] }, { @@ -431,16 +440,16 @@ "source": [ "### Create a Learner and Train\n", "\n", - "Finally, we'll create the `fastai.learner.Learner` to train DistilRoBERTa on IMDb. We'll pass in the `HuggingFaceCallback` to `cbs` to handle loss function compatibility between Transformers and fastai.\n", + "Finally, we'll create the `Learner` to train DistilRoBERTa on IMDb. We'll pass in the `HuggingFaceCallback` to `cbs` to handle loss function compatibility between Transformers and fastai.\n", "\n", "Transformer models contain an internal loss method, which we'll use by passing `HuggingFaceLoss` to `loss_func`.\n", "\n", ":::{.callout-note collapse=\"true\"}\n", "#### Note: Non-Default Transformers Keys\n", "\n", - "`HuggingFaceCallback` expects the Transformer model to have and output logits and loss keys. If these exist but are named differently, you'll need to pass the non-standard key names to HuggingFaceCallback.\n", + "HuggingFaceCallback expects the Transformer model to have and output logits and loss keys. If these exist but are named differently, you'll need to pass the non-standard key names to HuggingFaceCallback.\n", "\n", - "If your input doesn't have a label key, perhaps because you are pretraining a causal language model, you can set it to `None`.\n", + "If your input doesn't have a label key, perhaps because you are pretraining a causal language model, you should set it to `None`.\n", ":::\n", "\n", "We now can use any fastai and/or fastxtend callbacks, optimizers, or metrics to train our Transformers model like usual." @@ -550,7 +559,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "If we wanted to define our own loss, such as `nn.CrossEntropyLoss` with label smoothing, we could pass in any PyTorch compatible loss function to `Learner` and HuggingFaceCallback will automatically use it instead of DistilRoBERTa's internal loss function.\n", + "If we want to use our own loss, such as `nn.CrossEntropyLoss` with label smoothing, we could pass in any PyTorch compatible loss function to `Learner` and HuggingFaceCallback will automatically use it instead of DistilRoBERTa's internal loss function.\n", "\n", "In this example, we use fastxtend's `CompilerCallback` via the `Learner.compile` convenience method to accelerate training throughput using `torch.compile`. After compiling the model in the first epoch, training speed is increased, and memory usage is reduced. In this small example it's an overall loss, but we'd want to compile DistilRoBERTa if training on the entirety of IMDb.\n", "\n", @@ -657,7 +666,7 @@ "source": [ "### Saving the Model\n", "\n", - "After training, the HuggingFaceCallback will automatically unwrap our model, leaving `Learner.model` as the original Transformers model. \n", + "After training, the HuggingFaceCallback will automatically unwrap our model, leaving `Learner.model` as the original Transformers model.\n", "\n", "We use any Transformers method to save the model, such as `save_pretrained`." ] @@ -670,7 +679,6 @@ "source": [ "#|hide\n", "#|cuda\n", - "import tempfile\n", "temp_path = tempfile.TemporaryDirectory(dir=learn.path)\n", "model_path = temp_path.name" ] diff --git a/settings.ini b/settings.ini index 0ad3a5a..2605140 100644 --- a/settings.ini +++ b/settings.ini @@ -8,7 +8,7 @@ author = Benjamin Warner author_email = me@benjaminwarner.dev copyright = Benjamin Warner branch = main -version = 0.1.4 +version = 0.1.5 min_python = 3.8 audience = Developers language = English