diff --git a/devshells/README.md b/devshells/README.md index e732e31..9660bdb 100644 --- a/devshells/README.md +++ b/devshells/README.md @@ -34,37 +34,35 @@ nix develop .#python-uv - Ruff (linter/formatter) - Pyright (type checker) -### `ml-ai` -Machine Learning and AI development with HuggingFace model management. +### `ai-notebook` +Notebook-focused ML/AI environment with HuggingFace tooling and JupyterLab. ```bash -nix develop .#ml-ai +nix develop .#ai-notebook ``` **Includes:** -- Python 3.12 with huggingface-hub -- `huggingface-cli` for downloading models -- UV for package management -- Git LFS for large files -- Ruff and Pyright +- Python 3.12 with JupyterLab, NumPy/Pandas/SciPy stack +- HuggingFace `transformers`, `datasets`, `accelerate`, `tokenizers` +- Vision tooling: OpenCV, scikit-image, Pillow, albumentations, Altair/Plotly +- Optional CPU PyTorch (Linux hosts) plus torchvision/torchaudio +- UV, Ruff, Pyright, Git LFS -**Model cache:** `~/.cache/huggingface` (persistent across projects) +**HuggingFace caches:** +- Models: `~/.cache/huggingface/hub` +- Transformers: `~/.cache/huggingface/transformers` **Example usage:** ```bash -# Download a TTS model -huggingface-cli download facebook/mms-tts-eng +# Launch JupyterLab with AI helpers +jupyter lab -# Download a small LLM +# Download a model or dataset huggingface-cli download meta-llama/Llama-3.2-1B +python -c "from datasets import load_dataset; load_dataset('ag_news')" -# Download specific files only -huggingface-cli download openai/whisper-large-v3 --include "*.json" "*.safetensors" - -# Use in your project -uv init my-ml-project -cd my-ml-project -uv add transformers torch +# Fine-tune with Accelerate +python -m accelerate.commands.launch train.py ``` ### `go` @@ -114,7 +112,7 @@ Use the full path to your dotfiles: ```bash cd ~/my-project -nix develop ~/Development/dotfiles#ml-ai +nix develop ~/Development/dotfiles#ai-notebook ``` ### Option 2: Flake Registry Alias (Recommended) @@ -130,7 +128,7 @@ nix registry list | grep dotfiles **Then from anywhere:** ```bash cd ~/any-project -nix develop dotfiles#ml-ai +nix develop dotfiles#ai-notebook ``` Benefits: Shorter commands, cleaner paths, easy to remember! @@ -145,14 +143,14 @@ nix registry add dotfiles ~/Development/dotfiles # In each project: cd ~/my-ml-project -echo "use flake dotfiles#ml-ai" > .envrc +echo "use flake dotfiles#ai-notebook" > .envrc direnv allow ``` **Alternative: Without registry (using full path):** ```bash cd ~/my-ml-project -echo "use flake ~/Development/dotfiles#ml-ai" > .envrc +echo "use flake ~/Development/dotfiles#ai-notebook" > .envrc direnv allow ``` @@ -172,7 +170,7 @@ Create a `flake.nix` in your project that imports the shell: let system = "aarch64-darwin"; in { - devShells.${system}.default = dotfiles.devShells.${system}.ml-ai; + devShells.${system}.default = dotfiles.devShells.${system}.ai-notebook; }; } ``` @@ -187,9 +185,9 @@ nix develop | Method | Command | Pros | |--------|---------|------| -| Direct path | `nix develop ~/Development/dotfiles#ml-ai` | Simple, no setup | -| Registry | `nix develop dotfiles#ml-ai` | Clean, short commands | -| direnv + registry | `echo "use flake dotfiles#ml-ai" > .envrc` | Auto-loads, clean | +| Direct path | `nix develop ~/Development/dotfiles#ai-notebook` | Simple, no setup | +| Registry | `nix develop dotfiles#ai-notebook` | Clean, short commands | +| direnv + registry | `echo "use flake dotfiles#ai-notebook" > .envrc` | Auto-loads, clean | | Project flake | `nix develop` | Shareable, version-controlled | ## Adding New Shells diff --git a/devshells/ai-notebook.nix b/devshells/ai-notebook.nix new file mode 100644 index 0000000..5da154c --- /dev/null +++ b/devshells/ai-notebook.nix @@ -0,0 +1,54 @@ +{ pkgs, lib, python ? pkgs.python312, ... }: + +let + pythonWithAI = python.withPackages (ps: + let + basePackages = with ps; [ + accelerate + albumentations + altair + datasets + evaluate + huggingface-hub + ipykernel + jupyterlab + matplotlib + numpy + opencv4 + pandas + pillow + plotly + scipy + scikit-image + scikit-learn + seaborn + sentencepiece + tokenizers + transformers + ]; + linuxOnlyPackages = lib.optionals pkgs.stdenv.isLinux + (with ps; [ torch torchaudio torchvision ]); + in basePackages ++ linuxOnlyPackages); +in pkgs.mkShell { + name = "ai-notebook"; + packages = with pkgs; [ pythonWithAI uv git-lfs ruff pyright ]; + + shellHook = '' + export UV_PYTHON="${pythonWithAI}/bin/python3" + export UV_LINK_MODE=copy + + export PYTHONNOUSERSITE=1 + export HF_HOME="''${XDG_CACHE_HOME:-$HOME/.cache}/huggingface" + export HF_HUB_CACHE="$HF_HOME/hub" + export TRANSFORMERS_CACHE="$HF_HOME/transformers" + export JUPYTER_CONFIG_DIR="''${XDG_CONFIG_HOME:-$HOME/.config}/jupyter" + + mkdir -p "$HF_HOME" "$JUPYTER_CONFIG_DIR" + + echo "AI notebook shell ready" + echo "Python: ${python.version}" + echo "HuggingFace cache: $HF_HOME" + echo "Launch JupyterLab: jupyter lab" + echo "Launch classic notebook: jupyter notebook" + ''; +} diff --git a/devshells/jupyter-notebook.nix b/devshells/jupyter-notebook.nix new file mode 100644 index 0000000..f8db924 --- /dev/null +++ b/devshells/jupyter-notebook.nix @@ -0,0 +1,31 @@ +{ pkgs, python ? pkgs.python312, ... }: + +let + # Python environment with notebook tooling and core data-science libs + pythonWithNotebook = python.withPackages (ps: + with ps; [ + altair + ipykernel + jupyterlab + matplotlib + numpy + pandas + polars + plotly + scipy + scikit-learn + seaborn + ]); +in pkgs.mkShell { + name = "jupyter-notebook"; + packages = [ pythonWithNotebook pkgs.ruff pkgs.pyright pkgs.git ]; + shellHook = '' + export PYTHONNOUSERSITE=1 + export JUPYTER_CONFIG_DIR="''${XDG_CONFIG_HOME:-$HOME/.config}/jupyter" + mkdir -p "$JUPYTER_CONFIG_DIR" + echo "Jupyter notebook shell ready" + echo "Python: ${python.version}" + echo "Launch notebook: jupyter lab" + echo "Launch classic: jupyter notebook" + ''; +} diff --git a/devshells/ml-ai.nix b/devshells/ml-ai.nix deleted file mode 100644 index 4e31ebd..0000000 --- a/devshells/ml-ai.nix +++ /dev/null @@ -1,35 +0,0 @@ -{ pkgs, python ? pkgs.python312, ... }: - -let - # Create a Python environment with huggingface-cli - pythonWithHF = python.withPackages (ps: [ ps.huggingface-hub ]); -in pkgs.mkShell { - name = "ml-ai"; - packages = with pkgs; [ pythonWithHF uv git-lfs ruff pyright ]; - shellHook = '' - # Set up UV - export UV_PYTHON="${pythonWithHF}/bin/python3" - export UV_LINK_MODE=copy - - # Set up HuggingFace cache (persistent across projects) - export HF_HOME="''${XDG_CACHE_HOME:-$HOME/.cache}/huggingface" - export HF_HUB_CACHE="$HF_HOME/hub" - mkdir -p "$HF_HOME" - - # Optional: Set offline mode (uncomment to use only cached models) - # export HF_HUB_OFFLINE=1 - - echo "ML/AI shell ready" - echo "Python: ${python.version}" - echo "HuggingFace cache: $HF_HOME" - echo "" - echo "Download models with:" - echo " huggingface-cli download " - echo "" - echo "Example TTS:" - echo " huggingface-cli download facebook/mms-tts-eng" - echo "" - echo "Example LLM:" - echo " huggingface-cli download meta-llama/Llama-3.2-1B" - ''; -}