feat: add notebook-focused dev shell
This commit is contained in:
parent
8a8b5bd48d
commit
3da4f4d493
4 changed files with 110 additions and 62 deletions
|
|
@ -34,37 +34,35 @@ nix develop .#python-uv
|
|||
- Ruff (linter/formatter)
|
||||
- Pyright (type checker)
|
||||
|
||||
### `ml-ai`
|
||||
Machine Learning and AI development with HuggingFace model management.
|
||||
### `ai-notebook`
|
||||
Notebook-focused ML/AI environment with HuggingFace tooling and JupyterLab.
|
||||
|
||||
```bash
|
||||
nix develop .#ml-ai
|
||||
nix develop .#ai-notebook
|
||||
```
|
||||
|
||||
**Includes:**
|
||||
- Python 3.12 with huggingface-hub
|
||||
- `huggingface-cli` for downloading models
|
||||
- UV for package management
|
||||
- Git LFS for large files
|
||||
- Ruff and Pyright
|
||||
- Python 3.12 with JupyterLab, NumPy/Pandas/SciPy stack
|
||||
- HuggingFace `transformers`, `datasets`, `accelerate`, `tokenizers`
|
||||
- Vision tooling: OpenCV, scikit-image, Pillow, albumentations, Altair/Plotly
|
||||
- Optional CPU PyTorch (Linux hosts) plus torchvision/torchaudio
|
||||
- UV, Ruff, Pyright, Git LFS
|
||||
|
||||
**Model cache:** `~/.cache/huggingface` (persistent across projects)
|
||||
**HuggingFace caches:**
|
||||
- Models: `~/.cache/huggingface/hub`
|
||||
- Transformers: `~/.cache/huggingface/transformers`
|
||||
|
||||
**Example usage:**
|
||||
```bash
|
||||
# Download a TTS model
|
||||
huggingface-cli download facebook/mms-tts-eng
|
||||
# Launch JupyterLab with AI helpers
|
||||
jupyter lab
|
||||
|
||||
# Download a small LLM
|
||||
# Download a model or dataset
|
||||
huggingface-cli download meta-llama/Llama-3.2-1B
|
||||
python -c "from datasets import load_dataset; load_dataset('ag_news')"
|
||||
|
||||
# Download specific files only
|
||||
huggingface-cli download openai/whisper-large-v3 --include "*.json" "*.safetensors"
|
||||
|
||||
# Use in your project
|
||||
uv init my-ml-project
|
||||
cd my-ml-project
|
||||
uv add transformers torch
|
||||
# Fine-tune with Accelerate
|
||||
python -m accelerate.commands.launch train.py
|
||||
```
|
||||
|
||||
### `go`
|
||||
|
|
@ -114,7 +112,7 @@ Use the full path to your dotfiles:
|
|||
|
||||
```bash
|
||||
cd ~/my-project
|
||||
nix develop ~/Development/dotfiles#ml-ai
|
||||
nix develop ~/Development/dotfiles#ai-notebook
|
||||
```
|
||||
|
||||
### Option 2: Flake Registry Alias (Recommended)
|
||||
|
|
@ -130,7 +128,7 @@ nix registry list | grep dotfiles
|
|||
**Then from anywhere:**
|
||||
```bash
|
||||
cd ~/any-project
|
||||
nix develop dotfiles#ml-ai
|
||||
nix develop dotfiles#ai-notebook
|
||||
```
|
||||
|
||||
Benefits: Shorter commands, cleaner paths, easy to remember!
|
||||
|
|
@ -145,14 +143,14 @@ nix registry add dotfiles ~/Development/dotfiles
|
|||
|
||||
# In each project:
|
||||
cd ~/my-ml-project
|
||||
echo "use flake dotfiles#ml-ai" > .envrc
|
||||
echo "use flake dotfiles#ai-notebook" > .envrc
|
||||
direnv allow
|
||||
```
|
||||
|
||||
**Alternative: Without registry (using full path):**
|
||||
```bash
|
||||
cd ~/my-ml-project
|
||||
echo "use flake ~/Development/dotfiles#ml-ai" > .envrc
|
||||
echo "use flake ~/Development/dotfiles#ai-notebook" > .envrc
|
||||
direnv allow
|
||||
```
|
||||
|
||||
|
|
@ -172,7 +170,7 @@ Create a `flake.nix` in your project that imports the shell:
|
|||
let
|
||||
system = "aarch64-darwin";
|
||||
in {
|
||||
devShells.${system}.default = dotfiles.devShells.${system}.ml-ai;
|
||||
devShells.${system}.default = dotfiles.devShells.${system}.ai-notebook;
|
||||
};
|
||||
}
|
||||
```
|
||||
|
|
@ -187,9 +185,9 @@ nix develop
|
|||
|
||||
| Method | Command | Pros |
|
||||
|--------|---------|------|
|
||||
| Direct path | `nix develop ~/Development/dotfiles#ml-ai` | Simple, no setup |
|
||||
| Registry | `nix develop dotfiles#ml-ai` | Clean, short commands |
|
||||
| direnv + registry | `echo "use flake dotfiles#ml-ai" > .envrc` | Auto-loads, clean |
|
||||
| Direct path | `nix develop ~/Development/dotfiles#ai-notebook` | Simple, no setup |
|
||||
| Registry | `nix develop dotfiles#ai-notebook` | Clean, short commands |
|
||||
| direnv + registry | `echo "use flake dotfiles#ai-notebook" > .envrc` | Auto-loads, clean |
|
||||
| Project flake | `nix develop` | Shareable, version-controlled |
|
||||
|
||||
## Adding New Shells
|
||||
|
|
|
|||
54
devshells/ai-notebook.nix
Normal file
54
devshells/ai-notebook.nix
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
{ pkgs, lib, python ? pkgs.python312, ... }:
|
||||
|
||||
let
|
||||
pythonWithAI = python.withPackages (ps:
|
||||
let
|
||||
basePackages = with ps; [
|
||||
accelerate
|
||||
albumentations
|
||||
altair
|
||||
datasets
|
||||
evaluate
|
||||
huggingface-hub
|
||||
ipykernel
|
||||
jupyterlab
|
||||
matplotlib
|
||||
numpy
|
||||
opencv4
|
||||
pandas
|
||||
pillow
|
||||
plotly
|
||||
scipy
|
||||
scikit-image
|
||||
scikit-learn
|
||||
seaborn
|
||||
sentencepiece
|
||||
tokenizers
|
||||
transformers
|
||||
];
|
||||
linuxOnlyPackages = lib.optionals pkgs.stdenv.isLinux
|
||||
(with ps; [ torch torchaudio torchvision ]);
|
||||
in basePackages ++ linuxOnlyPackages);
|
||||
in pkgs.mkShell {
|
||||
name = "ai-notebook";
|
||||
packages = with pkgs; [ pythonWithAI uv git-lfs ruff pyright ];
|
||||
|
||||
shellHook = ''
|
||||
export UV_PYTHON="${pythonWithAI}/bin/python3"
|
||||
export UV_LINK_MODE=copy
|
||||
|
||||
export PYTHONNOUSERSITE=1
|
||||
export HF_HOME="''${XDG_CACHE_HOME:-$HOME/.cache}/huggingface"
|
||||
export HF_HUB_CACHE="$HF_HOME/hub"
|
||||
export TRANSFORMERS_CACHE="$HF_HOME/transformers"
|
||||
export JUPYTER_CONFIG_DIR="''${XDG_CONFIG_HOME:-$HOME/.config}/jupyter"
|
||||
|
||||
mkdir -p "$HF_HOME" "$JUPYTER_CONFIG_DIR"
|
||||
|
||||
echo "AI notebook shell ready"
|
||||
echo "Python: ${python.version}"
|
||||
echo "HuggingFace cache: $HF_HOME"
|
||||
echo "Launch JupyterLab: jupyter lab"
|
||||
echo "Launch classic notebook: jupyter notebook"
|
||||
'';
|
||||
}
|
||||
31
devshells/jupyter-notebook.nix
Normal file
31
devshells/jupyter-notebook.nix
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
{ pkgs, python ? pkgs.python312, ... }:
|
||||
|
||||
let
|
||||
# Python environment with notebook tooling and core data-science libs
|
||||
pythonWithNotebook = python.withPackages (ps:
|
||||
with ps; [
|
||||
altair
|
||||
ipykernel
|
||||
jupyterlab
|
||||
matplotlib
|
||||
numpy
|
||||
pandas
|
||||
polars
|
||||
plotly
|
||||
scipy
|
||||
scikit-learn
|
||||
seaborn
|
||||
]);
|
||||
in pkgs.mkShell {
|
||||
name = "jupyter-notebook";
|
||||
packages = [ pythonWithNotebook pkgs.ruff pkgs.pyright pkgs.git ];
|
||||
shellHook = ''
|
||||
export PYTHONNOUSERSITE=1
|
||||
export JUPYTER_CONFIG_DIR="''${XDG_CONFIG_HOME:-$HOME/.config}/jupyter"
|
||||
mkdir -p "$JUPYTER_CONFIG_DIR"
|
||||
echo "Jupyter notebook shell ready"
|
||||
echo "Python: ${python.version}"
|
||||
echo "Launch notebook: jupyter lab"
|
||||
echo "Launch classic: jupyter notebook"
|
||||
'';
|
||||
}
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
{ pkgs, python ? pkgs.python312, ... }:
|
||||
|
||||
let
|
||||
# Create a Python environment with huggingface-cli
|
||||
pythonWithHF = python.withPackages (ps: [ ps.huggingface-hub ]);
|
||||
in pkgs.mkShell {
|
||||
name = "ml-ai";
|
||||
packages = with pkgs; [ pythonWithHF uv git-lfs ruff pyright ];
|
||||
shellHook = ''
|
||||
# Set up UV
|
||||
export UV_PYTHON="${pythonWithHF}/bin/python3"
|
||||
export UV_LINK_MODE=copy
|
||||
|
||||
# Set up HuggingFace cache (persistent across projects)
|
||||
export HF_HOME="''${XDG_CACHE_HOME:-$HOME/.cache}/huggingface"
|
||||
export HF_HUB_CACHE="$HF_HOME/hub"
|
||||
mkdir -p "$HF_HOME"
|
||||
|
||||
# Optional: Set offline mode (uncomment to use only cached models)
|
||||
# export HF_HUB_OFFLINE=1
|
||||
|
||||
echo "ML/AI shell ready"
|
||||
echo "Python: ${python.version}"
|
||||
echo "HuggingFace cache: $HF_HOME"
|
||||
echo ""
|
||||
echo "Download models with:"
|
||||
echo " huggingface-cli download <model-id>"
|
||||
echo ""
|
||||
echo "Example TTS:"
|
||||
echo " huggingface-cli download facebook/mms-tts-eng"
|
||||
echo ""
|
||||
echo "Example LLM:"
|
||||
echo " huggingface-cli download meta-llama/Llama-3.2-1B"
|
||||
'';
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue