From 92efc85b3b4a2a62eb14ae7c1d6d5ce6ff8afa82 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Mon, 11 Jul 2022 20:40:23 -0700 Subject: [PATCH] [air/docs] checkpoints (#25901) --- doc/source/_toc.yml | 1 + doc/source/ray-air/checkpoints.rst | 84 +++++++++++ .../ray-air/doc_code/checkpoint_mlflow.py | 28 ++++ .../ray-air/doc_code/checkpoint_usage.py | 130 ++++++++++++++++++ doc/source/ray-air/images/checkpoints.jpg | Bin 0 -> 22422 bytes doc/source/ray-air/user-guides.rst | 9 ++ python/ray/air/checkpoint.py | 2 +- 7 files changed, 253 insertions(+), 1 deletion(-) create mode 100644 doc/source/ray-air/checkpoints.rst create mode 100644 doc/source/ray-air/doc_code/checkpoint_mlflow.py create mode 100644 doc/source/ray-air/doc_code/checkpoint_usage.py create mode 100644 doc/source/ray-air/images/checkpoints.jpg diff --git a/doc/source/_toc.yml b/doc/source/_toc.yml index 8ded7fee52121..ec11fb24ced65 100644 --- a/doc/source/_toc.yml +++ b/doc/source/_toc.yml @@ -16,6 +16,7 @@ parts: sections: - file: ray-air/preprocessors - file: ray-air/check-ingest + - file: ray-air/checkpoints - file: ray-air/examples/analyze_tuning_results - file: ray-air/examples/upload_to_comet_ml - file: ray-air/examples/upload_to_wandb diff --git a/doc/source/ray-air/checkpoints.rst b/doc/source/ray-air/checkpoints.rst new file mode 100644 index 0000000000000..b360b1917fe82 --- /dev/null +++ b/doc/source/ray-air/checkpoints.rst @@ -0,0 +1,84 @@ +Checkpoints +=========== + +Checkpoints are the common format for models that are used across different components of the Ray AI Runtime. + +.. image:: images/checkpoints.jpg + +What exactly is a checkpoint? +----------------------------- + +The Checkpoint object is a serializable reference to a model. The model can represented in one of three ways: + +- a directory located on local (on-disk) storage +- a directory located on external storage (e.g. cloud storage) +- an in-memory dictionary + +The flexibility provided in the Checkpoint model representation is useful in distributed environments, +where you may want to recreate the same model on multiple nodes in your Ray cluster for inference +or across different Ray clusters. + + +Creating a checkpoint +--------------------- + +There are two ways of generating a checkpoint. + +The first way is to generate it from a pretrained model. Each framework that AIR supports has a ``to_air_checkpoint`` method that can be used to generate an AIR checkpoint: + +.. literalinclude:: doc_code/checkpoint_usage.py + :language: python + :start-after: __checkpoint_quick_start__ + :end-before: __checkpoint_quick_end__ + + +Another way is to retrieve it from the results of a Trainer or a Tuner. + +.. literalinclude:: doc_code/checkpoint_usage.py + :language: python + :start-after: __use_trainer_checkpoint_start__ + :end-before: __use_trainer_checkpoint_end__ + +What can I do with a checkpoint? +-------------------------------- + +Checkpoints can be used to instantiate a :class:`Predictor`, :class:`BatchPredictor`, or :class:`PredictorDeployment`. +Upon usage, the model held by the Checkpoint will be instantiated in memory and used for inference. + +Below is an example using a checkpoint in the :class:`BatchPredictor` for scalable batch inference: + +.. literalinclude:: doc_code/checkpoint_usage.py + :language: python + :start-after: __batch_pred_start__ + :end-before: __batch_pred_end__ + +Below is an example using a checkpoint in a service for online inference via :class:`PredictorDeployment`: + +.. literalinclude:: doc_code/checkpoint_usage.py + :language: python + :start-after: __online_inference_start__ + :end-before: __online_inference_end__ + +The Checkpoint object has methods to translate between different checkpoint storage locations. +With this flexibility, Checkpoint objects can be serialized and used in different contexts +(e.g., on a different process or a different machine): + +.. literalinclude:: doc_code/checkpoint_usage.py + :language: python + :start-after: __basic_checkpoint_start__ + :end-before: __basic_checkpoint_end__ + + +Example: Using Checkpoints with MLflow +-------------------------------------- + +MLflow has its own `checkpoint format `__ called the "MLflow Model". It is a standard format for packaging machine learning models that can be used in a variety of downstream tools. + +Below is an example of using MLflow models as a Ray AIR Checkpoint. + +.. literalinclude:: doc_code/checkpoint_mlflow.py + :language: python + :start-after: __mlflow_checkpoint_start__ + :end-before: __mlflow_checkpoint_end__ + + diff --git a/doc/source/ray-air/doc_code/checkpoint_mlflow.py b/doc/source/ray-air/doc_code/checkpoint_mlflow.py new file mode 100644 index 0000000000000..fd7a6af65b329 --- /dev/null +++ b/doc/source/ray-air/doc_code/checkpoint_mlflow.py @@ -0,0 +1,28 @@ +# flake8: noqa +# isort: skip_file + +# __mlflow_checkpoint_start__ +from ray.air.checkpoint import Checkpoint +from sklearn.ensemble import RandomForestClassifier +import mlflow.sklearn + +# Create an sklearn classifier +clf = RandomForestClassifier(max_depth=7, random_state=0) +# ... e.g. train model with clf.fit() +# Save model using MLflow +mlflow.sklearn.save_model(clf, "model_directory") + +# Create checkpoint object from path +checkpoint = Checkpoint.from_directory("model_directory") + +# Write it to some other directory +checkpoint.to_directory("other_directory") +# You can also use `checkpoint.to_uri/from_uri` to +# read from/write to cloud storage + +# We can now use MLflow to re-load the model +clf = mlflow.sklearn.load_model("other_directory") + +# It is guaranteed that the original data was recovered +assert isinstance(clf, RandomForestClassifier) +# __mlflow_checkpoint_end__ diff --git a/doc/source/ray-air/doc_code/checkpoint_usage.py b/doc/source/ray-air/doc_code/checkpoint_usage.py new file mode 100644 index 0000000000000..c0bdc4181798f --- /dev/null +++ b/doc/source/ray-air/doc_code/checkpoint_usage.py @@ -0,0 +1,130 @@ +# flake8: noqa +# isort: skip_file + +# __checkpoint_quick_start__ +from ray.train.tensorflow import to_air_checkpoint +import tensorflow as tf + +# This can be a trained model. +def build_model() -> tf.keras.Model: + model = tf.keras.Sequential( + [ + tf.keras.layers.InputLayer(input_shape=(1,)), + tf.keras.layers.Dense(1), + ] + ) + return model + + +model = build_model() + +checkpoint = to_air_checkpoint(model) +# __checkpoint_quick_end__ + + +# __use_trainer_checkpoint_start__ +import pandas as pd +import ray +from ray.air import train_test_split +from ray.train.xgboost import XGBoostTrainer + + +bc_df = pd.read_csv( + "https://air-example-data.s3.us-east-2.amazonaws.com/breast_cancer.csv" +) +dataset = ray.data.from_pandas(bc_df) +# Optionally, read directly from s3 +# dataset = ray.data.read_csv("s3://air-example-data/breast_cancer.csv") + +# Split data into train and validation. +train_dataset, valid_dataset = train_test_split(dataset, test_size=0.3) + +trainer = XGBoostTrainer( + scaling_config={"num_workers": 2}, + label_column="target", + params={ + "objective": "binary:logistic", + "eval_metric": ["logloss", "error"], + }, + datasets={"train": train_dataset}, + num_boost_round=5, +) + +result = trainer.fit() +checkpoint = result.checkpoint +# __use_trainer_checkpoint_end__ + +# __batch_pred_start__ +from ray.train.batch_predictor import BatchPredictor +from ray.train.xgboost import XGBoostPredictor + +# Create a test dataset by dropping the target column. +test_dataset = valid_dataset.map_batches( + lambda df: df.drop("target", axis=1), batch_format="pandas" +) + +batch_predictor = BatchPredictor.from_checkpoint(checkpoint, XGBoostPredictor) + +# Bulk batch prediction. +batch_predictor.predict(test_dataset) +# __batch_pred_end__ + + +# __online_inference_start__ +import requests +from fastapi import Request +import pandas as pd + +from ray import serve +from ray.serve import PredictorDeployment +from ray.serve.http_adapters import json_request + + +async def adapter(request: Request): + content = await request.json() + print(content) + return pd.DataFrame.from_dict(content) + + +serve.start(detached=True) +deployment = PredictorDeployment.options(name="XGBoostService") + +deployment.deploy( + XGBoostPredictor, checkpoint, batching_params=False, http_adapter=adapter +) + +print(deployment.url) + +sample_input = test_dataset.take(1) +sample_input = dict(sample_input[0]) + +output = requests.post(deployment.url, json=[sample_input]).json() +print(output) +# __online_inference_end__ + +# __basic_checkpoint_start__ +from ray.air.checkpoint import Checkpoint + +# Create checkpoint data dict +checkpoint_data = {"data": 123} + +# Create checkpoint object from data +checkpoint = Checkpoint.from_dict(checkpoint_data) + +# Save checkpoint to a directory on the file system. +path = checkpoint.to_directory() + +# This path can then be passed around, +# # e.g. to a different function or a different script. +# You can also use `checkpoint.to_uri/from_uri` to +# read from/write to cloud storage + +# In another function or script, recover Checkpoint object from path +checkpoint = Checkpoint.from_directory(path) + +# Convert into dictionary again +recovered_data = checkpoint.to_dict() + +# It is guaranteed that the original data has been recovered +assert recovered_data == checkpoint_data +# __basic_checkpoint_end__ diff --git a/doc/source/ray-air/images/checkpoints.jpg b/doc/source/ray-air/images/checkpoints.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3d868bf311c341243c5762f47ebbc54e2f1deda8 GIT binary patch literal 22422 zcmeIa1za58mM+@3yM*8gt_dCp5<;+G0fJiy5VUcZbOHeqT!I7w1lPuCBxrDVcXtgg z-M8|$Imwxsd(OPyeRJQN)BK9+uCBdntyR0$Uh7-mZo~v)0YvyzMqUPlgbV^90e>LG zG)M}BiH?qej)sYWfq{jEiH%E&hl_)Qdz+YqfRu)kj+Ta!nwp-Gn}z-^Cj&J#s}LLK zeI9;(emWLmu?M`O+VUypYxJ}8#%)-jX&MzP+^gvid z>WQ?BtepH)_2(LzTG~3gCZ=Yu%q=XfoSa=;-P}Dq{oeTp1it?e6czn3CN?hqQ$kvL zMrKxaPHtXlS$PGlvZ}h~TXRcmTYE=m*U<3D=-BwgF zi_0H+A%ReSR}1+6yJElSMFi-DjEahait$4)BxE;WLm@&%yTgM{EUAiN>_Bpt_bn#r z?*(Oiw^!u4$g=>MAYX-fxPa_6i|)PaqcO;VycYu0mV2ym#b~}u(SNR7 zZgjC)fB-E-ogqM>st6F)STF)4T9I;1Y={6I=J}9%G13l(AVBZ7r4gWH-F`UMJ;?be z-9eH?!}o3-1gME=9|5Aw9+WMW{8xYfKeylQ{LHP`gudjsG2VyMUqz4)`E71fq$3xg zAm5g2qRCrzdX0_%McS%dVOh~#>IgW;1Pe+^lRiWG@7na~{3KSS4-lXmKXAjb+PKFr z2gb@P`EpAtrAam0?);sMPHxHK2e1?mvCOd&T=m9QrpE=Ykt5XDlfxAcX$(8>@o!fZ zLR0*sm~r`Le0*{Z@Oyxc)CpBJvPZbMil8=GpVZd2A(6MYu4MG+B&7pVE%S7svmdtw zzVflx95ZR>UcYBJ$UsYA=LnAH;+pgNlpa8O+mi7`o_NMcc_V9D5a{3A%-=Sw=taZ7 z=?21Ml?<5v&f;BXBAQLoq-qqZ(xgaj{rTr%`-Wp&k#RX;h9Q04X1z%7)b#S@{{@NXw?ZwN9R(}6_zXll;Jv>t=C3m(WY+sQG9Tj;1Xv8 za?YBF07Y3`$%aU5+)D-AMw%U5KL1#K<$d+av!9Qq@6z5!fWFiNuF3a9!v%BEpe&lftTHug+-COGR` z=}h&rb_B4T^xL5*9t@IK)$`n?g?pjZH*BhkAEw;Y&Xzptl5fQrGD z@F3l>*fkeH!Hu+_iI!hI%y$U_YtOJmsCyxm1peHX+*D72nXlaF^Bu|Zy%$bx5XvCA zTIt0){3|v8J45>?p^n6{em42oU)c#K7Gk$t;pkuMZar3_^wE5tlqurBZBWb+>K`LB zxP4|hwHK7I3N0t#dqV#;V1rD5rLF1BwS%hp%c9&iYmvVEw5>5(n(FcW&$R>|4mpz9 zSg$y$*xPd6ch#Q?i5J7jl z!1(rSY9CqE@=+TBEdE{PNlK<96Jn0G2__l>)d5E9*vW?VeQnqB1{7jXc+An{5H#W*9lL{sKElx;R8nDzHkpuKYfM zMb6q<(-QA8B@epU`y}4{Rx!M>WO`!?49&{qzcmGJ8J?PZ-^6bAinAcwH$xEflvKxm z&~G_O$Ov{IQ$%)Jd7A$ud2)C4C_-_eBQ@#~AvX%y3oX=TjIf$D1MBC90&Q>Y3YDz! zKUtH^%w`bXE@X^$6)7BjcI)#!-B<{UU~1;pz?N?9nb1^3sNu$woV!G@6)ZY>kca5X z<;bwR*0U=ck<6^x)62@q*1FO5NsaXCXYy$29Y_JnnEZe!eFo==EvlQ)tacylE!fnt zh>4*G4orLHwiep}_5BWt&34`W<9vm$G-CAiBc>xPYVuW40*@P$(MgdSmx|L4#A;nN z2L+Cg^qF%KGFy9saP*&MS5aV#fG{3$3HL|Kyq?of$geRUzHJC`E=;?3%Yt_+UEV+qeMCrvLdNknUxTPaD0F;aNYPbrqku|< z^4t~<=imVAVzI;owcLN)A#3(huyVm7u!VdKnx~L!<@$1N?gTuNV|Ju~_DxvSw%djJ zMG&`d&sg3^cV^ZjS$QU@ z-uz6+#CG3iE%-~B5Nm<;KDmX6ZCZ`bS|+Ym$R@?Rl@jx-TBP~$8O>f~z`XIl`ytgw zgzi#siEF@8#?KL;p@&vm9B~JM*AT#Qf6M)C>L1bKtTb121>(2gVhIpnWsnxnvda1Q z3IwPy@Ip&Pzdcl9xo;d^L$_*Z>i}O>zuXGzX}DRV8-LtFDU&>Q$WtB?^(H#pD*B2_ z8Sw3L2oQT2kfYERBS2LWF$($v&4a6kOIMMOP__Lzh==UIohDIIzJGBXew06=;bRzn zbg!CZHxI&h)g>`8<#v53d!c$0nhojCfW#Q4h35~#rO)V3MCYJNxH|bWja?-bsA(1{EVihWD z|A7#yC!P`=^((Yf+218<)Bn6S-5!sMNh{;mWD zEcd#9ADCzVxU_cG9ETl+Zac0q3A?>`Kuyp(x5SEb#bwcvKzm5KGHq`go7dpxnv1cO zva*#@gG&-Hkr+Mo_${1%Hak15t*hYXCTg$O@--t?20KqoUO-lB99$mGem6>gUM14b z9E8W&;?lT5>SprhVM3;5s{Yn32P?z7?rX_{^L8e;nde4MMG>Ha=$q09aKgGq`q^Cf zqCKBo&l&0)#F}wGEr=B*7#LcS(XB?O#6Vmmx0|nA#woF4dZ=~tF8#MrK3o{@ z&AJ}DC{3zC{KIAZ<~HUifsi(E8FIO828{frGG}Imd${g%hn(a`D!T@fQ+U=Rf2%sA30-2&G`mwg3?+M6_H4m>H`c7pH6*6bQ=|7Ke?oIB zk6dcxwTe|ihP~vGBgfbffA~WpEfyA(+x%PlsXY#ccY|ck{lDP6WJp#Lxgct%UJ`4F z6lk0hWodl2Aym(tAl$~aGI(cr$vRaK9)yLfyr1am^WAV=>1ep;yCC9?>Vu;s&h(8x-e4B&?}R!juFH=pjYwq{=4Gigz0!Yh3A>aMVdnsz&Ucvw;s#z% zYla=h7Gg!+P=A5}aohh%aXn-jsK8&}ecZi9-g%3Su@Own^&-f)N&1tw85P#Z3 z*kjXRK=7;$nW5{-%nAJVA$A9_QtF@H@$dFdadbx_x;I)ElpEWhtAIf^G``i)F^(@` z1zG+u{%#JkXl^V`hPJv3OETx0T{&#BBi{2c1?Q#Dj1MI1h&hdW?)Kiy)R+gV3GXog zVP7dScvr>V0bZ(dinM$lbQsSAJbL@TJ$kk#HD76O7J-b+C*{=ZSTojjj6Cv>M{i;& z^l@JJBQ%ie5I+Z)r8izQPH1>x#;{LHfsF=93^}apK8CUG$3MbNkI?voHHcehS$h0;3&BO8RU!PWcrJasMx$M&hJjft`>~!`DAwVI6KYtuok+$r( zitPJJ-9B|KfB;FGBS0+t^Kd@dpFbLikvfVm{_C{A?DId*AmrX$;OwdM*Q7%y_jWmC zqb&W`@WMDR9mvYmh=-!Z;ESgsy`fArlJE00-TfpHXnT^iNmy;48t!ZBY#% zL;I2N+NxKqyNH&{AIEhacgS z#JS=%7Xl>39f|pT+jpq&Lc|fo?lLky!i+l%_H@G!IaEw;KzXBl{R*HYahS^}lf%rW*`|okx z|EC2r!i2N~_d;TaJq3uv4Wu1uQEMwkm=3u{V|s)Hhph5TE%ZeMxwzgoQPWD&`MaF- za!n4>L;#*Wfb4_Yq8R(q>fY{hv`U>*Y)x@Q9RYCPa5-#)KFfO=E*V7q(Kr~Dmmc?GRN-sG`-T#k*K*@7 zP`xV>{mP0ew*>hY$={aPp2#`6HpI4oQQkPE=&uW`zZ!IGdah8p<+4Dj{a(<^zw+9% z8NLA5;FA8E;8&-4@p0^qO%x+fc~nh?6>m>Fh(OJ%A~rXBOS0Gh1UjHh(ZpLnUr0dK z?jAcUDtSmJbFh1Pk?>qoZHsrha;7Tc!4v$4{(TO^V=$v85mC>%l>E&B=1(pbEj*>6 zCM0b#xQ?LvvQk?-t(InB`RWGJgFfnJb~&Oh!8QzQ{4dL?v@uK`%)yYQ(1(hL0jU{< zv77}iNsnE_HVrbWrdH#TGffr{r$v|#YH#LuFX-zvNa?hSUDSk%sx7yR`r6EgBT%kAaJ*l==|Uadj3ZOp z75RzkCHe~?p)V(*v>Xu-B5TjR29K6I@;u7ayw5x>bXXpuFz4m8el(#|8W6 z3Z^5&lDBFMQl^& zNUF4i9S|Uf%%V|9WMNqCe%sXSTJRFbt+Gn};+Dxe4*N0modpyV5*#Wv~g z$;|7H?QfmZy$!48UisUwOZgxq>Nu_M>pQ2{!ANB^}_A=olK;WoOO+yrm&397x7?h8_NwI`7ZGFuD8)#}NT zO>}RU(raoeODm$*wk#HWnP!gUnlxl{Su#ON(sv`eZp+b!_&yxtY7g-iuykUoYir1o z7FAS}o7+U;z{jLd#=Dc~{9JQf^9;7>U@fZNMBgP(5}--j0$d3Q(sN8}VL^LTIxL~w zEVzSDdQcwd+tRzW$-FK{5+K9NT&=jt)1OkIJ`i`6CqdV2YHFfg7>SaLyyHQmilJ%p zh81b#kF4g;?B>UwuW|l}s}q<A^>l%&b z<+tfaqjsSmgGo!LD{8X7Q_8usJp92RP`Hvz7`M(y!*Bf_os1!!+{X>NSeB+*ok{x zRoW2&I*2zwfCkbFA@ki5aJh6i2~NW$sZ0OW1rVZkk~-Em^?XhGpDm`22zgDyj@COJ z104lRhf|N#nxixHi70&4Pjb_D&!ARPzlUBKub#`ey~s;v`p}5{hCJ6Qp)MfQhNYoO zJ0{VJ8%H#BQdLGls2Fuj%K9u}n8KS6*RlDtV6HvEJ~Q%I2%#=}LgN+jErn<2NxN&z0|wNNekyT8y?srkLX)s|ok7Oq*u|p3q<7R@NkQ zBmj|*|KdH7b-4;2vE!>0r`QSP@EG~^+nwU;10XT>>TpU(ds5MjvqV?z%<@fF_Q@#C zC&*QgL}38*-<@n#3H(X8CVS6Qn&Ui^)>85}&LStbMI0P(diiX-|ua-KT3nZq^hnHuZ^RtAz4ku!T1N~ zwqz3Sd>1c%MZOBuiz%wGbkU_8I&`Z9pSH-HO|{LTxoOKt%SwC0d=EbM7Xg}}m={hH zE3~7tgR*=$)QWTXqZN4Q(hZ|raawxkCpenExRN%9ktLwTPoBOi5(2m8Xb+X^%<^RO zt;hwJd<3l@dMOA-d^418|8(aCR{JB+G7HpoM78Hz`yKN|sw4)zp$ai>Yj@Wvrr2zB zOoeySNHq51m!Fr#B-_b4+@+)sx$6%4=H(T3#knYtd!iIo;ltE2TKGZYn|=5c$w(Ux zhIL9|{2!2Ig5Ha;hajmW`G?6E;cbhEH@M}Vzfuxy^}$u0(Ve%$h7#|{$~;q`;J<2#7pLW0-LvR`--}olbu!2#b-G`&P8s*M zRubAoo|HFZamFKUU&>u*Z0t>>e;bT5|uXXZ&9}O<>DSItv0$&n`aGOCyMJ{8Af5L( zj#*-zZ|Q0#d8JfL=+2lIC1Te@cyz!E;W)9J3Ft{U$xu`r61rA+o;sO zY!)bhk9;SjpmIX81zU&5Z&k8O?5FVAZOUFVXaQ+Kdu5P7hYylFfKm-cmK6E|!7l<- z>MeWgGKGXO<>V|o9q_fdA8iib866#_&O)ZKpx*(Eqt2Q18OCRp5N|i|GDk1(c9W6K za5~Ss>XMH+x1~9sMEUCovxLWCgnZU0`guk@{Rd{$+A_uJhQJm$m!1k10%X(XaxcM1 zoOHsw;=+ENWjtK>L}Xe0+Z=uI=!tb5i&1%Huxiznl&rwpkGWH~isHq@*mc<`=E&<) zln!LdPi6YP!Np{>62>lStDJXt@aHCKvR$s|{?m28xdA~ECe+t@q?fI@Us!>q%Luj4}2GxBYfWP8z;MXit zJI6BYGOMmx8LRMM^}U{+Zu}DVb{LkUQv?>v04{!cLA$eKJ;lfgn6A;&MzHd}l3Sit z)a2Af@glf0L2YkB*sey7+6B41)_~wWSd?l>PC((S*A+>2T+r%E!j4jNG;Ciw1vn+3v4|%dXTkE3QkKbq4`O&`GP{0&9ng*40mCAGlta z`jW9Q!EQ0dM+mdLZA|NHa&g2gx|U7bGNh%9xyc+Y%w?HefrU+XUp@3Ni?g<6dNb{p zG7Mu0tO&No*(Z?{3J?%eO4P5h6B+9?dxS;D_EJ`S?qt>~0cpg1km=T4iJk69dzqe# zco{{P<$d#pr$z)8vX9}Ty^V>xQz7H&Se37?z3Y@6!pe1Kv@{>+sTa9xHA4krK8rX{ zwZA9T(@{p@s#UxGa$o&zM<+_A;h3mOu0=v-T%g^xq%~R-%=&AR;zV?{Ld+AnCVEeVk@oKxmjk7^rE1pN-&-#zef-FAF zkiB^!iPu4Y#;vOxt>EQ=8_}+L=%bwBY|?(r)50|arV1C@CntGxm*Q~ z(cP^FAj^@i6c0Xs_KD3A{Lv!tf_3Oa*+GWlQDQN5Io=B)0`!d)9W2hU#4^8j`V|xY z?SAwvk*v_%)*NfE6RJ()7*jbpUlbB_X}NA%USZ+R{t9weocy{PJN)!*tzA=YBqD;(1Mq#J1u!Cck*t$p=6JdRZ;1P4-0vn-o&!a zi%8FIvDbE99c-jDYO^%L4P6enVXiO_c%|qc`N;l<`50fO2R1HIDT(2y4b4jBhw5%E z0ys(d2iIbH&}-F>c73$;X9F9@RrjFf8)+kU#7DbM_0EA^1fA{O(P2Dug{smzztt)J zpw*y={sPtMcXsT^{hrC1~$Lbw2=Wel%_yHf?S#f<3fj~)?rJ#sd(PRNv6h$ZJM5npQ=%;Yb_G-pxEmkD5 z$2l4xRtk_aX*q!BI??h2=6`5(xdDlBKKUE8Vr`vnPeYSE2SWXB#DCO6IAi0HK&txN@BS%*RqHQwDV^A# zOVHzr;-pU9_J3R37M_K)`ZXWPF%48CRxt$jMXe2?T8zp#Md>?&I+sFGTGLy2rB^L< zdozfe*JYjDkU15zQC|d#Np>tj-OJ!lcAel&z_h*rD^%C6N*G>n$vFKkkDHr4`BH8o zAO9srzB5&7pcNYFoOI zqg+g1qlSz<=m{}by;+~RM%X(=crF*Gb`Xob4*E_bvxNY)(B6w}^AWjvR#skI83P8} zNteb&5S!X1DL6N-yagP3waDbbZa|`&hnt1U2Zr)6=~^8YJy(nsHyMfhoz&kO`JFJ> zwn;U(I20-e%Qtqwift5~9P3Yqgcwt5Tar&)b0UD^9q5k@nE*55UagvZBW2!7%H?-h zu2G8?WqGLGVmw0)D8@&|CRn@QC&P*D)bJrW*@x5(!flc956YouAD0KRp zFX=1}?_FQgU7fOCUIGoA0AN{aEBrs}j?T#F`yS*R(+7wrMPGkUPpk6LY6WH-Namzr zTDy3ru}sfGYq&?wi0-y6^4# zA|ihk$oc;=R@TJC);Q%&-Rx``y?X3Rfb*}dS-4&Gm@uv-IcF+{N&2Z;)a1!ScGk4| zFNZ&nApBQbZQIRpz}Och5PMf(S~qM$y>b#ubhEdUM3vL*v%5{TZJ^x6G0isQ&;|ie%szwcnqsw`ET8TxWeE7wA+`=+m1{>-tIi#04sWJ|(==Nf0 zo)z}5ae>0(t!4!16b~p)4k~D&&h>l%D5?lhB~Y|Xi&-eTI*_>F4&R*sXkYFe@M{9N zLSbRhaI8VkMV!e;_-r6NNXxfjzY#!GY9xMYQ&Ypb8fU*|5T<_RN_WX-hz;BY6ac%~ z__@*6MtZD4^CwqfmA}14`p<2)^V7m~-0;mNR)4<6uW|{S%?STXtp7f1F&_Aolh0B5 zgHIF;l!aearR=(X!OXB^_Mo!Bok4DWeYmMt(R)q>m#&uBVakDQnvM=>;?DkA`ldGc zg+28N7=sKfK@B@hbnwV&%_QgMMOo2G05ST@dn4ma%emm-1arJm6e8kR1KB?_io1stG+FWJL ze7Z{&j)Los0Aa(DZx53+kBd;Y0RU$8ckk9YNAEVynUgIg8&&Cuy&f^yZ=Sa40(DK+ z`Fda7Yu{CuS6P@I0kPTBWZBmif!{;~>im;FYVMTi8bG(`c!K$xS-g0JUphJ^%%la= ze+A@^_x%%72KJbucMu>tIDj)2=Rp7#nG0F>`L9f=SiQ2WhxC8prT;-t@xAE1ICj|~ zT~IBTn?uf);1LxJCTVwW+OeCUSiNn zL*rVjEPf=CtPyB$wfkbLMyyAxU3Z|1IkOyV!3oAOLPK^#0l_(;!IvdPs{4fT@xW_6NMXbo!`j0c@XbTk6h>aR%ISNE9SA2hxCQszmNK;`4=MsU&n;|TWbdkq_lBo^LVz&2fjmRZCFPoC4p@#eC9#MG zEbA#OLxB9g&ch$dR!HtlT(YJ7(meg=3zvI6r~OxNzuIk*?w72vtUQ#s`LVBi0r3l# zxW1UIswg+Nsr;xRKA~A@8ZXUI)7jZxwIA$+zv0*a$+Q2rJwBvG;)v=X<;LF#pr94z ze<}R+z#r87e`|vLH~#0xdwha6L$nbW2(e{7^FW10`A7kFN?BKYzyg~nAM1>`rdj7~ z(*qP1mVbP4i>d*nJ!Af)w;@JB@e^)D$D4)H$dsHf1e6C!>+hx;VdUe<#MLAB@T#>e z%AFXVPKq&3kI##0?j?FG)uG581O}vr4PVKePA&I(&GxTgc9r^-hb-#ZialDv-l=6t zS6JM~vAk3~$|#*`ElJ|@byT^1Ib=o2S((f}=;9JX&VgA-G_cOKhKfR`l$9#`czR^j zPSnA(rnN81C56B;+VZfil6c;>wPhzYUgqma9_}(@wEEQf)|uXLxt#NRFp2ueYO;`r zb|>y#x|1-$2z~0pIT<&6tCRVF6KgKWvyo$&VJ0P6lYaCsq3>5Trt5Bd5u!goqp@T< zNY05(5SyXQ;ImFJ=~%K{nkN0GOoLs-&J#iuB^Xw2MM*t#Mwt{=wEW@3%L^y*J->E5 zT4-&eKeC$89bwQBX1__VvjTLRGAt84G?mLg*Y_J$V(#8uOEMh4QvOHHR~akGXp$`N z1nQ*V#PTN`+YAo*o7(m&li6pIsYj1zmATZVs!D|j%*ZlFZW*^Y zdU#B2b$2i@nouwd1>YZN8GCgo-B!6sP?6~$x=@#d4Yo9K)t*1|!^a;p46>ILz0@q% z?f+0Qc_Ms#CrirQ#EGdu-i_+(tf;q8PXRKwJ2ma6jG6;j?bxN#lojkuw=QNU&UXAM zT>d-R$Fw){8qX%^o_d}`T6Cq`GRghCvbkbznrHl6!%Sh3aQJNP{eF6Cleti-$Klt^ z34N6;flCKBbsv@Tb-Eu`EWA3R6nrMzC9{~+yzSTi)b>OAq*V%`sLoAU_JmrZs9$)pjH@Qs<_O982%m)^bLpXGS-wT3 zh-^$QCU0CxM^6C%^fjYr0_UawxD*dl>48L;fCcCBk#pdn-t*WYZgOD;pNzC)gXZEr z_KJp=xNsfmE59Y*xgP{ITWlbaCJRUsJB^Li3`s29U68O$B1wyQt-*rANFp6? zA;M+U>Io|=)ZI*()I7a3YYY;q3XmAqwo0O7k1gLC%-y;#W zKiwy|1Ot0N*U}$_x)3lwl;cbv0W)jR*Hx?6!sJ)AUE5JLRj%*S-le4#C!88I8^kq#xCpa^ap81-X%klvE z3mxIU*}={@dF1AJLMSX3&oADTdCuVug){xCBffASrkJ&4?VVh0eao7F7_N755tf76 z`pxd4t4|5N3o{%us85B_GvdB^Lf68ng{J(=W|Neu)`?n6StF_GxG>Zkx4aSrX#Jok zcEnTX5f+}&n~n+|W?Zcq2U2$n9Pgl5=X#f31DTzbgntUy4p~QRW}qWvhyuodj?!bZ z#}uaQ)X`3t1UA?xopEna8q<7Eq&wA%BI8sY977*|wU&P_Bt0VfFdbrMsASDgC^!Eq zim)k&kNl!(S@bj;EbW6I1YmWHKVYeU29EkK@vHwMKoB^N)PEibmwHVC#KQT&k_xg> zXR^OL^1ml_|H?A`uad+^Ly11ckSS1?pxhQe-U4RNbi&7FfBe0EQIInfM{~@%qe4&4b}wpf5xP+IPM)@RAkgMMgT1&mZO_Qi*W$#| z>QPmF|GD5yqi#E{wNK^_OvN}Ty~x$?@!!hR-CI5PZqMzV!)hYdlzqvz$|PQ2F>S5hq+biedz%kP^9mV!Jw`qaH(ni@_=I?ZSIJ@M;5 z3i_qPVr%p)1zA*N=U@S|qF-i?NGSSQKD(t)-amc6V^JuwEjhRk=7Mq`CsGAl8^bWo zDrMyY({d>c{jxK~w`GU#6)t?O9fopqgBuBZ`LXxO@nyys7%-AqPZh<|M7B7hz${8W z+E%^UQ#5kUwfBoYKbSr{{F+VUsS3{6gn5Qcg%+k*q8(3uw|(iasnyvz(`dT#Mp=or zGw_G<4r2-Ak=dL(}#Zywa2 z?^4d<%KcCrW6joK=5o;O%f_XD9kgyQ+QbPh&89NU+S$e59-WZLpF*|q`1fvULdiM} zEJeVl>G-K`*;D#Pl9ip+v1A?$Q;vcqw<#DS!vfxBY^H0$$0>cjjPzgEdvL7#%r-!% z>i20p9b`+5cZtRZ^;ov$KD*(mbUDqn7H8%N%P*Z# zvJJaq_&_oIqrHeHw)drSqD!uHuD<1Iv{7p~_m-SSrlRBh6;qS#KedRm6tVHlw<_5@ z{g4B5GbmtZ7Bz`$^K=-C8kQN_;pXP%Jf42&Q7B2$iHjY|%EID+!WI2{<8f7e7nooA z7U>!Q)_SDg{ObAHhDc$5YPQcb^;aEBb>#CY()L z61!^7Q(93 z0wj3@u!d0qh~Y?y?w6)DU99VI_N%fiH-1YgmjnRymnH7UdTDsmp~?-$vE4OEH^4;_ z7*mxM{Tj$FEb+^gWKRi#AR7_@M>*LDE?-z0B)mE!tO_u_o{65s_uFShe+dd9{01KNf{MN}*V{3}iuIU<|#?Ro!s7 z0Nz!=R8bYTOR*}*&+tM#v_awkNrT`630DBHC~L^c-|;g0y@2rb<4cS{DAe?;FImFg z+CgQEE!AG(=xY4JXHy3QY5U0esdJ+x?>u2Ekf!iic<+AWYmtEApoZkogOIY}n3@@pzOYhv`<0=jP#j;)zLVW{rcQ=u;WlzZ+9)rVs5SCvFGVDR7|cYk*crJ?Ejnl?S#rsehm8P~mya1gHFNvfqFKue zoPDw4@d|ahN_;(2B*(&xCw9yZ#r@E8_&Z&K zN$XkcCZ}8olACXD?~Y}RcuS>Q;%?_`#kv-})rqx?k;D6J@3D??B(Me(N}!g&*gu|? z-))1Iy6=Ha8FmTWbzn&i?j>HCF<4VwMU*r{jcXL@)W23^SAnX$*H@NhT0=y0&5)I(-H>Kg&ky;4FZJK}!CVHK>Zodq3;cN^E z!TsJY@ifYaaAUwI^Ic75f}v>r)5cc@s|WZq=EBYRV(n0mlxb&$B7a)i;c^dfqSigj zQ|*@wy$TTzDi3H)cVb+N|4i!a%N_zxrA)`B*W1PpZvp;yp>V~ zp*J6E>LMFvD|eP$=D!aAFkX)2vKX%&`4%OP{OFKB#@pWB(57(o( z#kz`jaE77XwiT&Y2@cj1dv0NVy||{c^Rzb{g*C}I5K8yFxh0F(H6L;Y;u7qk7BNefE-e-^QY6V3b=+}FVz=}xNfbu8?$PqZX`A6b z_l$VxHI9w|O->1Yv%{-AHTamdqr+w$lEKCRf}3V~uv0}RuLa|DnxE%q`J`(O+pWX< ztF%PCsfc-wBK#|1%k!v*?bL%k-0G#phcpGcWHD=Hd6qm%qDpt7aNe#|sdqkZXT%Nf zE9bv4R~jvt`mWMYSq)Q@QXqXBBe&w~^;%$cj-tYMOCo^7f%s`Y*mF_e)E2wy!RfY| zd)k!!Yli0RRD;3IBe{}$&zrt0W9WYoKDnL#X~H`@eY2i%P$2jN*YT`w-=0OA!JSXq zZ@1rgdX2e#5H-)-fQdv&nxQn4~r}1VCeb!NXswE=;trug! zi&8k=F}WyvMqn<{+HnCRfi%+h81;>NzJM%%c}9=}EH}eJpn`q3NQ00}MfHjx27Hsg z=0OLpm@X?yT+fa<_ekX1O{ji;LKUoM0nMm|T|JRG9TQNIr*wX9ca9TdcaFzq(@rxu z2KcZZXMhFrpW@|z7Bv2k?v1xe3=7MQQ5{}2?XG)dOrM!)zL(*fzLU8;Jypp;pdEI- zEl4bLJ{&velH@4-vD{Fn-H=q_VpMABN&icB*0dUB`gSZP zL7;%CvAHfHJZSB+CnAh;WU>qFJXsT*rCprRcTo|h4#cY+!nYr+M5mxT|S)N&Fy}xuUYlV#@t)Od9ww?olh%h`AeuIe`p3=US|g z7r=tXwR2HvvDQ2mz&kjsTGwaACZ11s?q|BbzJpiCy?8fWu#B&I8LbP53W3ce6gd zik-U^gQVg5NJnzvjSrq<*#bc#==|=@3p;IZ4-$a-Y*~FL~#=SiZGL~jV&NR zUlaLJhW{1x{(t>*u}h-7k^b;LT8H*aSk;FwGmbmESa@MqkMH+iy*{bahgaBH&D|Nc zHCzVIxW@6cVw?>o6YnBGNbG)ajNf9YcWFT!8S6kn;IE1j_V-WT5_v-35FjE;-%j|n zlZ*2~b7UMIkC~WPnDt;z#%Nnon3n#KZBecIVs5&BZ#j`cW=nqf?EZUp_^axPrSa4Q zl5wBMf+nm}shFLR>XBGhl&Na;v-ec3DeE|iIb1q+b z3$i4}Ac=d{=*DjzVTfS}_?A8cM5V zs`_d@A70j|?}`!7(b`5(hkf(vv1rIz9ea?lRQ!7FE%YM0#gTB`DA$zUh!$>r(Qf?x z2)Sn8iERTsQuW6_Du2Fz-unF6W*>XlW_I^#5BKhDqAMS9S19YX2|myKgM2bg_`qB{ zn6O3jF6M7yj}`Z86ikU>o~wt^7sFuD zhp7`{&668bK3`{i%gP0F0-kvWG+xvsS7b_q%$gRH(8t=msC-D20{WJ3Txwk-3mhS} zUtYlZ>2Nn#6e*1d`G>Ou=eS;X(tV^Ff5nsHt(i%R6z2rAJL;S3+kP@^UIC-<#R4;! zzK4`%7(=jK#skaswu8Nbd+AiFl4p=mqxBfdA5n*9goMyN9ZGw{H!nH3GODHF!`xIs zc2xmP)fMndqC&gel0isL=|Wp`ad!Tw;%; zz2Tj27Qsz0^)%T?u(mv`Kd!Kr>v{Sc2ov4+FoFA;@5}tuw4R|w0=TsV_B5!FVLr~C?eaB28)Wm)PEh~M=LoER zb=4_%bIIY|J@8JslHe3h=A;9)U}RcP6$3t~8JDE#L?9QqTkU|=xp)nD#YmKt#I@#` z@}^|&x-Lh>H%YLr(*$M`pPNf2Dg`RVJsCDF_OEQUI*p7^xAl15hO}}_(+{;$eRSLH zHRXWa5-jNZlVMwv3WL~9fxHL)?u$rttP21?NZ#Sg##us$ZfIY$vDop0qe3&7Ow9e? z045d&vQ%l1d7)o=R-TK-kj*VMG27NstZ<5}^L}NK2?vj|>HW@~2FhpZ;BBabMLu1qfLu^9^9$VsX<-Tg6*=YMU0hr06#XclU4Iy!8-lwbB;rj@vXNi|dq&-C$CLr4Hou8sK*`F)sXv9w0)E94#AJ#)ctS#UBJKlt-t>}WWhj8{x4hE-zxwB literal 0 HcmV?d00001 diff --git a/doc/source/ray-air/user-guides.rst b/doc/source/ray-air/user-guides.rst index 98a6cf99cbad9..388048e467257 100644 --- a/doc/source/ray-air/user-guides.rst +++ b/doc/source/ray-air/user-guides.rst @@ -22,6 +22,15 @@ AIR Feature Guides :type: ref :text: How to use AIR Preprocessors? :classes: btn-link btn-block stretched-link + --- + :img-top: /ray-overview/images/ray_svg_logo.svg + + +++ + .. link-button:: /ray-air/checkpoints + :type: ref + :text: What are AIR Checkpoints? + :classes: btn-link btn-block stretched-link + --- :img-top: /ray-overview/images/ray_svg_logo.svg diff --git a/python/ray/air/checkpoint.py b/python/ray/air/checkpoint.py index 8a0c830e7aec3..cb65d207dcdf3 100644 --- a/python/ray/air/checkpoint.py +++ b/python/ray/air/checkpoint.py @@ -33,7 +33,7 @@ @PublicAPI(stability="alpha") class Checkpoint: - """Ray ML Checkpoint. + """Ray AIR Checkpoint. This implementation provides methods to translate between different checkpoint storage locations: Local storage, external storage