mirror of
https://github.com/Azure/MachineLearningNotebooks.git
synced 2025-12-20 09:37:04 -05:00
Compare commits
2 Commits
jeffshep/w
...
jeffshep/u
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
113543dfbf | ||
|
|
f0d7379af7 |
@@ -36,7 +36,10 @@ except Exception:
|
|||||||
last_train_time = datetime.min.replace(tzinfo=pytz.UTC)
|
last_train_time = datetime.min.replace(tzinfo=pytz.UTC)
|
||||||
|
|
||||||
train_ds = Dataset.get_by_name(ws, args.ds_name)
|
train_ds = Dataset.get_by_name(ws, args.ds_name)
|
||||||
dataset_changed_time = train_ds.data_changed_time
|
dataset_changed_time = train_ds.data_changed_time.replace(tzinfo=pytz.UTC)
|
||||||
|
|
||||||
|
print("dataset_changed_time=" + str(dataset_changed_time))
|
||||||
|
print("last_train_time=" + str(last_train_time))
|
||||||
|
|
||||||
if not dataset_changed_time > last_train_time:
|
if not dataset_changed_time > last_train_time:
|
||||||
print("Cancelling run since there is no new data.")
|
print("Cancelling run since there is no new data.")
|
||||||
|
|||||||
@@ -94,15 +94,15 @@ def main():
|
|||||||
os.makedirs(output_dir, exist_ok=True)
|
os.makedirs(output_dir, exist_ok=True)
|
||||||
|
|
||||||
kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
|
kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
|
||||||
# Use Azure Open Datasets for MNIST dataset
|
# MNIST dataset
|
||||||
datasets.MNIST.resources = [
|
datasets.MNIST.resources = [
|
||||||
("https://azureopendatastorage.azurefd.net/mnist/train-images-idx3-ubyte.gz",
|
("train-images-idx3-ubyte.gz",
|
||||||
"f68b3c2dcbeaaa9fbdd348bbdeb94873"),
|
"f68b3c2dcbeaaa9fbdd348bbdeb94873"),
|
||||||
("https://azureopendatastorage.azurefd.net/mnist/train-labels-idx1-ubyte.gz",
|
("train-labels-idx1-ubyte.gz",
|
||||||
"d53e105ee54ea40749a09fcbcd1e9432"),
|
"d53e105ee54ea40749a09fcbcd1e9432"),
|
||||||
("https://azureopendatastorage.azurefd.net/mnist/t10k-images-idx3-ubyte.gz",
|
("t10k-images-idx3-ubyte.gz",
|
||||||
"9fb629c4189551a2d022fa330f9573f3"),
|
"9fb629c4189551a2d022fa330f9573f3"),
|
||||||
("https://azureopendatastorage.azurefd.net/mnist/t10k-labels-idx1-ubyte.gz",
|
("t10k-labels-idx1-ubyte.gz",
|
||||||
"ec29112dd5afa0611ce80d1b7f02629c")
|
"ec29112dd5afa0611ce80d1b7f02629c")
|
||||||
]
|
]
|
||||||
train_loader = torch.utils.data.DataLoader(
|
train_loader = torch.utils.data.DataLoader(
|
||||||
|
|||||||
@@ -200,18 +200,14 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"from azureml.train.dnn import PyTorch\n",
|
"from azureml.core import Environment, ScriptRunConfig\n",
|
||||||
"\n",
|
"\n",
|
||||||
"estimator = PyTorch(source_directory=project_folder, \n",
|
"pytorch_env = Environment.get(ws, name='AzureML-acpt-pytorch-1.13-cuda11.7')\n",
|
||||||
" script_params={'--output-dir': './outputs'},\n",
|
"\n",
|
||||||
|
"estimator = ScriptRunConfig(source_directory=project_folder,\n",
|
||||||
" compute_target=compute_target,\n",
|
" compute_target=compute_target,\n",
|
||||||
" entry_script='mnist.py',\n",
|
" script='mnist.py',\n",
|
||||||
" use_gpu=True)\n",
|
" environment=pytorch_env)"
|
||||||
"\n",
|
|
||||||
"# upgrade to PyTorch 1.0 Preview, which has better support for ONNX\n",
|
|
||||||
"estimator.conda_dependencies.remove_conda_package('pytorch=0.4.0')\n",
|
|
||||||
"estimator.conda_dependencies.add_conda_package('pytorch-nightly')\n",
|
|
||||||
"estimator.conda_dependencies.add_channel('pytorch')"
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -578,7 +574,7 @@
|
|||||||
"name": "python",
|
"name": "python",
|
||||||
"nbconvert_exporter": "python",
|
"nbconvert_exporter": "python",
|
||||||
"pygments_lexer": "ipython3",
|
"pygments_lexer": "ipython3",
|
||||||
"version": "3.6.6"
|
"version": "3.9.18"
|
||||||
},
|
},
|
||||||
"star_tag": [],
|
"star_tag": [],
|
||||||
"tags": [
|
"tags": [
|
||||||
@@ -659,5 +655,5 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"nbformat": 4,
|
"nbformat": 4,
|
||||||
"nbformat_minor": 2
|
"nbformat_minor": 4
|
||||||
}
|
}
|
||||||
@@ -245,7 +245,7 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"from azureml.core import Environment\n",
|
"from azureml.core import Environment\n",
|
||||||
"\n",
|
"\n",
|
||||||
"pytorch_env = Environment.get(ws, name='AzureML-PyTorch-1.6-GPU')"
|
"pytorch_env = Environment.get(ws, name='AzureML-acpt-pytorch-1.13-cuda11.7')"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -366,7 +366,7 @@
|
|||||||
"name": "python",
|
"name": "python",
|
||||||
"nbconvert_exporter": "python",
|
"nbconvert_exporter": "python",
|
||||||
"pygments_lexer": "ipython3",
|
"pygments_lexer": "ipython3",
|
||||||
"version": "3.6.9"
|
"version": "3.9.18"
|
||||||
},
|
},
|
||||||
"tags": [
|
"tags": [
|
||||||
"None"
|
"None"
|
||||||
@@ -374,5 +374,5 @@
|
|||||||
"task": "Train a model using the distributed training via Horovod"
|
"task": "Train a model using the distributed training via Horovod"
|
||||||
},
|
},
|
||||||
"nbformat": 4,
|
"nbformat": 4,
|
||||||
"nbformat_minor": 2
|
"nbformat_minor": 4
|
||||||
}
|
}
|
||||||
@@ -51,15 +51,15 @@ if args.cuda:
|
|||||||
|
|
||||||
|
|
||||||
kwargs = {}
|
kwargs = {}
|
||||||
# Use Azure Open Datasets for MNIST dataset
|
# MNIST dataset
|
||||||
datasets.MNIST.resources = [
|
datasets.MNIST.resources = [
|
||||||
("https://azureopendatastorage.azurefd.net/mnist/train-images-idx3-ubyte.gz",
|
("train-images-idx3-ubyte.gz",
|
||||||
"f68b3c2dcbeaaa9fbdd348bbdeb94873"),
|
"f68b3c2dcbeaaa9fbdd348bbdeb94873"),
|
||||||
("https://azureopendatastorage.azurefd.net/mnist/train-labels-idx1-ubyte.gz",
|
("train-labels-idx1-ubyte.gz",
|
||||||
"d53e105ee54ea40749a09fcbcd1e9432"),
|
"d53e105ee54ea40749a09fcbcd1e9432"),
|
||||||
("https://azureopendatastorage.azurefd.net/mnist/t10k-images-idx3-ubyte.gz",
|
("t10k-images-idx3-ubyte.gz",
|
||||||
"9fb629c4189551a2d022fa330f9573f3"),
|
"9fb629c4189551a2d022fa330f9573f3"),
|
||||||
("https://azureopendatastorage.azurefd.net/mnist/t10k-labels-idx1-ubyte.gz",
|
("t10k-labels-idx1-ubyte.gz",
|
||||||
"ec29112dd5afa0611ce80d1b7f02629c")
|
"ec29112dd5afa0611ce80d1b7f02629c")
|
||||||
]
|
]
|
||||||
train_dataset = \
|
train_dataset = \
|
||||||
|
|||||||
@@ -280,8 +280,7 @@
|
|||||||
"# get a curated environment\n",
|
"# get a curated environment\n",
|
||||||
"env = Environment.get(\n",
|
"env = Environment.get(\n",
|
||||||
" workspace=ws, \n",
|
" workspace=ws, \n",
|
||||||
" name=\"AzureML-sklearn-1.0-ubuntu20.04-py38-cpu\",\n",
|
" name=\"AzureML-sklearn-1.0\"\n",
|
||||||
" version=1\n",
|
|
||||||
")\n",
|
")\n",
|
||||||
"env.inferencing_stack_version='latest'\n",
|
"env.inferencing_stack_version='latest'\n",
|
||||||
"\n",
|
"\n",
|
||||||
@@ -490,7 +489,7 @@
|
|||||||
"name": "python",
|
"name": "python",
|
||||||
"nbconvert_exporter": "python",
|
"nbconvert_exporter": "python",
|
||||||
"pygments_lexer": "ipython3",
|
"pygments_lexer": "ipython3",
|
||||||
"version": "3.6.9"
|
"version": "3.9.18"
|
||||||
},
|
},
|
||||||
"notice": "Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License.",
|
"notice": "Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License.",
|
||||||
"nteract": {
|
"nteract": {
|
||||||
|
|||||||
Reference in New Issue
Block a user