update samples from Release-140 as a part of SDK release

update samples from Release-138 as a part of  SDK release

configuration.ipynb

@@ -103,7 +103,7 @@
       "source": [
         "import azureml.core\n",
         "\n",
-        "print(\"This notebook was created using version 1.36.0 of the Azure ML SDK\")\n",
+        "print(\"This notebook was created using version 1.41.0 of the Azure ML SDK\")\n",
         "print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
       ]
     },

contrib/RAPIDS/azure-ml-with-nvidia-rapids.ipynb

@@ -188,13 +188,6 @@
         "### Script to process data and train model"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "The _process_data.py_ script used in the step below is a slightly modified implementation of [RAPIDS Mortgage E2E example](https://github.com/rapidsai/notebooks-contrib/blob/master/intermediate_notebooks/E2E/mortgage/mortgage_e2e.ipynb)."
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -373,7 +366,7 @@
         "run_config.target = gpu_cluster_name\n",
         "run_config.environment.docker.enabled = True\n",
         "run_config.environment.docker.gpu_support = True\n",
-        "run_config.environment.docker.base_image = \"mcr.microsoft.com/azureml/base-gpu:intelmpi2018.3-cuda10.0-cudnn7-ubuntu16.04\"\n",
+        "run_config.environment.docker.base_image = \"mcr.microsoft.com/azureml/openmpi4.1.0-cuda11.1-cudnn8-ubuntu20.04\"\n",
         "run_config.environment.spark.precache_packages = False\n",
         "run_config.data_references={'data':data_ref.to_config()}"
       ]

contrib/fairness/fairlearn-azureml-mitigation.ipynb

@@ -49,7 +49,7 @@
         "* `fairlearn>=0.6.2` (pre-v0.5.0 will work with minor modifications)\n",
         "* `joblib`\n",
         "* `liac-arff`\n",
-        "* `raiwidgets~=0.7.0`\n",
+        "* `raiwidgets`\n",
         "\n",
         "Fairlearn relies on features introduced in v0.22.1 of `scikit-learn`. If you have an older version already installed, please uncomment and run the following cell:"
       ]

contrib/fairness/fairlearn-azureml-mitigation.yml

@@ -6,4 +6,6 @@ dependencies:
   - fairlearn>=0.6.2
   - joblib
   - liac-arff
-  - raiwidgets~=0.13.0
+  - raiwidgets~=0.17.0
+  - itsdangerous==2.0.1
+  - markupsafe<2.1.0

contrib/fairness/upload-fairness-dashboard.ipynb

@@ -51,7 +51,7 @@
         "* `fairlearn>=0.6.2` (also works for pre-v0.5.0 with slight modifications)\n",
         "* `joblib`\n",
         "* `liac-arff`\n",
-        "* `raiwidgets~=0.7.0`\n",
+        "* `raiwidgets`\n",
         "\n",
         "Fairlearn relies on features introduced in v0.22.1 of `scikit-learn`. If you have an older version already installed, please uncomment and run the following cell:"
       ]

contrib/fairness/upload-fairness-dashboard.yml

@@ -6,4 +6,6 @@ dependencies:
   - fairlearn>=0.6.2
   - joblib
   - liac-arff
-  - raiwidgets~=0.13.0
+  - raiwidgets~=0.17.0
+  - itsdangerous==2.0.1
+  - markupsafe<2.1.0

how-to-use-azureml/automated-machine-learning/automl_env.yml

@@ -1,30 +1,31 @@
 name: azure_automl
+channels:
+  - conda-forge
+  - pytorch
+  - main
 dependencies:
   # The python interpreter version.
-  # Currently Azure ML only supports 3.5.2 and later.
-- pip==21.1.2
-- python>=3.5.2,<3.8
-- nb_conda
-- boto3==1.15.18
-- matplotlib==2.1.0
-- numpy==1.18.5
-- cython
-- urllib3<1.24
-- scipy>=1.4.1,<=1.5.2
-- scikit-learn==0.22.1
-- pandas==0.25.1
-- py-xgboost<=0.90
-- conda-forge::fbprophet==0.5
-- holidays==0.9.11
+  # Currently Azure ML only supports 3.6.0 and later.
+- pip==20.2.4
+- python>=3.6,<3.9
+- matplotlib==3.2.1
+- py-xgboost==1.3.3
 - pytorch::pytorch=1.4.0
+- conda-forge::fbprophet==0.7.1
 - cudatoolkit=10.1.243
-- tornado==6.1.0
+- scipy==1.5.2
+- notebook
+- pywin32==227
+- PySocks==1.7.1
+- Pygments==2.11.2
+- conda-forge::pyqt==5.12.3
 
 - pip:
   # Required packages for AzureML execution, history, and data preparation.
-  - azureml-widgets~=1.36.0
+  - azureml-widgets~=1.41.0
   - pytorch-transformers==1.0.0
-  - spacy==2.1.8
+  - spacy==2.2.4
+  - pystan==2.19.1.1
   - https://aka.ms/automl-resources/packages/en_core_web_sm-2.1.0.tar.gz
-  - -r https://automlsdkdataresources.blob.core.windows.net/validated-requirements/1.36.0/validated_win32_requirements.txt [--no-deps]
+  - -r https://automlsdkdataresources.blob.core.windows.net/validated-requirements/1.41.0/validated_win32_requirements.txt [--no-deps]
   - arch==4.14

how-to-use-azureml/automated-machine-learning/automl_env_linux.yml

@@ -1,30 +1,33 @@
 name: azure_automl
+channels:
+  - conda-forge
+  - pytorch
+  - main
 dependencies:
   # The python interpreter version.
-  # Currently Azure ML only supports 3.5.2 and later.
-- pip==21.1.2
-- python>=3.5.2,<3.8
-- nb_conda
-- boto3==1.15.18
-- matplotlib==2.1.0
-- numpy==1.18.5
-- cython
-- urllib3<1.24
+  # Currently Azure ML only supports 3.6.0 and later.
+- pip==20.2.4
+- python>=3.6,<3.9
+- boto3==1.20.19
+- botocore<=1.23.19
+- matplotlib==3.2.1
+- numpy==1.19.5
+- cython==0.29.14
+- urllib3==1.26.7
 - scipy>=1.4.1,<=1.5.2
 - scikit-learn==0.22.1
-- pandas==0.25.1
-- py-xgboost<=0.90
-- conda-forge::fbprophet==0.5
-- holidays==0.9.11
+- py-xgboost<=1.3.3
+- holidays==0.10.3
+- conda-forge::fbprophet==0.7.1
 - pytorch::pytorch=1.4.0
 - cudatoolkit=10.1.243
-- tornado==6.1.0
 
 - pip:
   # Required packages for AzureML execution, history, and data preparation.
-  - azureml-widgets~=1.36.0
+  - azureml-widgets~=1.41.0
   - pytorch-transformers==1.0.0
-  - spacy==2.1.8
+  - spacy==2.2.4
+  - pystan==2.19.1.1
   - https://aka.ms/automl-resources/packages/en_core_web_sm-2.1.0.tar.gz
-  - -r https://automlsdkdataresources.blob.core.windows.net/validated-requirements/1.36.0/validated_linux_requirements.txt [--no-deps]
+  - -r https://automlsdkdataresources.blob.core.windows.net/validated-requirements/1.41.0/validated_linux_requirements.txt [--no-deps]
   - arch==4.14

how-to-use-azureml/automated-machine-learning/automl_env_mac.yml

@@ -1,31 +1,34 @@
 name: azure_automl
+channels:
+  - conda-forge
+  - pytorch
+  - main
 dependencies:
   # The python interpreter version.
-  # Currently Azure ML only supports 3.5.2 and later.
-- pip==21.1.2
+  # Currently Azure ML only supports 3.6.0 and later.
+- pip==20.2.4
 - nomkl
-- python>=3.5.2,<3.8
-- nb_conda
-- boto3==1.15.18
-- matplotlib==2.1.0
-- numpy==1.18.5
-- cython
-- urllib3<1.24
+- python>=3.6,<3.9
+- boto3==1.20.19
+- botocore<=1.23.19
+- matplotlib==3.2.1
+- numpy==1.19.5
+- cython==0.29.14
+- urllib3==1.26.7
 - scipy>=1.4.1,<=1.5.2
 - scikit-learn==0.22.1
-- pandas==0.25.1
-- py-xgboost<=0.90
-- conda-forge::fbprophet==0.5
-- holidays==0.9.11
+- py-xgboost<=1.3.3
+- holidays==0.10.3
+- conda-forge::fbprophet==0.7.1
 - pytorch::pytorch=1.4.0
 - cudatoolkit=9.0
-- tornado==6.1.0
 
 - pip:
   # Required packages for AzureML execution, history, and data preparation.
-  - azureml-widgets~=1.36.0
+  - azureml-widgets~=1.41.0
   - pytorch-transformers==1.0.0
-  - spacy==2.1.8
+  - spacy==2.2.4
+  - pystan==2.19.1.1
   - https://aka.ms/automl-resources/packages/en_core_web_sm-2.1.0.tar.gz
-  - -r https://automlsdkdataresources.blob.core.windows.net/validated-requirements/1.36.0/validated_darwin_requirements.txt [--no-deps]
+  - -r https://automlsdkdataresources.blob.core.windows.net/validated-requirements/1.41.0/validated_darwin_requirements.txt [--no-deps]
   - arch==4.14

how-to-use-azureml/automated-machine-learning/classification-bank-marketing-all-features/auto-ml-classification-bank-marketing-all-features.ipynb

@@ -1,21 +1,5 @@
 {
   "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
-        "\n",
-        "Licensed under the MIT License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/automated-machine-learning/classification-bank-marketing-all-features/auto-ml-classification-bank-marketing.png)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {},
@@ -30,6 +14,7 @@
         "1. [Results](#Results)\n",
         "1. [Deploy](#Deploy)\n",
         "1. [Test](#Test)\n",
+        "1. [Use auto-generated code for retraining](#Using-the-auto-generated-model-training-code-for-retraining-on-new-data)\n",
         "1. [Acknowledgements](#Acknowledgements)"
       ]
     },
@@ -55,6 +40,7 @@
         "7. Create a container image.\n",
         "8. Create an Azure Container Instance (ACI) service.\n",
         "9. Test the ACI service.\n",
+        "10. Leverage the auto generated training code and use it for retraining on an updated dataset\n",
         "\n",
         "In addition this notebook showcases the following features\n",
         "- **Blocking** certain pipelines\n",
@@ -74,9 +60,12 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "name": "automl-import"
+      },
       "outputs": [],
       "source": [
+        "import json\n",
         "import logging\n",
         "\n",
         "from matplotlib import pyplot as plt\n",
@@ -98,16 +87,6 @@
         "This sample notebook may use features that are not available in previous versions of the Azure ML SDK."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "print(\"This notebook was created using version 1.36.0 of the Azure ML SDK\")\n",
-        "print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {},
@@ -137,24 +116,27 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "name": "ws-setup"
+      },
       "outputs": [],
       "source": [
         "ws = Workspace.from_config()\n",
         "\n",
         "# choose a name for experiment\n",
-        "experiment_name = 'automl-classification-bmarketing-all'\n",
+        "experiment_name = \"automl-classification-bmarketing-all\"\n",
         "\n",
-        "experiment=Experiment(ws, experiment_name)\n",
+        "experiment = Experiment(ws, experiment_name)\n",
         "\n",
         "output = {}\n",
-        "output['Subscription ID'] = ws.subscription_id\n",
-        "output['Workspace'] = ws.name\n",
-        "output['Resource Group'] = ws.resource_group\n",
-        "output['Location'] = ws.location\n",
-        "output['Experiment Name'] = experiment.name\n",
-        "pd.set_option('display.max_colwidth', -1)\n",
-        "outputDf = pd.DataFrame(data = output, index = [''])\n",
+        "output[\"Subscription ID\"] = ws.subscription_id\n",
+        "output[\"Workspace\"] = ws.name\n",
+        "output[\"Resource Group\"] = ws.resource_group\n",
+        "output[\"Location\"] = ws.location\n",
+        "output[\"Experiment Name\"] = experiment.name\n",
+        "output[\"SDK Version\"] = azureml.core.VERSION\n",
+        "pd.set_option(\"display.max_colwidth\", None)\n",
+        "outputDf = pd.DataFrame(data=output, index=[\"\"])\n",
         "outputDf.T"
       ]
     },
@@ -175,7 +157,9 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "tags": []
+      },
       "outputs": [],
       "source": [
         "from azureml.core.compute import ComputeTarget, AmlCompute\n",
@@ -187,12 +171,12 @@
         "# Verify that cluster does not exist already\n",
         "try:\n",
         "    compute_target = ComputeTarget(workspace=ws, name=cpu_cluster_name)\n",
-        "    print('Found existing cluster, use it.')\n",
+        "    print(\"Found existing cluster, use it.\")\n",
         "except ComputeTargetException:\n",
-        "    compute_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_DS12_V2',\n",
-        "                                                           max_nodes=6)\n",
+        "    compute_config = AmlCompute.provisioning_configuration(\n",
+        "        vm_size=\"STANDARD_DS12_V2\", max_nodes=6\n",
+        "    )\n",
         "    compute_target = ComputeTarget.create(ws, cpu_cluster_name, compute_config)\n",
-        "\n",
         "compute_target.wait_for_completion(show_output=True)"
       ]
     },
@@ -225,7 +209,9 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "data = pd.read_csv(\"https://automlsamplenotebookdata.blob.core.windows.net/automl-sample-notebook-data/bankmarketing_train.csv\")\n",
+        "data = pd.read_csv(\n",
+        "    \"https://automlsamplenotebookdata.blob.core.windows.net/automl-sample-notebook-data/bankmarketing_train.csv\"\n",
+        ")\n",
         "data.head()"
       ]
     },
@@ -240,7 +226,12 @@
         "\n",
         "missing_rate = 0.75\n",
         "n_missing_samples = int(np.floor(data.shape[0] * missing_rate))\n",
-        "missing_samples = np.hstack((np.zeros(data.shape[0] - n_missing_samples, dtype=np.bool), np.ones(n_missing_samples, dtype=np.bool)))\n",
+        "missing_samples = np.hstack(\n",
+        "    (\n",
+        "        np.zeros(data.shape[0] - n_missing_samples, dtype=np.bool),\n",
+        "        np.ones(n_missing_samples, dtype=np.bool),\n",
+        "    )\n",
+        ")\n",
         "rng = np.random.RandomState(0)\n",
         "rng.shuffle(missing_samples)\n",
         "missing_features = rng.randint(0, data.shape[1], n_missing_samples)\n",
@@ -253,19 +244,21 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "if not os.path.isdir('data'):\n",
-        "    os.mkdir('data')\n",
-        "    \n",
+        "if not os.path.isdir(\"data\"):\n",
+        "    os.mkdir(\"data\")\n",
         "# Save the train data to a csv to be uploaded to the datastore\n",
         "pd.DataFrame(data).to_csv(\"data/train_data.csv\", index=False)\n",
         "\n",
         "ds = ws.get_default_datastore()\n",
-        "ds.upload(src_dir='./data', target_path='bankmarketing', overwrite=True, show_progress=True)\n",
+        "ds.upload(\n",
+        "    src_dir=\"./data\", target_path=\"bankmarketing\", overwrite=True, show_progress=True\n",
+        ")\n",
         "\n",
-        " \n",
         "\n",
         "# Upload the training data as a tabular dataset for access during training on remote compute\n",
-        "train_data = Dataset.Tabular.from_delimited_files(path=ds.path('bankmarketing/train_data.csv'))\n",
+        "train_data = Dataset.Tabular.from_delimited_files(\n",
+        "    path=ds.path(\"bankmarketing/train_data.csv\")\n",
+        ")\n",
         "label = \"y\""
       ]
     },
@@ -325,6 +318,7 @@
         "|**n_cross_validations**|Number of cross validation splits.|\n",
         "|**training_data**|Input dataset, containing both features and label column.|\n",
         "|**label_column_name**|The name of the label column.|\n",
+        "|**enable_code_generation**|Flag to enable generation of training code for each of the models that AutoML is creating.\n",
         "\n",
         "**_You can find more information about primary metrics_** [here](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-configure-auto-train#primary-metric)"
       ]
@@ -336,33 +330,37 @@
       "outputs": [],
       "source": [
         "automl_settings = {\n",
-        "    \"experiment_timeout_hours\" : 0.3,\n",
-        "    \"enable_early_stopping\" : True,\n",
+        "    \"experiment_timeout_hours\": 0.3,\n",
+        "    \"enable_early_stopping\": True,\n",
         "    \"iteration_timeout_minutes\": 5,\n",
         "    \"max_concurrent_iterations\": 4,\n",
         "    \"max_cores_per_iteration\": -1,\n",
-        "    #\"n_cross_validations\": 2,\n",
-        "    \"primary_metric\": 'AUC_weighted',\n",
-        "    \"featurization\": 'auto',\n",
+        "    # \"n_cross_validations\": 2,\n",
+        "    \"primary_metric\": \"AUC_weighted\",\n",
+        "    \"featurization\": \"auto\",\n",
         "    \"verbosity\": logging.INFO,\n",
+        "    \"enable_code_generation\": True,\n",
         "}\n",
         "\n",
-        "automl_config = AutoMLConfig(task = 'classification',\n",
-        "                             debug_log = 'automl_errors.log',\n",
-        "                             compute_target=compute_target,\n",
-        "                             experiment_exit_score = 0.9984,\n",
-        "                             blocked_models = ['KNN','LinearSVM'],\n",
-        "                             enable_onnx_compatible_models=True,\n",
-        "                             training_data = train_data,\n",
-        "                             label_column_name = label,\n",
-        "                             validation_data = validation_dataset,\n",
-        "                             **automl_settings\n",
-        "                            )"
+        "automl_config = AutoMLConfig(\n",
+        "    task=\"classification\",\n",
+        "    debug_log=\"automl_errors.log\",\n",
+        "    compute_target=compute_target,\n",
+        "    experiment_exit_score=0.9984,\n",
+        "    blocked_models=[\"KNN\", \"LinearSVM\"],\n",
+        "    enable_onnx_compatible_models=True,\n",
+        "    training_data=train_data,\n",
+        "    label_column_name=label,\n",
+        "    validation_data=validation_dataset,\n",
+        "    **automl_settings,\n",
+        ")"
       ]
     },
     {
       "cell_type": "markdown",
-      "metadata": {},
+      "metadata": {
+        "tags": []
+      },
       "source": [
         "Call the `submit` method on the experiment object and pass the run configuration. Execution of local runs is synchronous. Depending on the data and the number of iterations this can run for a while. Validation errors and current status will be shown when setting `show_output=True` and the execution will be synchronous."
       ]
@@ -370,15 +368,19 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "name": "experiment-submit"
+      },
       "outputs": [],
       "source": [
-        "remote_run = experiment.submit(automl_config, show_output = False)"
+        "remote_run = experiment.submit(automl_config, show_output=False)"
       ]
     },
     {
       "cell_type": "markdown",
-      "metadata": {},
+      "metadata": {
+        "tags": []
+      },
       "source": [
         "Run the following cell to access previous runs. Uncomment the cell below and update the run_id."
       ]
@@ -389,9 +391,9 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "#from azureml.train.automl.run import AutoMLRun\n",
-        "#remote_run = AutoMLRun(experiment=experiment, run_id='<run_ID_goes_here')\n",
-        "#remote_run"
+        "# from azureml.train.automl.run import AutoMLRun\n",
+        "# remote_run = AutoMLRun(experiment=experiment, run_id='<run_ID_goes_here')\n",
+        "# remote_run"
       ]
     },
     {
@@ -410,7 +412,8 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "best_run_customized, fitted_model_customized = remote_run.get_output()"
+        "# Retrieve the best Run object\n",
+        "best_run = remote_run.get_best_child()"
       ]
     },
     {
@@ -419,7 +422,7 @@
       "source": [
         "## Transparency\n",
         "\n",
-        "View updated featurization summary"
+        "View featurization summary for the best model - to study how different features were transformed. This is stored as a JSON file in the outputs directory for the run."
       ]
     },
     {
@@ -428,36 +431,16 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "custom_featurizer = fitted_model_customized.named_steps['datatransformer']\n",
-        "df = custom_featurizer.get_featurization_summary()\n",
-        "pd.DataFrame(data=df)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "Set `is_user_friendly=False` to get a more detailed summary for the transforms being applied."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "df = custom_featurizer.get_featurization_summary(is_user_friendly=False)\n",
-        "pd.DataFrame(data=df)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "df = custom_featurizer.get_stats_feature_type_summary()\n",
-        "pd.DataFrame(data=df)"
+        "# Download the featurization summary JSON file locally\n",
+        "best_run.download_file(\n",
+        "    \"outputs/featurization_summary.json\", \"featurization_summary.json\"\n",
+        ")\n",
+        "\n",
+        "# Render the JSON as a pandas DataFrame\n",
+        "with open(\"featurization_summary.json\", \"r\") as f:\n",
+        "    records = json.load(f)\n",
+        "\n",
+        "pd.DataFrame.from_records(records)"
       ]
     },
     {
@@ -470,11 +453,14 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "name": "run-details"
+      },
       "outputs": [],
       "source": [
         "from azureml.widgets import RunDetails\n",
-        "RunDetails(remote_run).show() "
+        "\n",
+        "RunDetails(remote_run).show()"
       ]
     },
     {
@@ -493,13 +479,16 @@
       "source": [
         "# Wait for the best model explanation run to complete\n",
         "from azureml.core.run import Run\n",
+        "\n",
         "model_explainability_run_id = remote_run.id + \"_\" + \"ModelExplain\"\n",
         "print(model_explainability_run_id)\n",
-        "model_explainability_run = Run(experiment=experiment, run_id=model_explainability_run_id)\n",
+        "model_explainability_run = Run(\n",
+        "    experiment=experiment, run_id=model_explainability_run_id\n",
+        ")\n",
         "model_explainability_run.wait_for_completion()\n",
         "\n",
         "# Get the best run object\n",
-        "best_run, fitted_model = remote_run.get_output()"
+        "best_run = remote_run.get_best_child()"
       ]
     },
     {
@@ -576,6 +565,7 @@
       "outputs": [],
       "source": [
         "from azureml.automl.runtime.onnx_convert import OnnxConverter\n",
+        "\n",
         "onnx_fl_path = \"./best_model.onnx\"\n",
         "OnnxConverter.save_onnx_model(onnx_mdl, onnx_fl_path)"
       ]
@@ -600,13 +590,17 @@
         "\n",
         "from azureml.automl.runtime.onnx_convert import OnnxInferenceHelper\n",
         "\n",
+        "\n",
         "def get_onnx_res(run):\n",
-        "    res_path = 'onnx_resource.json'\n",
-        "    run.download_file(name=constants.MODEL_RESOURCE_PATH_ONNX, output_file_path=res_path)\n",
+        "    res_path = \"onnx_resource.json\"\n",
+        "    run.download_file(\n",
+        "        name=constants.MODEL_RESOURCE_PATH_ONNX, output_file_path=res_path\n",
+        "    )\n",
         "    with open(res_path) as f:\n",
         "        result = json.load(f)\n",
         "    return result\n",
         "\n",
+        "\n",
         "if sys.version_info < OnnxConvertConstants.OnnxIncompatiblePythonVersion:\n",
         "    test_df = test_dataset.to_pandas_dataframe()\n",
         "    mdl_bytes = onnx_mdl.SerializeToString()\n",
@@ -618,7 +612,7 @@
         "    print(pred_onnx)\n",
         "    print(pred_prob_onnx)\n",
         "else:\n",
-        "    print('Please use Python version 3.6 or 3.7 to run the inference helper.')"
+        "    print(\"Please use Python version 3.6 or 3.7 to run the inference helper.\")"
       ]
     },
     {
@@ -629,7 +623,16 @@
         "\n",
         "### Retrieve the Best Model\n",
         "\n",
-        "Below we select the best pipeline from our iterations.  The `get_output` method returns the best run and the fitted model. Overloads on `get_output` allow you to retrieve the best run and fitted model for *any* logged metric or for a particular *iteration*."
+        "Below we select the best pipeline from our iterations.  The `get_best_child` method returns the Run object for the best model based on the default primary metric. There are additional flags that can be passed to the method if we want to retrieve the best Run based on any of the other supported metrics, or if we are just interested in the best run among the ONNX compatible runs. As always, you can execute `??remote_run.get_best_child` in a new cell to view the source or docs for the function."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "??remote_run.get_best_child"
       ]
     },
     {
@@ -649,7 +652,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "best_run, fitted_model = remote_run.get_output()"
+        "best_run = remote_run.get_best_child()"
       ]
     },
     {
@@ -658,11 +661,11 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "model_name = best_run.properties['model_name']\n",
+        "model_name = best_run.properties[\"model_name\"]\n",
         "\n",
-        "script_file_name = 'inference/score.py'\n",
+        "script_file_name = \"inference/score.py\"\n",
         "\n",
-        "best_run.download_file('outputs/scoring_file_v_1_0_0.py', 'inference/score.py')"
+        "best_run.download_file(\"outputs/scoring_file_v_1_0_0.py\", \"inference/score.py\")"
       ]
     },
     {
@@ -679,11 +682,15 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "description = 'AutoML Model trained on bank marketing data to predict if a client will subscribe to a term deposit'\n",
+        "description = \"AutoML Model trained on bank marketing data to predict if a client will subscribe to a term deposit\"\n",
         "tags = None\n",
-        "model = remote_run.register_model(model_name = model_name, description = description, tags = tags)\n",
+        "model = remote_run.register_model(\n",
+        "    model_name=model_name, description=description, tags=tags\n",
+        ")\n",
         "\n",
-        "print(remote_run.model_id) # This will be written to the script file later in the notebook."
+        "print(\n",
+        "    remote_run.model_id\n",
+        ")  # This will be written to the script file later in the notebook."
       ]
     },
     {
@@ -701,16 +708,20 @@
       "source": [
         "from azureml.core.model import InferenceConfig\n",
         "from azureml.core.webservice import AciWebservice\n",
+        "from azureml.core.webservice import Webservice\n",
         "from azureml.core.model import Model\n",
+        "from azureml.core.environment import Environment\n",
         "\n",
-        "inference_config = InferenceConfig(environment = best_run.get_environment(), entry_script=script_file_name)\n",
+        "inference_config = InferenceConfig(entry_script=script_file_name)\n",
         "\n",
-        "aciconfig = AciWebservice.deploy_configuration(cpu_cores = 2, \n",
-        "                                               memory_gb = 2, \n",
-        "                                               tags = {'area': \"bmData\", 'type': \"automl_classification\"}, \n",
-        "                                               description = 'sample service for Automl Classification')\n",
+        "aciconfig = AciWebservice.deploy_configuration(\n",
+        "    cpu_cores=2,\n",
+        "    memory_gb=2,\n",
+        "    tags={\"area\": \"bmData\", \"type\": \"automl_classification\"},\n",
+        "    description=\"sample service for Automl Classification\",\n",
+        ")\n",
         "\n",
-        "aci_service_name = 'automl-sample-bankmarketing-all'\n",
+        "aci_service_name = model_name.lower()\n",
         "print(aci_service_name)\n",
         "aci_service = Model.deploy(ws, aci_service_name, [model], inference_config, aciconfig)\n",
         "aci_service.wait_for_deployment(True)\n",
@@ -732,7 +743,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "#aci_service.get_logs()"
+        "# aci_service.get_logs()"
       ]
     },
     {
@@ -762,8 +773,8 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "X_test = test_dataset.drop_columns(columns=['y'])\n",
-        "y_test = test_dataset.keep_columns(columns=['y'], validate=True)\n",
+        "X_test = test_dataset.drop_columns(columns=[\"y\"])\n",
+        "y_test = test_dataset.keep_columns(columns=[\"y\"], validate=True)\n",
         "test_dataset.take(5).to_pandas_dataframe()"
       ]
     },
@@ -785,13 +796,13 @@
       "source": [
         "import requests\n",
         "\n",
-        "X_test_json = X_test.to_json(orient='records')\n",
-        "data = \"{\\\"data\\\": \" + X_test_json +\"}\"\n",
-        "headers = {'Content-Type': 'application/json'}\n",
+        "X_test_json = X_test.to_json(orient=\"records\")\n",
+        "data = '{\"data\": ' + X_test_json + \"}\"\n",
+        "headers = {\"Content-Type\": \"application/json\"}\n",
         "\n",
         "resp = requests.post(aci_service.scoring_uri, data, headers=headers)\n",
         "\n",
-        "y_pred = json.loads(json.loads(resp.text))['result']"
+        "y_pred = json.loads(json.loads(resp.text))[\"result\"]"
       ]
     },
     {
@@ -801,7 +812,7 @@
       "outputs": [],
       "source": [
         "actual = array(y_test)\n",
-        "actual = actual[:,0]\n",
+        "actual = actual[:, 0]\n",
         "print(len(y_pred), \" \", len(actual))"
       ]
     },
@@ -817,27 +828,35 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "scrolled": true
+      },
       "outputs": [],
       "source": [
         "%matplotlib notebook\n",
         "from sklearn.metrics import confusion_matrix\n",
         "import itertools\n",
         "\n",
-        "cf =confusion_matrix(actual,y_pred)\n",
-        "plt.imshow(cf,cmap=plt.cm.Blues,interpolation='nearest')\n",
+        "cf = confusion_matrix(actual, y_pred)\n",
+        "plt.imshow(cf, cmap=plt.cm.Blues, interpolation=\"nearest\")\n",
         "plt.colorbar()\n",
-        "plt.title('Confusion Matrix')\n",
-        "plt.xlabel('Predicted')\n",
-        "plt.ylabel('Actual')\n",
-        "class_labels = ['no','yes']\n",
+        "plt.title(\"Confusion Matrix\")\n",
+        "plt.xlabel(\"Predicted\")\n",
+        "plt.ylabel(\"Actual\")\n",
+        "class_labels = [\"no\", \"yes\"]\n",
         "tick_marks = np.arange(len(class_labels))\n",
-        "plt.xticks(tick_marks,class_labels)\n",
-        "plt.yticks([-0.5,0,1,1.5],['','no','yes',''])\n",
+        "plt.xticks(tick_marks, class_labels)\n",
+        "plt.yticks([-0.5, 0, 1, 1.5], [\"\", \"no\", \"yes\", \"\"])\n",
         "# plotting text value inside cells\n",
-        "thresh = cf.max() / 2.\n",
-        "for i,j in itertools.product(range(cf.shape[0]),range(cf.shape[1])):\n",
-        "    plt.text(j,i,format(cf[i,j],'d'),horizontalalignment='center',color='white' if cf[i,j] >thresh else 'black')\n",
+        "thresh = cf.max() / 2.0\n",
+        "for i, j in itertools.product(range(cf.shape[0]), range(cf.shape[1])):\n",
+        "    plt.text(\n",
+        "        j,\n",
+        "        i,\n",
+        "        format(cf[i, j], \"d\"),\n",
+        "        horizontalalignment=\"center\",\n",
+        "        color=\"white\" if cf[i, j] > thresh else \"black\",\n",
+        "    )\n",
         "plt.show()"
       ]
     },
@@ -859,6 +878,142 @@
         "aci_service.delete()"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Using the auto generated model training code for retraining on new data\n",
+        "\n",
+        "Because we enabled code generation when the original experiment was created, we now have access to the code that was used to generate any of the AutoML tried models. Below we'll be using the generated training script of the best model to retrain on a new dataset.\n",
+        "\n",
+        "For this demo, we'll begin by creating new retraining dataset by combining the Train & Validation datasets that were used in the original experiment."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "original_train_data = pd.read_csv(\n",
+        "    \"https://automlsamplenotebookdata.blob.core.windows.net/automl-sample-notebook-data/bankmarketing_train.csv\"\n",
+        ")\n",
+        "\n",
+        "valid_data = pd.read_csv(\n",
+        "    \"https://automlsamplenotebookdata.blob.core.windows.net/automl-sample-notebook-data/bankmarketing_validate.csv\"\n",
+        ")\n",
+        "\n",
+        "# we'll emulate an updated dataset for retraining by combining the Train & Validation datasets into a new one\n",
+        "retrain_pd = pd.concat([original_train_data, valid_data])\n",
+        "retrain_pd.to_csv(\"data/retrain_data.csv\", index=False)\n",
+        "ds.upload_files(\n",
+        "    files=[\"data/retrain_data.csv\"],\n",
+        "    target_path=\"bankmarketing/\",\n",
+        "    overwrite=True,\n",
+        "    show_progress=True,\n",
+        ")\n",
+        "retrain_dataset = Dataset.Tabular.from_delimited_files(\n",
+        "    path=ds.path(\"bankmarketing/retrain_data.csv\")\n",
+        ")\n",
+        "\n",
+        "# after creating and uploading the retraining dataset, let's register it with the workspace for reuse\n",
+        "retrain_dataset = retrain_dataset.register(\n",
+        "    workspace=ws,\n",
+        "    name=\"Bankmarketing_retrain\",\n",
+        "    description=\"Updated training dataset, includes validation data\",\n",
+        "    create_new_version=True,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Next, we'll download the generated script for the best run and use it for retraining. For more advanced scenarios, you can customize the training script as you need: change the featurization pipeline, change the learner algorithm or its hyperparameters, etc. \n",
+        "\n",
+        "For this exercise, we'll leave the script as it was generated."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# download the autogenerated training script into the generated_code folder\n",
+        "best_run.download_file(\n",
+        "    \"outputs/generated_code/script.py\", \"generated_code/training_script.py\"\n",
+        ")\n",
+        "\n",
+        "# view the contents of the autogenerated training script\n",
+        "! cat generated_code/training_script.py"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import uuid\n",
+        "from azureml.core import ScriptRunConfig\n",
+        "from azureml._restclient.models import RunTypeV2\n",
+        "from azureml._restclient.models.create_run_dto import CreateRunDto\n",
+        "from azureml._restclient.run_client import RunClient\n",
+        "\n",
+        "codegen_runid = str(uuid.uuid4())\n",
+        "client = RunClient(\n",
+        "    experiment.workspace.service_context,\n",
+        "    experiment.name,\n",
+        "    codegen_runid,\n",
+        "    experiment_id=experiment.id,\n",
+        ")\n",
+        "\n",
+        "# override the training_dataset_id to point to our new retraining dataset we just registered above\n",
+        "dataset_arguments = [\"--training_dataset_id\", retrain_dataset.id]\n",
+        "\n",
+        "# create the retraining run as a child of the AutoML generated training run\n",
+        "create_run_dto = CreateRunDto(\n",
+        "    run_id=codegen_runid,\n",
+        "    parent_run_id=best_run.id,\n",
+        "    description=\"AutoML Codegen Script Run using an updated training dataset\",\n",
+        "    target=cpu_cluster_name,\n",
+        "    run_type_v2=RunTypeV2(orchestrator=\"Execution\", traits=[\"automl-codegen\"]),\n",
+        ")\n",
+        "\n",
+        "# the script for retraining run is pointing to the AutoML generated script\n",
+        "src = ScriptRunConfig(\n",
+        "    source_directory=\"generated_code\",\n",
+        "    script=\"training_script.py\",\n",
+        "    arguments=dataset_arguments,\n",
+        "    compute_target=cpu_cluster_name,\n",
+        "    environment=best_run.get_environment(),\n",
+        ")\n",
+        "run_dto = client.create_run(run_id=codegen_runid, create_run_dto=create_run_dto)\n",
+        "\n",
+        "# submit the experiment\n",
+        "retraining_run = experiment.submit(config=src, run_id=codegen_runid)\n",
+        "retraining_run"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "After the run completes, we can get download/test/deploy to the model it has built."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "retraining_run.wait_for_completion()\n",
+        "\n",
+        "retraining_run.download_file(\"outputs/model.pkl\", \"generated_code/model.pkl\")"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {},
@@ -901,6 +1056,9 @@
     ],
     "friendly_name": "Automated ML run with basic edition features.",
     "index_order": 5,
+    "kernel_info": {
+      "name": "python3-azureml"
+    },
     "kernelspec": {
       "display_name": "Python 3.6",
       "language": "python",
@@ -916,7 +1074,10 @@
       "name": "python",
       "nbconvert_exporter": "python",
       "pygments_lexer": "ipython3",
-      "version": "3.6.7"
+      "version": "3.6.9"
+    },
+    "nteract": {
+      "version": "nteract-front-end@1.0.0"
     },
     "tags": [
       "featurization",
@@ -927,5 +1088,5 @@
     "task": "Classification"
   },
   "nbformat": 4,
-  "nbformat_minor": 2
+  "nbformat_minor": 1
 }

how-to-use-azureml/automated-machine-learning/classification-credit-card-fraud/auto-ml-classification-credit-card-fraud.ipynb

@@ -1,21 +1,5 @@
 {
   "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
-        "\n",
-        "Licensed under the MIT License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/automated-machine-learning/classification-credit-card-fraud/auto-ml-classification-credit-card-fraud.png)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {},
@@ -87,16 +71,6 @@
         "This sample notebook may use features that are not available in previous versions of the Azure ML SDK."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "print(\"This notebook was created using version 1.36.0 of the Azure ML SDK\")\n",
-        "print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -106,18 +80,19 @@
         "ws = Workspace.from_config()\n",
         "\n",
         "# choose a name for experiment\n",
-        "experiment_name = 'automl-classification-ccard-remote'\n",
+        "experiment_name = \"automl-classification-ccard-remote\"\n",
         "\n",
-        "experiment=Experiment(ws, experiment_name)\n",
+        "experiment = Experiment(ws, experiment_name)\n",
         "\n",
         "output = {}\n",
-        "output['Subscription ID'] = ws.subscription_id\n",
-        "output['Workspace'] = ws.name\n",
-        "output['Resource Group'] = ws.resource_group\n",
-        "output['Location'] = ws.location\n",
-        "output['Experiment Name'] = experiment.name\n",
-        "pd.set_option('display.max_colwidth', -1)\n",
-        "outputDf = pd.DataFrame(data = output, index = [''])\n",
+        "output[\"Subscription ID\"] = ws.subscription_id\n",
+        "output[\"Workspace\"] = ws.name\n",
+        "output[\"Resource Group\"] = ws.resource_group\n",
+        "output[\"Location\"] = ws.location\n",
+        "output[\"Experiment Name\"] = experiment.name\n",
+        "output[\"SDK Version\"] = azureml.core.VERSION\n",
+        "pd.set_option(\"display.max_colwidth\", None)\n",
+        "outputDf = pd.DataFrame(data=output, index=[\"\"])\n",
         "outputDf.T"
       ]
     },
@@ -150,12 +125,12 @@
         "# Verify that cluster does not exist already\n",
         "try:\n",
         "    compute_target = ComputeTarget(workspace=ws, name=cpu_cluster_name)\n",
-        "    print('Found existing cluster, use it.')\n",
+        "    print(\"Found existing cluster, use it.\")\n",
         "except ComputeTargetException:\n",
-        "    compute_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_DS12_V2',\n",
-        "                                                           max_nodes=6)\n",
+        "    compute_config = AmlCompute.provisioning_configuration(\n",
+        "        vm_size=\"STANDARD_DS12_V2\", max_nodes=6\n",
+        "    )\n",
         "    compute_target = ComputeTarget.create(ws, cpu_cluster_name, compute_config)\n",
-        "\n",
         "compute_target.wait_for_completion(show_output=True)"
       ]
     },
@@ -178,13 +153,15 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "name": "load-data"
+      },
       "outputs": [],
       "source": [
         "data = \"https://automlsamplenotebookdata.blob.core.windows.net/automl-sample-notebook-data/creditcard.csv\"\n",
         "dataset = Dataset.Tabular.from_delimited_files(data)\n",
         "training_data, validation_data = dataset.random_split(percentage=0.8, seed=223)\n",
-        "label_column_name = 'Class'"
+        "label_column_name = \"Class\""
       ]
     },
     {
@@ -210,25 +187,28 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "name": "automl-config"
+      },
       "outputs": [],
       "source": [
         "automl_settings = {\n",
         "    \"n_cross_validations\": 3,\n",
-        "    \"primary_metric\": 'AUC_weighted',\n",
+        "    \"primary_metric\": \"average_precision_score_weighted\",\n",
         "    \"enable_early_stopping\": True,\n",
-        "    \"max_concurrent_iterations\": 2, # This is a limit for testing purpose, please increase it as per cluster size\n",
-        "    \"experiment_timeout_hours\": 0.25, # This is a time limit for testing purposes, remove it for real use cases, this will drastically limit ablity to find the best model possible\n",
+        "    \"max_concurrent_iterations\": 2,  # This is a limit for testing purpose, please increase it as per cluster size\n",
+        "    \"experiment_timeout_hours\": 0.25,  # This is a time limit for testing purposes, remove it for real use cases, this will drastically limit ablity to find the best model possible\n",
         "    \"verbosity\": logging.INFO,\n",
         "}\n",
         "\n",
-        "automl_config = AutoMLConfig(task = 'classification',\n",
-        "                             debug_log = 'automl_errors.log',\n",
-        "                             compute_target = compute_target,\n",
-        "                             training_data = training_data,\n",
-        "                             label_column_name = label_column_name,\n",
-        "                             **automl_settings\n",
-        "                            )"
+        "automl_config = AutoMLConfig(\n",
+        "    task=\"classification\",\n",
+        "    debug_log=\"automl_errors.log\",\n",
+        "    compute_target=compute_target,\n",
+        "    training_data=training_data,\n",
+        "    label_column_name=label_column_name,\n",
+        "    **automl_settings,\n",
+        ")"
       ]
     },
     {
@@ -244,7 +224,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "remote_run = experiment.submit(automl_config, show_output = False)"
+        "remote_run = experiment.submit(automl_config, show_output=False)"
       ]
     },
     {
@@ -254,8 +234,8 @@
       "outputs": [],
       "source": [
         "# If you need to retrieve a run that already started, use the following code\n",
-        "#from azureml.train.automl.run import AutoMLRun\n",
-        "#remote_run = AutoMLRun(experiment = experiment, run_id = '<replace with your run id>')"
+        "# from azureml.train.automl.run import AutoMLRun\n",
+        "# remote_run = AutoMLRun(experiment = experiment, run_id = '<replace with your run id>')"
       ]
     },
     {
@@ -287,6 +267,7 @@
       "outputs": [],
       "source": [
         "from azureml.widgets import RunDetails\n",
+        "\n",
         "RunDetails(remote_run).show()"
       ]
     },
@@ -353,8 +334,12 @@
       "outputs": [],
       "source": [
         "# convert the test data to dataframe\n",
-        "X_test_df = validation_data.drop_columns(columns=[label_column_name]).to_pandas_dataframe()\n",
-        "y_test_df = validation_data.keep_columns(columns=[label_column_name], validate=True).to_pandas_dataframe()"
+        "X_test_df = validation_data.drop_columns(\n",
+        "    columns=[label_column_name]\n",
+        ").to_pandas_dataframe()\n",
+        "y_test_df = validation_data.keep_columns(\n",
+        "    columns=[label_column_name], validate=True\n",
+        ").to_pandas_dataframe()"
       ]
     },
     {
@@ -388,20 +373,26 @@
         "import numpy as np\n",
         "import itertools\n",
         "\n",
-        "cf =confusion_matrix(y_test_df.values,y_pred)\n",
-        "plt.imshow(cf,cmap=plt.cm.Blues,interpolation='nearest')\n",
+        "cf = confusion_matrix(y_test_df.values, y_pred)\n",
+        "plt.imshow(cf, cmap=plt.cm.Blues, interpolation=\"nearest\")\n",
         "plt.colorbar()\n",
-        "plt.title('Confusion Matrix')\n",
-        "plt.xlabel('Predicted')\n",
-        "plt.ylabel('Actual')\n",
-        "class_labels = ['False','True']\n",
+        "plt.title(\"Confusion Matrix\")\n",
+        "plt.xlabel(\"Predicted\")\n",
+        "plt.ylabel(\"Actual\")\n",
+        "class_labels = [\"False\", \"True\"]\n",
         "tick_marks = np.arange(len(class_labels))\n",
-        "plt.xticks(tick_marks,class_labels)\n",
-        "plt.yticks([-0.5,0,1,1.5],['','False','True',''])\n",
+        "plt.xticks(tick_marks, class_labels)\n",
+        "plt.yticks([-0.5, 0, 1, 1.5], [\"\", \"False\", \"True\", \"\"])\n",
         "# plotting text value inside cells\n",
-        "thresh = cf.max() / 2.\n",
-        "for i,j in itertools.product(range(cf.shape[0]),range(cf.shape[1])):\n",
-        "    plt.text(j,i,format(cf[i,j],'d'),horizontalalignment='center',color='white' if cf[i,j] >thresh else 'black')\n",
+        "thresh = cf.max() / 2.0\n",
+        "for i, j in itertools.product(range(cf.shape[0]), range(cf.shape[1])):\n",
+        "    plt.text(\n",
+        "        j,\n",
+        "        i,\n",
+        "        format(cf[i, j], \"d\"),\n",
+        "        horizontalalignment=\"center\",\n",
+        "        color=\"white\" if cf[i, j] > thresh else \"black\",\n",
+        "    )\n",
         "plt.show()"
       ]
     },

how-to-use-azureml/automated-machine-learning/classification-text-dnn/auto-ml-classification-text-dnn.ipynb

@@ -1,21 +1,5 @@
 {
   "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
-        "\n",
-        "Licensed under the MIT License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/automated-machine-learning/classification-text-dnn/auto-ml-classification-text-dnn.png)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {},
@@ -63,6 +47,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
+        "import json\n",
         "import logging\n",
         "import os\n",
         "import shutil\n",
@@ -77,7 +62,7 @@
         "from azureml.core.compute import ComputeTarget\n",
         "from azureml.core.run import Run\n",
         "from azureml.widgets import RunDetails\n",
-        "from azureml.core.model import Model \n",
+        "from azureml.core.model import Model\n",
         "from helper import run_inference, get_result_df\n",
         "from azureml.train.automl import AutoMLConfig\n",
         "from sklearn.datasets import fetch_20newsgroups"
@@ -90,16 +75,6 @@
         "This sample notebook may use features that are not available in previous versions of the Azure ML SDK."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "print(\"This notebook was created using version 1.36.0 of the Azure ML SDK\")\n",
-        "print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {},
@@ -116,18 +91,19 @@
         "ws = Workspace.from_config()\n",
         "\n",
         "# Choose an experiment name.\n",
-        "experiment_name = 'automl-classification-text-dnn'\n",
+        "experiment_name = \"automl-classification-text-dnn\"\n",
         "\n",
         "experiment = Experiment(ws, experiment_name)\n",
         "\n",
         "output = {}\n",
-        "output['Subscription ID'] = ws.subscription_id\n",
-        "output['Workspace Name'] = ws.name\n",
-        "output['Resource Group'] = ws.resource_group\n",
-        "output['Location'] = ws.location\n",
-        "output['Experiment Name'] = experiment.name\n",
-        "pd.set_option('display.max_colwidth', -1)\n",
-        "outputDf = pd.DataFrame(data = output, index = [''])\n",
+        "output[\"Subscription ID\"] = ws.subscription_id\n",
+        "output[\"Workspace Name\"] = ws.name\n",
+        "output[\"Resource Group\"] = ws.resource_group\n",
+        "output[\"Location\"] = ws.location\n",
+        "output[\"Experiment Name\"] = experiment.name\n",
+        "output[\"SDK Version\"] = azureml.core.VERSION\n",
+        "pd.set_option(\"display.max_colwidth\", None)\n",
+        "outputDf = pd.DataFrame(data=output, index=[\"\"])\n",
         "outputDf.T"
       ]
     },
@@ -160,13 +136,16 @@
         "# Verify that cluster does not exist already\n",
         "try:\n",
         "    compute_target = ComputeTarget(workspace=ws, name=amlcompute_cluster_name)\n",
-        "    print('Found existing cluster, use it.')\n",
+        "    print(\"Found existing cluster, use it.\")\n",
         "except ComputeTargetException:\n",
-        "    compute_config = AmlCompute.provisioning_configuration(vm_size = \"STANDARD_NC6\", # CPU for BiLSTM, such as \"STANDARD_DS12_V2\" \n",
-        "                                                           # To use BERT (this is recommended for best performance), select a GPU such as \"STANDARD_NC6\" \n",
-        "                                                           # or similar GPU option\n",
-        "                                                           # available in your workspace\n",
-        "                                                           max_nodes = num_nodes)\n",
+        "    compute_config = AmlCompute.provisioning_configuration(\n",
+        "        vm_size=\"STANDARD_NC6\",  # CPU for BiLSTM, such as \"STANDARD_D2_V2\"\n",
+        "        # To use BERT (this is recommended for best performance), select a GPU such as \"STANDARD_NC6\"\n",
+        "        # or similar GPU option\n",
+        "        # available in your workspace\n",
+        "        idle_seconds_before_scaledown=60,\n",
+        "        max_nodes=num_nodes,\n",
+        "    )\n",
         "    compute_target = ComputeTarget.create(ws, amlcompute_cluster_name, compute_config)\n",
         "\n",
         "compute_target.wait_for_completion(show_output=True)"
@@ -186,41 +165,55 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "data_dir = \"text-dnn-data\" # Local directory to store data\n",
-        "blobstore_datadir = data_dir # Blob store directory to store data in\n",
-        "target_column_name = 'y'\n",
-        "feature_column_name = 'X'\n",
+        "data_dir = \"text-dnn-data\"  # Local directory to store data\n",
+        "blobstore_datadir = data_dir  # Blob store directory to store data in\n",
+        "target_column_name = \"y\"\n",
+        "feature_column_name = \"X\"\n",
+        "\n",
         "\n",
         "def get_20newsgroups_data():\n",
-        "    '''Fetches 20 Newsgroups data from scikit-learn\n",
-        "       Returns them in form of pandas dataframes\n",
-        "    '''\n",
-        "    remove = ('headers', 'footers', 'quotes')\n",
+        "    \"\"\"Fetches 20 Newsgroups data from scikit-learn\n",
+        "    Returns them in form of pandas dataframes\n",
+        "    \"\"\"\n",
+        "    remove = (\"headers\", \"footers\", \"quotes\")\n",
         "    categories = [\n",
-        "        'rec.sport.baseball',\n",
-        "        'rec.sport.hockey',\n",
-        "        'comp.graphics',\n",
-        "        'sci.space',\n",
-        "        ]\n",
+        "        \"rec.sport.baseball\",\n",
+        "        \"rec.sport.hockey\",\n",
+        "        \"comp.graphics\",\n",
+        "        \"sci.space\",\n",
+        "    ]\n",
         "\n",
-        "    data = fetch_20newsgroups(subset = 'train', categories = categories,\n",
-        "                                    shuffle = True, random_state = 42,\n",
-        "                                    remove = remove)\n",
-        "    data = pd.DataFrame({feature_column_name: data.data, target_column_name: data.target})\n",
+        "    data = fetch_20newsgroups(\n",
+        "        subset=\"train\",\n",
+        "        categories=categories,\n",
+        "        shuffle=True,\n",
+        "        random_state=42,\n",
+        "        remove=remove,\n",
+        "    )\n",
+        "    data = pd.DataFrame(\n",
+        "        {feature_column_name: data.data, target_column_name: data.target}\n",
+        "    )\n",
         "\n",
         "    data_train = data[:200]\n",
-        "    data_test = data[200:300]    \n",
+        "    data_test = data[200:300]\n",
         "\n",
-        "    data_train = remove_blanks_20news(data_train, feature_column_name, target_column_name)\n",
+        "    data_train = remove_blanks_20news(\n",
+        "        data_train, feature_column_name, target_column_name\n",
+        "    )\n",
         "    data_test = remove_blanks_20news(data_test, feature_column_name, target_column_name)\n",
-        "    \n",
+        "\n",
         "    return data_train, data_test\n",
-        "    \n",
+        "\n",
+        "\n",
         "def remove_blanks_20news(data, feature_column_name, target_column_name):\n",
-        "    \n",
-        "    data[feature_column_name] = data[feature_column_name].replace(r'\\n', ' ', regex=True).apply(lambda x: x.strip())\n",
-        "    data = data[data[feature_column_name] != '']\n",
-        "    \n",
+        "\n",
+        "    data[feature_column_name] = (\n",
+        "        data[feature_column_name]\n",
+        "        .replace(r\"\\n\", \" \", regex=True)\n",
+        "        .apply(lambda x: x.strip())\n",
+        "    )\n",
+        "    data = data[data[feature_column_name] != \"\"]\n",
+        "\n",
         "    return data"
       ]
     },
@@ -241,16 +234,15 @@
         "\n",
         "if not os.path.isdir(data_dir):\n",
         "    os.mkdir(data_dir)\n",
-        "    \n",
-        "train_data_fname = data_dir + '/train_data.csv'\n",
-        "test_data_fname = data_dir + '/test_data.csv'\n",
+        "\n",
+        "train_data_fname = data_dir + \"/train_data.csv\"\n",
+        "test_data_fname = data_dir + \"/test_data.csv\"\n",
         "\n",
         "data_train.to_csv(train_data_fname, index=False)\n",
         "data_test.to_csv(test_data_fname, index=False)\n",
         "\n",
         "datastore = ws.get_default_datastore()\n",
-        "datastore.upload(src_dir=data_dir, target_path=blobstore_datadir,\n",
-        "                    overwrite=True)"
+        "datastore.upload(src_dir=data_dir, target_path=blobstore_datadir, overwrite=True)"
       ]
     },
     {
@@ -259,7 +251,9 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "train_dataset = Dataset.Tabular.from_delimited_files(path = [(datastore, blobstore_datadir + '/train_data.csv')])"
+        "train_dataset = Dataset.Tabular.from_delimited_files(\n",
+        "    path=[(datastore, blobstore_datadir + \"/train_data.csv\")]\n",
+        ")"
       ]
     },
     {
@@ -284,8 +278,8 @@
       "source": [
         "automl_settings = {\n",
         "    \"experiment_timeout_minutes\": 30,\n",
-        "    \"primary_metric\": 'AUC_weighted',\n",
-        "    \"max_concurrent_iterations\": num_nodes, \n",
+        "    \"primary_metric\": \"accuracy\",\n",
+        "    \"max_concurrent_iterations\": num_nodes,\n",
         "    \"max_cores_per_iteration\": -1,\n",
         "    \"enable_dnn\": True,\n",
         "    \"enable_early_stopping\": True,\n",
@@ -295,14 +289,15 @@
         "    \"enable_stack_ensemble\": False,\n",
         "}\n",
         "\n",
-        "automl_config = AutoMLConfig(task = 'classification',\n",
-        "                             debug_log = 'automl_errors.log',\n",
-        "                             compute_target=compute_target,\n",
-        "                             training_data=train_dataset,\n",
-        "                             label_column_name=target_column_name,\n",
-        "                             blocked_models = ['LightGBM', 'XGBoostClassifier'],\n",
-        "                             **automl_settings\n",
-        "                            )"
+        "automl_config = AutoMLConfig(\n",
+        "    task=\"classification\",\n",
+        "    debug_log=\"automl_errors.log\",\n",
+        "    compute_target=compute_target,\n",
+        "    training_data=train_dataset,\n",
+        "    label_column_name=target_column_name,\n",
+        "    blocked_models=[\"LightGBM\", \"XGBoostClassifier\"],\n",
+        "    **automl_settings,\n",
+        ")"
       ]
     },
     {
@@ -340,8 +335,8 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "You can test the model locally to get a feel of the input/output. When the model contains BERT, this step will require pytorch and pytorch-transformers installed in your local environment. The exact versions of these packages can be found in the **automl_env.yml** file located in the local copy of your MachineLearningNotebooks folder here:\n",
-        "MachineLearningNotebooks/how-to-use-azureml/automated-machine-learning/automl_env.yml"
+        "For local inferencing, you can load the model locally via. the method `remote_run.get_output()`. For more information on the arguments expected by this method, you can run `remote_run.get_output??`.\n",
+        "Note that when the model contains BERT, this step will require pytorch and pytorch-transformers installed in your local environment. The exact versions of these packages can be found in the **automl_env.yml** file located in the local copy of your azureml-examples folder here: \"azureml-examples/python-sdk/tutorials/automl-with-azureml\""
       ]
     },
     {
@@ -350,7 +345,8 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "best_run, fitted_model = automl_run.get_output()"
+        "# Retrieve the best Run object\n",
+        "best_run = automl_run.get_best_child()"
       ]
     },
     {
@@ -366,10 +362,17 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "text_transformations_used = []\n",
-        "for column_group in fitted_model.named_steps['datatransformer'].get_featurization_summary():\n",
-        "    text_transformations_used.extend(column_group['Transformations'])\n",
-        "text_transformations_used"
+        "# Download the featurization summary JSON file locally\n",
+        "best_run.download_file(\n",
+        "    \"outputs/featurization_summary.json\", \"featurization_summary.json\"\n",
+        ")\n",
+        "\n",
+        "# Render the JSON as a pandas DataFrame\n",
+        "with open(\"featurization_summary.json\", \"r\") as f:\n",
+        "    records = json.load(f)\n",
+        "\n",
+        "featurization_summary = pd.DataFrame.from_records(records)\n",
+        "featurization_summary[\"Transformations\"].tolist()"
       ]
     },
     {
@@ -394,7 +397,7 @@
       "outputs": [],
       "source": [
         "summary_df = get_result_df(automl_run)\n",
-        "best_dnn_run_id = summary_df['run_id'].iloc[0]\n",
+        "best_dnn_run_id = summary_df[\"run_id\"].iloc[0]\n",
         "best_dnn_run = Run(experiment, best_dnn_run_id)"
       ]
     },
@@ -404,11 +407,11 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "model_dir = 'Model' # Local folder where the model will be stored temporarily\n",
+        "model_dir = \"Model\"  # Local folder where the model will be stored temporarily\n",
         "if not os.path.isdir(model_dir):\n",
         "    os.mkdir(model_dir)\n",
-        "    \n",
-        "best_dnn_run.download_file('outputs/model.pkl', model_dir + '/model.pkl')"
+        "\n",
+        "best_dnn_run.download_file(\"outputs/model.pkl\", model_dir + \"/model.pkl\")"
       ]
     },
     {
@@ -425,11 +428,10 @@
       "outputs": [],
       "source": [
         "# Register the model\n",
-        "model_name = 'textDNN-20News'\n",
-        "model = Model.register(model_path = model_dir + '/model.pkl',\n",
-        "                       model_name = model_name,\n",
-        "                       tags=None,\n",
-        "                       workspace=ws)"
+        "model_name = \"textDNN-20News\"\n",
+        "model = Model.register(\n",
+        "    model_path=model_dir + \"/model.pkl\", model_name=model_name, tags=None, workspace=ws\n",
+        ")"
       ]
     },
     {
@@ -454,7 +456,9 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "test_dataset = Dataset.Tabular.from_delimited_files(path = [(datastore, blobstore_datadir + '/test_data.csv')])\n",
+        "test_dataset = Dataset.Tabular.from_delimited_files(\n",
+        "    path=[(datastore, blobstore_datadir + \"/test_data.csv\")]\n",
+        ")\n",
         "\n",
         "# preview the first 3 rows of the dataset\n",
         "test_dataset.take(3).to_pandas_dataframe()"
@@ -475,9 +479,9 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "script_folder = os.path.join(os.getcwd(), 'inference')\n",
+        "script_folder = os.path.join(os.getcwd(), \"inference\")\n",
         "os.makedirs(script_folder, exist_ok=True)\n",
-        "shutil.copy('infer.py', script_folder)"
+        "shutil.copy(\"infer.py\", script_folder)"
       ]
     },
     {
@@ -486,8 +490,15 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "test_run = run_inference(test_experiment, compute_target, script_folder, best_dnn_run,\n",
-        "                         test_dataset, target_column_name, model_name)"
+        "test_run = run_inference(\n",
+        "    test_experiment,\n",
+        "    compute_target,\n",
+        "    script_folder,\n",
+        "    best_dnn_run,\n",
+        "    test_dataset,\n",
+        "    target_column_name,\n",
+        "    model_name,\n",
+        ")"
       ]
     },
     {

how-to-use-azureml/automated-machine-learning/classification-text-dnn/helper.py

@@ -4,52 +4,65 @@ from azureml.train.estimator import Estimator
 from azureml.core.run import Run
 
 
-def run_inference(test_experiment, compute_target, script_folder, train_run,
-                  test_dataset, target_column_name, model_name):
+def run_inference(
+    test_experiment,
+    compute_target,
+    script_folder,
+    train_run,
+    test_dataset,
+    target_column_name,
+    model_name,
+):
 
     inference_env = train_run.get_environment()
 
-    est = Estimator(source_directory=script_folder,
-                    entry_script='infer.py',
-                    script_params={
-                        '--target_column_name': target_column_name,
-                        '--model_name': model_name
-                    },
-                    inputs=[
-                        test_dataset.as_named_input('test_data')
-                    ],
-                    compute_target=compute_target,
-                    environment_definition=inference_env)
+    est = Estimator(
+        source_directory=script_folder,
+        entry_script="infer.py",
+        script_params={
+            "--target_column_name": target_column_name,
+            "--model_name": model_name,
+        },
+        inputs=[test_dataset.as_named_input("test_data")],
+        compute_target=compute_target,
+        environment_definition=inference_env,
+    )
 
     run = test_experiment.submit(
-        est, tags={
-            'training_run_id': train_run.id,
-            'run_algorithm': train_run.properties['run_algorithm'],
-            'valid_score': train_run.properties['score'],
-            'primary_metric': train_run.properties['primary_metric']
-        })
+        est,
+        tags={
+            "training_run_id": train_run.id,
+            "run_algorithm": train_run.properties["run_algorithm"],
+            "valid_score": train_run.properties["score"],
+            "primary_metric": train_run.properties["primary_metric"],
+        },
+    )
 
-    run.log("run_algorithm", run.tags['run_algorithm'])
+    run.log("run_algorithm", run.tags["run_algorithm"])
     return run
 
 
 def get_result_df(remote_run):
 
     children = list(remote_run.get_children(recursive=True))
-    summary_df = pd.DataFrame(index=['run_id', 'run_algorithm',
-                                     'primary_metric', 'Score'])
+    summary_df = pd.DataFrame(
+        index=["run_id", "run_algorithm", "primary_metric", "Score"]
+    )
     goal_minimize = False
     for run in children:
-        if('run_algorithm' in run.properties and 'score' in run.properties):
-            summary_df[run.id] = [run.id, run.properties['run_algorithm'],
-                                  run.properties['primary_metric'],
-                                  float(run.properties['score'])]
-            if('goal' in run.properties):
-                goal_minimize = run.properties['goal'].split('_')[-1] == 'min'
+        if "run_algorithm" in run.properties and "score" in run.properties:
+            summary_df[run.id] = [
+                run.id,
+                run.properties["run_algorithm"],
+                run.properties["primary_metric"],
+                float(run.properties["score"]),
+            ]
+            if "goal" in run.properties:
+                goal_minimize = run.properties["goal"].split("_")[-1] == "min"
 
     summary_df = summary_df.T.sort_values(
-        'Score',
-        ascending=goal_minimize).drop_duplicates(['run_algorithm'])
-    summary_df = summary_df.set_index('run_algorithm')
+        "Score", ascending=goal_minimize
+    ).drop_duplicates(["run_algorithm"])
+    summary_df = summary_df.set_index("run_algorithm")
 
     return summary_df

how-to-use-azureml/automated-machine-learning/classification-text-dnn/infer.py

@@ -12,19 +12,22 @@ from azureml.core.model import Model
 
 parser = argparse.ArgumentParser()
 parser.add_argument(
-    '--target_column_name', type=str, dest='target_column_name',
-    help='Target Column Name')
+    "--target_column_name",
+    type=str,
+    dest="target_column_name",
+    help="Target Column Name",
+)
 parser.add_argument(
-    '--model_name', type=str, dest='model_name',
-    help='Name of registered model')
+    "--model_name", type=str, dest="model_name", help="Name of registered model"
+)
 
 args = parser.parse_args()
 target_column_name = args.target_column_name
 model_name = args.model_name
 
-print('args passed are: ')
-print('Target column name: ', target_column_name)
-print('Name of registered model: ', model_name)
+print("args passed are: ")
+print("Target column name: ", target_column_name)
+print("Name of registered model: ", model_name)
 
 model_path = Model.get_model_path(model_name)
 # deserialize the model file back into a sklearn model
@@ -32,13 +35,16 @@ model = joblib.load(model_path)
 
 run = Run.get_context()
 # get input dataset by name
-test_dataset = run.input_datasets['test_data']
+test_dataset = run.input_datasets["test_data"]
 
-X_test_df = test_dataset.drop_columns(columns=[target_column_name]) \
-                        .to_pandas_dataframe()
-y_test_df = test_dataset.with_timestamp_columns(None) \
-                        .keep_columns(columns=[target_column_name]) \
-                        .to_pandas_dataframe()
+X_test_df = test_dataset.drop_columns(
+    columns=[target_column_name]
+).to_pandas_dataframe()
+y_test_df = (
+    test_dataset.with_timestamp_columns(None)
+    .keep_columns(columns=[target_column_name])
+    .to_pandas_dataframe()
+)
 
 predicted = model.predict_proba(X_test_df)
 
@@ -47,11 +53,13 @@ if isinstance(predicted, pd.DataFrame):
 
 # Use the AutoML scoring module
 train_labels = model.classes_
-class_labels = np.unique(np.concatenate((y_test_df.values, np.reshape(train_labels, (-1, 1)))))
+class_labels = np.unique(
+    np.concatenate((y_test_df.values, np.reshape(train_labels, (-1, 1))))
+)
 classification_metrics = list(constants.CLASSIFICATION_SCALAR_SET)
-scores = scoring.score_classification(y_test_df.values, predicted,
-                                      classification_metrics,
-                                      class_labels, train_labels)
+scores = scoring.score_classification(
+    y_test_df.values, predicted, classification_metrics, class_labels, train_labels
+)
 
 print("scores:")
 print(scores)

how-to-use-azureml/automated-machine-learning/continuous-retraining/auto-ml-continuous-retraining.ipynb

@@ -1,20 +1,5 @@
 {
   "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "Copyright (c) Microsoft Corporation. All rights reserved.  \n",
-        "Licensed under the MIT License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/automated-machine-learning/continous-retraining/auto-ml-continuous-retraining.png)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {},
@@ -75,16 +60,6 @@
         "This sample notebook may use features that are not available in previous versions of the Azure ML SDK."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "print(\"This notebook was created using version 1.36.0 of the Azure ML SDK\")\n",
-        "print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {},
@@ -118,17 +93,18 @@
         "dstor = ws.get_default_datastore()\n",
         "\n",
         "# Choose a name for the run history container in the workspace.\n",
-        "experiment_name = 'retrain-noaaweather'\n",
+        "experiment_name = \"retrain-noaaweather\"\n",
         "experiment = Experiment(ws, experiment_name)\n",
         "\n",
         "output = {}\n",
-        "output['Subscription ID'] = ws.subscription_id\n",
-        "output['Workspace'] = ws.name\n",
-        "output['Resource Group'] = ws.resource_group\n",
-        "output['Location'] = ws.location\n",
-        "output['Run History Name'] = experiment_name\n",
-        "pd.set_option('display.max_colwidth', -1)\n",
-        "outputDf = pd.DataFrame(data = output, index = [''])\n",
+        "output[\"Subscription ID\"] = ws.subscription_id\n",
+        "output[\"Workspace\"] = ws.name\n",
+        "output[\"Resource Group\"] = ws.resource_group\n",
+        "output[\"Location\"] = ws.location\n",
+        "output[\"Run History Name\"] = experiment_name\n",
+        "output[\"SDK Version\"] = azureml.core.VERSION\n",
+        "pd.set_option(\"display.max_colwidth\", None)\n",
+        "outputDf = pd.DataFrame(data=output, index=[\"\"])\n",
         "outputDf.T"
       ]
     },
@@ -164,12 +140,12 @@
         "# Verify that cluster does not exist already\n",
         "try:\n",
         "    compute_target = ComputeTarget(workspace=ws, name=amlcompute_cluster_name)\n",
-        "    print('Found existing cluster, use it.')\n",
+        "    print(\"Found existing cluster, use it.\")\n",
         "except ComputeTargetException:\n",
-        "    compute_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_DS12_V2',\n",
-        "                                                           max_nodes=4)\n",
+        "    compute_config = AmlCompute.provisioning_configuration(\n",
+        "        vm_size=\"STANDARD_DS12_V2\", max_nodes=4\n",
+        "    )\n",
         "    compute_target = ComputeTarget.create(ws, amlcompute_cluster_name, compute_config)\n",
-        "\n",
         "compute_target.wait_for_completion(show_output=True)"
       ]
     },
@@ -196,12 +172,19 @@
         "\n",
         "conda_run_config.environment.docker.enabled = True\n",
         "\n",
-        "cd = CondaDependencies.create(pip_packages=['azureml-sdk[automl]', 'applicationinsights', 'azureml-opendatasets', 'azureml-defaults'], \n",
-        "                              conda_packages=['numpy==1.16.2'], \n",
-        "                              pin_sdk_version=False)\n",
+        "cd = CondaDependencies.create(\n",
+        "    pip_packages=[\n",
+        "        \"azureml-sdk[automl]\",\n",
+        "        \"applicationinsights\",\n",
+        "        \"azureml-opendatasets\",\n",
+        "        \"azureml-defaults\",\n",
+        "    ],\n",
+        "    conda_packages=[\"numpy==1.16.2\"],\n",
+        "    pin_sdk_version=False,\n",
+        ")\n",
         "conda_run_config.environment.python.conda_dependencies = cd\n",
         "\n",
-        "print('run config is ready')"
+        "print(\"run config is ready\")"
       ]
     },
     {
@@ -218,7 +201,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "# The name and target column of the Dataset to create \n",
+        "# The name and target column of the Dataset to create\n",
         "dataset = \"NOAA-Weather-DS4\"\n",
         "target_column_name = \"temperature\""
       ]
@@ -242,12 +225,14 @@
         "from azureml.pipeline.steps import PythonScriptStep\n",
         "\n",
         "ds_name = PipelineParameter(name=\"ds_name\", default_value=dataset)\n",
-        "upload_data_step = PythonScriptStep(script_name=\"upload_weather_data.py\", \n",
-        "                                         allow_reuse=False,\n",
-        "                                         name=\"upload_weather_data\",\n",
-        "                                         arguments=[\"--ds_name\", ds_name],\n",
-        "                                         compute_target=compute_target, \n",
-        "                                         runconfig=conda_run_config)"
+        "upload_data_step = PythonScriptStep(\n",
+        "    script_name=\"upload_weather_data.py\",\n",
+        "    allow_reuse=False,\n",
+        "    name=\"upload_weather_data\",\n",
+        "    arguments=[\"--ds_name\", ds_name],\n",
+        "    compute_target=compute_target,\n",
+        "    runconfig=conda_run_config,\n",
+        ")"
       ]
     },
     {
@@ -264,10 +249,11 @@
       "outputs": [],
       "source": [
         "data_pipeline = Pipeline(\n",
-        "    description=\"pipeline_with_uploaddata\",\n",
-        "    workspace=ws,    \n",
-        "    steps=[upload_data_step])\n",
-        "data_pipeline_run = experiment.submit(data_pipeline, pipeline_parameters={\"ds_name\":dataset})"
+        "    description=\"pipeline_with_uploaddata\", workspace=ws, steps=[upload_data_step]\n",
+        ")\n",
+        "data_pipeline_run = experiment.submit(\n",
+        "    data_pipeline, pipeline_parameters={\"ds_name\": dataset}\n",
+        ")"
       ]
     },
     {
@@ -307,13 +293,14 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "data_prep_step = PythonScriptStep(script_name=\"check_data.py\", \n",
-        "                                         allow_reuse=False,\n",
-        "                                         name=\"check_data\",\n",
-        "                                         arguments=[\"--ds_name\", ds_name,\n",
-        "                                                    \"--model_name\", model_name],\n",
-        "                                         compute_target=compute_target, \n",
-        "                                         runconfig=conda_run_config)"
+        "data_prep_step = PythonScriptStep(\n",
+        "    script_name=\"check_data.py\",\n",
+        "    allow_reuse=False,\n",
+        "    name=\"check_data\",\n",
+        "    arguments=[\"--ds_name\", ds_name, \"--model_name\", model_name],\n",
+        "    compute_target=compute_target,\n",
+        "    runconfig=conda_run_config,\n",
+        ")"
       ]
     },
     {
@@ -323,6 +310,7 @@
       "outputs": [],
       "source": [
         "from azureml.core import Dataset\n",
+        "\n",
         "train_ds = Dataset.get_by_name(ws, dataset)\n",
         "train_ds = train_ds.drop_columns([\"partition_date\"])"
       ]
@@ -348,21 +336,22 @@
         "    \"iteration_timeout_minutes\": 10,\n",
         "    \"experiment_timeout_hours\": 0.25,\n",
         "    \"n_cross_validations\": 3,\n",
-        "    \"primary_metric\": 'normalized_root_mean_squared_error',\n",
+        "    \"primary_metric\": \"r2_score\",\n",
         "    \"max_concurrent_iterations\": 3,\n",
         "    \"max_cores_per_iteration\": -1,\n",
         "    \"verbosity\": logging.INFO,\n",
-        "    \"enable_early_stopping\": True\n",
+        "    \"enable_early_stopping\": True,\n",
         "}\n",
         "\n",
-        "automl_config = AutoMLConfig(task = 'regression',\n",
-        "                             debug_log = 'automl_errors.log',\n",
-        "                             path = \".\",\n",
-        "                             compute_target=compute_target,\n",
-        "                             training_data = train_ds,\n",
-        "                             label_column_name = target_column_name,\n",
-        "                             **automl_settings\n",
-        "                            )"
+        "automl_config = AutoMLConfig(\n",
+        "    task=\"regression\",\n",
+        "    debug_log=\"automl_errors.log\",\n",
+        "    path=\".\",\n",
+        "    compute_target=compute_target,\n",
+        "    training_data=train_ds,\n",
+        "    label_column_name=target_column_name,\n",
+        "    **automl_settings,\n",
+        ")"
       ]
     },
     {
@@ -373,17 +362,21 @@
       "source": [
         "from azureml.pipeline.core import PipelineData, TrainingOutput\n",
         "\n",
-        "metrics_output_name = 'metrics_output'\n",
-        "best_model_output_name = 'best_model_output'\n",
+        "metrics_output_name = \"metrics_output\"\n",
+        "best_model_output_name = \"best_model_output\"\n",
         "\n",
-        "metrics_data = PipelineData(name='metrics_data',\n",
-        "                           datastore=dstor,\n",
-        "                           pipeline_output_name=metrics_output_name,\n",
-        "                           training_output=TrainingOutput(type='Metrics'))\n",
-        "model_data = PipelineData(name='model_data',\n",
-        "                           datastore=dstor,\n",
-        "                           pipeline_output_name=best_model_output_name,\n",
-        "                           training_output=TrainingOutput(type='Model'))"
+        "metrics_data = PipelineData(\n",
+        "    name=\"metrics_data\",\n",
+        "    datastore=dstor,\n",
+        "    pipeline_output_name=metrics_output_name,\n",
+        "    training_output=TrainingOutput(type=\"Metrics\"),\n",
+        ")\n",
+        "model_data = PipelineData(\n",
+        "    name=\"model_data\",\n",
+        "    datastore=dstor,\n",
+        "    pipeline_output_name=best_model_output_name,\n",
+        "    training_output=TrainingOutput(type=\"Model\"),\n",
+        ")"
       ]
     },
     {
@@ -393,10 +386,11 @@
       "outputs": [],
       "source": [
         "automl_step = AutoMLStep(\n",
-        "    name='automl_module',\n",
+        "    name=\"automl_module\",\n",
         "    automl_config=automl_config,\n",
         "    outputs=[metrics_data, model_data],\n",
-        "    allow_reuse=False)"
+        "    allow_reuse=False,\n",
+        ")"
       ]
     },
     {
@@ -413,13 +407,22 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "register_model_step = PythonScriptStep(script_name=\"register_model.py\",\n",
-        "                                       name=\"register_model\",\n",
-        "                                       allow_reuse=False,\n",
-        "                                       arguments=[\"--model_name\", model_name, \"--model_path\", model_data, \"--ds_name\", ds_name],\n",
-        "                                       inputs=[model_data],\n",
-        "                                       compute_target=compute_target,\n",
-        "                                       runconfig=conda_run_config)"
+        "register_model_step = PythonScriptStep(\n",
+        "    script_name=\"register_model.py\",\n",
+        "    name=\"register_model\",\n",
+        "    allow_reuse=False,\n",
+        "    arguments=[\n",
+        "        \"--model_name\",\n",
+        "        model_name,\n",
+        "        \"--model_path\",\n",
+        "        model_data,\n",
+        "        \"--ds_name\",\n",
+        "        ds_name,\n",
+        "    ],\n",
+        "    inputs=[model_data],\n",
+        "    compute_target=compute_target,\n",
+        "    runconfig=conda_run_config,\n",
+        ")"
       ]
     },
     {
@@ -437,8 +440,9 @@
       "source": [
         "training_pipeline = Pipeline(\n",
         "    description=\"training_pipeline\",\n",
-        "    workspace=ws,    \n",
-        "    steps=[data_prep_step, automl_step, register_model_step])"
+        "    workspace=ws,\n",
+        "    steps=[data_prep_step, automl_step, register_model_step],\n",
+        ")"
       ]
     },
     {
@@ -447,8 +451,10 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "training_pipeline_run = experiment.submit(training_pipeline, pipeline_parameters={\n",
-        "        \"ds_name\": dataset, \"model_name\": \"noaaweatherds\"})"
+        "training_pipeline_run = experiment.submit(\n",
+        "    training_pipeline,\n",
+        "    pipeline_parameters={\"ds_name\": dataset, \"model_name\": \"noaaweatherds\"},\n",
+        ")"
       ]
     },
     {
@@ -477,8 +483,8 @@
         "pipeline_name = \"Retraining-Pipeline-NOAAWeather\"\n",
         "\n",
         "published_pipeline = training_pipeline.publish(\n",
-        "    name=pipeline_name, \n",
-        "    description=\"Pipeline that retrains AutoML model\")\n",
+        "    name=pipeline_name, description=\"Pipeline that retrains AutoML model\"\n",
+        ")\n",
         "\n",
         "published_pipeline"
       ]
@@ -490,13 +496,17 @@
       "outputs": [],
       "source": [
         "from azureml.pipeline.core import Schedule\n",
-        "schedule = Schedule.create(workspace=ws, name=\"RetrainingSchedule\",\n",
-        "                           pipeline_parameters={\"ds_name\": dataset, \"model_name\": \"noaaweatherds\"},\n",
-        "                           pipeline_id=published_pipeline.id, \n",
-        "                           experiment_name=experiment_name, \n",
-        "                           datastore=dstor,\n",
-        "                           wait_for_provisioning=True,\n",
-        "                           polling_interval=1440)"
+        "\n",
+        "schedule = Schedule.create(\n",
+        "    workspace=ws,\n",
+        "    name=\"RetrainingSchedule\",\n",
+        "    pipeline_parameters={\"ds_name\": dataset, \"model_name\": \"noaaweatherds\"},\n",
+        "    pipeline_id=published_pipeline.id,\n",
+        "    experiment_name=experiment_name,\n",
+        "    datastore=dstor,\n",
+        "    wait_for_provisioning=True,\n",
+        "    polling_interval=1440,\n",
+        ")"
       ]
     },
     {
@@ -520,8 +530,8 @@
         "pipeline_name = \"DataIngestion-Pipeline-NOAAWeather\"\n",
         "\n",
         "published_pipeline = training_pipeline.publish(\n",
-        "    name=pipeline_name, \n",
-        "    description=\"Pipeline that updates NOAAWeather Dataset\")\n",
+        "    name=pipeline_name, description=\"Pipeline that updates NOAAWeather Dataset\"\n",
+        ")\n",
         "\n",
         "published_pipeline"
       ]
@@ -533,13 +543,17 @@
       "outputs": [],
       "source": [
         "from azureml.pipeline.core import Schedule\n",
-        "schedule = Schedule.create(workspace=ws, name=\"RetrainingSchedule-DataIngestion\",\n",
-        "                           pipeline_parameters={\"ds_name\":dataset},\n",
-        "                           pipeline_id=published_pipeline.id, \n",
-        "                           experiment_name=experiment_name, \n",
-        "                           datastore=dstor,\n",
-        "                           wait_for_provisioning=True,\n",
-        "                           polling_interval=1440)"
+        "\n",
+        "schedule = Schedule.create(\n",
+        "    workspace=ws,\n",
+        "    name=\"RetrainingSchedule-DataIngestion\",\n",
+        "    pipeline_parameters={\"ds_name\": dataset},\n",
+        "    pipeline_id=published_pipeline.id,\n",
+        "    experiment_name=experiment_name,\n",
+        "    datastore=dstor,\n",
+        "    wait_for_provisioning=True,\n",
+        "    polling_interval=1440,\n",
+        ")"
       ]
     }
   ],

how-to-use-azureml/automated-machine-learning/continuous-retraining/check_data.py

@@ -31,7 +31,7 @@ try:
     model = Model(ws, args.model_name)
     last_train_time = model.created_time
     print("Model was last trained on {0}.".format(last_train_time))
-except Exception:
+except Exception as e:
     print("Could not get last model train time.")
     last_train_time = datetime.min.replace(tzinfo=pytz.UTC)
 

how-to-use-azureml/automated-machine-learning/continuous-retraining/register_model.py

@@ -25,9 +25,11 @@ datasets = [(Dataset.Scenario.TRAINING, train_ds)]
 
 # Register model with training dataset
 
-model = Model.register(workspace=ws,
-                       model_path=args.model_path,
-                       model_name=args.model_name,
-                       datasets=datasets)
+model = Model.register(
+    workspace=ws,
+    model_path=args.model_path,
+    model_name=args.model_name,
+    datasets=datasets,
+)
 
 print("Registered version {0} of model {1}".format(model.version, model.name))

how-to-use-azureml/automated-machine-learning/continuous-retraining/upload_weather_data.py

@@ -16,26 +16,82 @@ if type(run) == _OfflineRun:
 else:
     ws = run.experiment.workspace
 
-usaf_list = ['725724', '722149', '723090', '722159', '723910', '720279',
-             '725513', '725254', '726430', '720381', '723074', '726682',
-             '725486', '727883', '723177', '722075', '723086', '724053',
-             '725070', '722073', '726060', '725224', '725260', '724520',
-             '720305', '724020', '726510', '725126', '722523', '703333',
-             '722249', '722728', '725483', '722972', '724975', '742079',
-             '727468', '722193', '725624', '722030', '726380', '720309',
-             '722071', '720326', '725415', '724504', '725665', '725424',
-             '725066']
+usaf_list = [
+    "725724",
+    "722149",
+    "723090",
+    "722159",
+    "723910",
+    "720279",
+    "725513",
+    "725254",
+    "726430",
+    "720381",
+    "723074",
+    "726682",
+    "725486",
+    "727883",
+    "723177",
+    "722075",
+    "723086",
+    "724053",
+    "725070",
+    "722073",
+    "726060",
+    "725224",
+    "725260",
+    "724520",
+    "720305",
+    "724020",
+    "726510",
+    "725126",
+    "722523",
+    "703333",
+    "722249",
+    "722728",
+    "725483",
+    "722972",
+    "724975",
+    "742079",
+    "727468",
+    "722193",
+    "725624",
+    "722030",
+    "726380",
+    "720309",
+    "722071",
+    "720326",
+    "725415",
+    "724504",
+    "725665",
+    "725424",
+    "725066",
+]
 
 
 def get_noaa_data(start_time, end_time):
-    columns = ['usaf', 'wban', 'datetime', 'latitude', 'longitude', 'elevation',
-               'windAngle', 'windSpeed', 'temperature', 'stationName', 'p_k']
+    columns = [
+        "usaf",
+        "wban",
+        "datetime",
+        "latitude",
+        "longitude",
+        "elevation",
+        "windAngle",
+        "windSpeed",
+        "temperature",
+        "stationName",
+        "p_k",
+    ]
     isd = NoaaIsdWeather(start_time, end_time, cols=columns)
     noaa_df = isd.to_pandas_dataframe()
     df_filtered = noaa_df[noaa_df["usaf"].isin(usaf_list)]
     df_filtered.reset_index(drop=True)
-    print("Received {0} rows of training data between {1} and {2}".format(
-        df_filtered.shape[0], start_time, end_time))
+    print(
+        "Received {0} rows of training data between {1} and {2}".format(
+            df_filtered.shape[0], start_time, end_time
+        )
+    )
     return df_filtered
 
 
@@ -54,11 +110,12 @@ end_time = datetime.utcnow()
 try:
     ds = Dataset.get_by_name(ws, args.ds_name)
     end_time_last_slice = ds.data_changed_time.replace(tzinfo=None)
-    print("Dataset {0} last updated on {1}".format(args.ds_name,
-                                                   end_time_last_slice))
+    print("Dataset {0} last updated on {1}".format(args.ds_name, end_time_last_slice))
 except Exception:
     print(traceback.format_exc())
-    print("Dataset with name {0} not found, registering new dataset.".format(args.ds_name))
+    print(
+        "Dataset with name {0} not found, registering new dataset.".format(args.ds_name)
+    )
     register_dataset = True
     end_time = datetime(2021, 5, 1, 0, 0)
     end_time_last_slice = end_time - relativedelta(weeks=2)
@@ -66,26 +123,35 @@ except Exception:
 train_df = get_noaa_data(end_time_last_slice, end_time)
 
 if train_df.size > 0:
-    print("Received {0} rows of new data after {1}.".format(
-        train_df.shape[0], end_time_last_slice))
-    folder_name = "{}/{:04d}/{:02d}/{:02d}/{:02d}/{:02d}/{:02d}".format(args.ds_name, end_time.year,
-                                                                        end_time.month, end_time.day,
-                                                                        end_time.hour, end_time.minute,
-                                                                        end_time.second)
+    print(
+        "Received {0} rows of new data after {1}.".format(
+            train_df.shape[0], end_time_last_slice
+        )
+    )
+    folder_name = "{}/{:04d}/{:02d}/{:02d}/{:02d}/{:02d}/{:02d}".format(
+        args.ds_name,
+        end_time.year,
+        end_time.month,
+        end_time.day,
+        end_time.hour,
+        end_time.minute,
+        end_time.second,
+    )
     file_path = "{0}/data.csv".format(folder_name)
 
     # Add a new partition to the registered dataset
     os.makedirs(folder_name, exist_ok=True)
     train_df.to_csv(file_path, index=False)
 
-    dstor.upload_files(files=[file_path],
-                       target_path=folder_name,
-                       overwrite=True,
-                       show_progress=True)
+    dstor.upload_files(
+        files=[file_path], target_path=folder_name, overwrite=True, show_progress=True
+    )
 else:
     print("No new data since {0}.".format(end_time_last_slice))
 
 if register_dataset:
-    ds = Dataset.Tabular.from_delimited_files(dstor.path("{}/**/*.csv".format(
-        args.ds_name)), partition_format='/{partition_date:yyyy/MM/dd/HH/mm/ss}/data.csv')
+    ds = Dataset.Tabular.from_delimited_files(
+        dstor.path("{}/**/*.csv".format(args.ds_name)),
+        partition_format="/{partition_date:yyyy/MM/dd/HH/mm/ss}/data.csv",
+    )
     ds.register(ws, name=args.ds_name)

how-to-use-azureml/automated-machine-learning/experimental/automl_thin_client_env.yml

@@ -1,17 +1,19 @@
 name: azure_automl_experimental
 dependencies:
   # The python interpreter version.
-  # Currently Azure ML only supports 3.5.2 and later.
-- pip<=19.3.1
-- python>=3.5.2,<3.8
-- nb_conda
-- cython
-- urllib3<1.24
+  # Currently Azure ML only supports 3.6.0 and later.
+- pip<=20.2.4
+- python>=3.6.0,<3.9
+- cython==0.29.14
+- urllib3==1.26.7
 - PyJWT < 2.0.0
 - numpy==1.18.5
+- pywin32==227
 
 - pip:
   # Required packages for AzureML execution, history, and data preparation.
+  - azure-core==1.21.1
+  - azure-identity==1.7.0
   - azureml-defaults
   - azureml-sdk
   - azureml-widgets

how-to-use-azureml/automated-machine-learning/experimental/automl_thin_client_env_mac.yml

@@ -1,18 +1,21 @@
 name: azure_automl_experimental
+channels:
+  - conda-forge
+  - main
 dependencies:
   # The python interpreter version.
-  # Currently Azure ML only supports 3.5.2 and later.
-- pip<=19.3.1
+  # Currently Azure ML only supports 3.6.0 and later.
+- pip<=20.2.4
 - nomkl
-- python>=3.5.2,<3.8
-- nb_conda
-- cython
-- urllib3<1.24
+- python>=3.6.0,<3.9
+- urllib3==1.26.7
 - PyJWT < 2.0.0
-- numpy==1.18.5
+- numpy==1.19.5
 
 - pip:
   # Required packages for AzureML execution, history, and data preparation.
+  - azure-core==1.21.1
+  - azure-identity==1.7.0
   - azureml-defaults
   - azureml-sdk
   - azureml-widgets

how-to-use-azureml/automated-machine-learning/experimental/classification-credit-card-fraud-local-managed/auto-ml-classification-credit-card-fraud-local-managed.ipynb

@@ -92,7 +92,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "print(\"This notebook was created using version 1.36.0 of the Azure ML SDK\")\n",
+        "print(\"This notebook was created using version 1.41.0 of the Azure ML SDK\")\n",
         "print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
       ]
     },
@@ -115,7 +115,7 @@
         "output['Resource Group'] = ws.resource_group\n",
         "output['Location'] = ws.location\n",
         "output['Experiment Name'] = experiment.name\n",
-        "pd.set_option('display.max_colwidth', -1)\n",
+        "pd.set_option('display.max_colwidth', None)\n",
         "outputDf = pd.DataFrame(data = output, index = [''])\n",
         "outputDf.T"
       ]

how-to-use-azureml/automated-machine-learning/experimental/regression-model-proxy/auto-ml-regression-model-proxy.ipynb

@@ -91,7 +91,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "print(\"This notebook was created using version 1.36.0 of the Azure ML SDK\")\n",
+        "print(\"This notebook was created using version 1.41.0 of the Azure ML SDK\")\n",
         "print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
       ]
     },
@@ -180,6 +180,29 @@
         "label = \"ERP\"\n"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "The split data will be used in the remote compute by ModelProxy and locally to compare results.\n",
+        "So, we need to persist the split data to avoid descrepencies from different package versions in the local and remote."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "ds = ws.get_default_datastore()\n",
+        "\n",
+        "train_data = Dataset.Tabular.register_pandas_dataframe(\n",
+        "    train_data.to_pandas_dataframe(), target=(ds, \"machineTrainData\"), name=\"train_data\")\n",
+        "\n",
+        "test_data = Dataset.Tabular.register_pandas_dataframe(\n",
+        "    test_data.to_pandas_dataframe(), target=(ds, \"machineTestData\"), name=\"test_data\")"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {},
@@ -304,7 +327,8 @@
       "metadata": {},
       "source": [
         "#### Show hyperparameters\n",
-        "Show the model pipeline used for the best run with its hyperparameters."
+        "Show the model pipeline used for the best run with its hyperparameters.\n",
+        "For ensemble pipelines it shows the iterations and algorithms that are ensembled."
       ]
     },
     {
@@ -313,8 +337,19 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "run_properties = json.loads(best_run.get_details()['properties']['pipeline_script'])\n",
-        "print(json.dumps(run_properties, indent = 1)) "
+        "run_properties = best_run.get_details()['properties']\n",
+        "pipeline_script = json.loads(run_properties['pipeline_script'])\n",
+        "print(json.dumps(pipeline_script, indent = 1)) \n",
+        "\n",
+        "if 'ensembled_iterations' in run_properties:\n",
+        "    print(\"\")\n",
+        "    print(\"Ensembled Iterations\")\n",
+        "    print(run_properties['ensembled_iterations'])\n",
+        "    \n",
+        "if 'ensembled_algorithms' in run_properties:\n",
+        "    print(\"\")\n",
+        "    print(\"Ensembled Algorithms\")\n",
+        "    print(run_properties['ensembled_algorithms'])"
       ]
     },
     {

how-to-use-azureml/automated-machine-learning/forecasting-backtest-many-models/assets/score.py

@@ -0,0 +1,171 @@
+from typing import Any, Dict, Optional, List
+
+import argparse
+import json
+import os
+import re
+
+import numpy as np
+import pandas as pd
+
+from matplotlib import pyplot as plt
+from matplotlib.backends.backend_pdf import PdfPages
+
+from azureml.automl.core.shared import constants
+from azureml.automl.core.shared.types import GrainType
+from azureml.automl.runtime.shared.score import scoring
+
+GRAIN = "time_series_id"
+BACKTEST_ITER = "backtest_iteration"
+ACTUALS = "actual_level"
+PREDICTIONS = "predicted_level"
+ALL_GRAINS = "all_sets"
+
+FORECASTS_FILE = "forecast.csv"
+SCORES_FILE = "scores.csv"
+PLOTS_FILE = "plots_fcst_vs_actual.pdf"
+RE_INVALID_SYMBOLS = re.compile("[: ]")
+
+
+def _compute_metrics(df: pd.DataFrame, metrics: List[str]):
+    """
+    Compute metrics for one data frame.
+
+    :param df: The data frame which contains actual_level and predicted_level columns.
+    :return: The data frame with two columns - metric_name and metric.
+    """
+    scores = scoring.score_regression(
+        y_test=df[ACTUALS], y_pred=df[PREDICTIONS], metrics=metrics
+    )
+    metrics_df = pd.DataFrame(list(scores.items()), columns=["metric_name", "metric"])
+    metrics_df.sort_values(["metric_name"], inplace=True)
+    metrics_df.reset_index(drop=True, inplace=True)
+    return metrics_df
+
+
+def _format_grain_name(grain: GrainType) -> str:
+    """
+    Convert grain name to string.
+
+    :param grain: the grain name.
+    :return: the string representation of the given grain.
+    """
+    if not isinstance(grain, tuple) and not isinstance(grain, list):
+        return str(grain)
+    grain = list(map(str, grain))
+    return "|".join(grain)
+
+
+def compute_all_metrics(
+    fcst_df: pd.DataFrame,
+    ts_id_colnames: List[str],
+    metric_names: Optional[List[set]] = None,
+):
+    """
+    Calculate metrics per grain.
+
+    :param fcst_df: forecast data frame. Must contain 2 columns: 'actual_level' and 'predicted_level'
+    :param metric_names: (optional) the list of metric names to return
+    :param ts_id_colnames: (optional) list of grain column names
+    :return: dictionary of summary table for all tests and final decision on stationary vs nonstaionary
+    """
+    if not metric_names:
+        metric_names = list(constants.Metric.SCALAR_REGRESSION_SET)
+
+    if ts_id_colnames is None:
+        ts_id_colnames = []
+
+    metrics_list = []
+    if ts_id_colnames:
+        for grain, df in fcst_df.groupby(ts_id_colnames):
+            one_grain_metrics_df = _compute_metrics(df, metric_names)
+            one_grain_metrics_df[GRAIN] = _format_grain_name(grain)
+            metrics_list.append(one_grain_metrics_df)
+
+    # overall metrics
+    one_grain_metrics_df = _compute_metrics(fcst_df, metric_names)
+    one_grain_metrics_df[GRAIN] = ALL_GRAINS
+    metrics_list.append(one_grain_metrics_df)
+
+    # collect into a data frame
+    return pd.concat(metrics_list)
+
+
+def _draw_one_plot(
+    df: pd.DataFrame,
+    time_column_name: str,
+    grain_column_names: List[str],
+    pdf: PdfPages,
+) -> None:
+    """
+    Draw the single plot.
+
+    :param df: The data frame with the data to build plot.
+    :param time_column_name: The name of a time column.
+    :param grain_column_names: The name of grain columns.
+    :param pdf: The pdf backend used to render the plot.
+    """
+    fig, _ = plt.subplots(figsize=(20, 10))
+    df = df.set_index(time_column_name)
+    plt.plot(df[[ACTUALS, PREDICTIONS]])
+    plt.xticks(rotation=45)
+    iteration = df[BACKTEST_ITER].iloc[0]
+    if grain_column_names:
+        grain_name = [df[grain].iloc[0] for grain in grain_column_names]
+        plt.title(f"Time series ID: {_format_grain_name(grain_name)} {iteration}")
+    plt.legend(["actual", "forecast"])
+    plt.close(fig)
+    pdf.savefig(fig)
+
+
+def calculate_scores_and_build_plots(
+    input_dir: str, output_dir: str, automl_settings: Dict[str, Any]
+):
+    os.makedirs(output_dir, exist_ok=True)
+    grains = automl_settings.get(constants.TimeSeries.TIME_SERIES_ID_COLUMN_NAMES)
+    time_column_name = automl_settings.get(constants.TimeSeries.TIME_COLUMN_NAME)
+    if grains is None:
+        grains = []
+    if isinstance(grains, str):
+        grains = [grains]
+    while BACKTEST_ITER in grains:
+        grains.remove(BACKTEST_ITER)
+
+    dfs = []
+    for fle in os.listdir(input_dir):
+        file_path = os.path.join(input_dir, fle)
+        if os.path.isfile(file_path) and file_path.endswith(".csv"):
+            df_iter = pd.read_csv(file_path, parse_dates=[time_column_name])
+            for _, iteration in df_iter.groupby(BACKTEST_ITER):
+                dfs.append(iteration)
+    forecast_df = pd.concat(dfs, sort=False, ignore_index=True)
+    # To make sure plots are in order, sort the predictions by grain and iteration.
+    ts_index = grains + [BACKTEST_ITER]
+    forecast_df.sort_values(by=ts_index, inplace=True)
+    pdf = PdfPages(os.path.join(output_dir, PLOTS_FILE))
+    for _, one_forecast in forecast_df.groupby(ts_index):
+        _draw_one_plot(one_forecast, time_column_name, grains, pdf)
+    pdf.close()
+    forecast_df.to_csv(os.path.join(output_dir, FORECASTS_FILE), index=False)
+    # Remove np.NaN and np.inf from the prediction and actuals data.
+    forecast_df.replace([np.inf, -np.inf], np.nan, inplace=True)
+    forecast_df.dropna(subset=[ACTUALS, PREDICTIONS], inplace=True)
+    metrics = compute_all_metrics(forecast_df, grains + [BACKTEST_ITER])
+    metrics.to_csv(os.path.join(output_dir, SCORES_FILE), index=False)
+
+
+if __name__ == "__main__":
+    args = {"forecasts": "--forecasts", "scores_out": "--output-dir"}
+    parser = argparse.ArgumentParser("Parsing input arguments.")
+    for argname, arg in args.items():
+        parser.add_argument(arg, dest=argname, required=True)
+    parsed_args, _ = parser.parse_known_args()
+    input_dir = parsed_args.forecasts
+    output_dir = parsed_args.scores_out
+    with open(
+        os.path.join(
+            os.path.dirname(os.path.realpath(__file__)), "automl_settings.json"
+        )
+    ) as json_file:
+        automl_settings = json.load(json_file)
+    calculate_scores_and_build_plots(input_dir, output_dir, automl_settings)

how-to-use-azureml/automated-machine-learning/forecasting-backtest-many-models/auto-ml-forecasting-backtest-many-models.ipynb

@@ -0,0 +1,726 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
+        "\n",
+        "Licensed under the MIT License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/automated-machine-learning/forecasting-hierarchical-timeseries/auto-ml-forecasting-hierarchical-timeseries.png)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# Many Models with Backtesting - Automated ML\n",
+        "**_Backtest many models time series forecasts with Automated Machine Learning_**\n",
+        "\n",
+        "---"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "For this notebook we are using a synthetic dataset to demonstrate the back testing in many model scenario. This allows us to check historical performance of AutoML on a historical data. To do that we step back on the backtesting period by the data set several times and split the data to train and test sets. Then these data sets are used for training and evaluation of model.<br>\n",
+        "\n",
+        "Thus, it is a quick way of evaluating AutoML as if it was in production. Here, we do not test historical performance of a particular model, for this see the [notebook](../forecasting-backtest-single-model/auto-ml-forecasting-backtest-single-model.ipynb). Instead, the best model for every backtest iteration can be different since AutoML chooses the best model for a given training set.\n",
+        "![Backtesting](Backtesting.png)\n",
+        "\n",
+        "**NOTE: There are limits on how many runs we can do in parallel per workspace, and we currently recommend to set the parallelism to maximum of 320 runs per experiment per workspace. If users want to have more parallelism and increase this limit they might encounter Too Many Requests errors (HTTP 429).**"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Prerequisites\n",
+        "You'll need to create a compute Instance by following the instructions in the [EnvironmentSetup.md](../Setup_Resources/EnvironmentSetup.md)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## 1.0 Set up workspace, datastore, experiment"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "gather": {
+          "logged": 1613003526897
+        }
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "import azureml.core\n",
+        "from azureml.core import Workspace, Datastore\n",
+        "import numpy as np\n",
+        "import pandas as pd\n",
+        "\n",
+        "from pandas.tseries.frequencies import to_offset\n",
+        "\n",
+        "# Set up your workspace\n",
+        "ws = Workspace.from_config()\n",
+        "ws.get_details()\n",
+        "\n",
+        "# Set up your datastores\n",
+        "dstore = ws.get_default_datastore()\n",
+        "\n",
+        "output = {}\n",
+        "output[\"SDK version\"] = azureml.core.VERSION\n",
+        "output[\"Subscription ID\"] = ws.subscription_id\n",
+        "output[\"Workspace\"] = ws.name\n",
+        "output[\"Resource Group\"] = ws.resource_group\n",
+        "output[\"Location\"] = ws.location\n",
+        "output[\"Default datastore name\"] = dstore.name\n",
+        "output[\"SDK Version\"] = azureml.core.VERSION\n",
+        "pd.set_option(\"display.max_colwidth\", None)\n",
+        "outputDf = pd.DataFrame(data=output, index=[\"\"])\n",
+        "outputDf.T"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "This notebook is compatible with Azure ML SDK version 1.35.1 or later."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Choose an experiment"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "gather": {
+          "logged": 1613003540729
+        }
+      },
+      "outputs": [],
+      "source": [
+        "from azureml.core import Experiment\n",
+        "\n",
+        "experiment = Experiment(ws, \"automl-many-models-backtest\")\n",
+        "\n",
+        "print(\"Experiment name: \" + experiment.name)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## 2.0 Data\n",
+        "\n",
+        "#### 2.1 Data generation\n",
+        "For this notebook we will generate the artificial data set with two [time series IDs](https://docs.microsoft.com/en-us/python/api/azureml-automl-core/azureml.automl.core.forecasting_parameters.forecastingparameters?view=azure-ml-py). Then we will generate backtest folds and will upload it to the default BLOB storage and create a [TabularDataset](https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.data.tabular_dataset.tabulardataset?view=azure-ml-py)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# simulate data: 2 grains - 700\n",
+        "TIME_COLNAME = \"date\"\n",
+        "TARGET_COLNAME = \"value\"\n",
+        "TIME_SERIES_ID_COLNAME = \"ts_id\"\n",
+        "\n",
+        "sample_size = 700\n",
+        "# Set the random seed for reproducibility of results.\n",
+        "np.random.seed(20)\n",
+        "X1 = pd.DataFrame(\n",
+        "    {\n",
+        "        TIME_COLNAME: pd.date_range(start=\"2018-01-01\", periods=sample_size),\n",
+        "        TARGET_COLNAME: np.random.normal(loc=100, scale=20, size=sample_size),\n",
+        "        TIME_SERIES_ID_COLNAME: \"ts_A\",\n",
+        "    }\n",
+        ")\n",
+        "X2 = pd.DataFrame(\n",
+        "    {\n",
+        "        TIME_COLNAME: pd.date_range(start=\"2018-01-01\", periods=sample_size),\n",
+        "        TARGET_COLNAME: np.random.normal(loc=100, scale=20, size=sample_size),\n",
+        "        TIME_SERIES_ID_COLNAME: \"ts_B\",\n",
+        "    }\n",
+        ")\n",
+        "\n",
+        "X = pd.concat([X1, X2], ignore_index=True, sort=False)\n",
+        "print(\"Simulated dataset contains {} rows \\n\".format(X.shape[0]))\n",
+        "X.head()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Now we will generate 8 backtesting folds with backtesting period of 7 days and with the same forecasting horizon. We will add the column \"backtest_iteration\", which will identify the backtesting period by the last training date."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "offset_type = \"7D\"\n",
+        "NUMBER_OF_BACKTESTS = 8  # number of train/test sets to generate\n",
+        "\n",
+        "dfs_train = []\n",
+        "dfs_test = []\n",
+        "for ts_id, df_one in X.groupby(TIME_SERIES_ID_COLNAME):\n",
+        "\n",
+        "    data_end = df_one[TIME_COLNAME].max()\n",
+        "\n",
+        "    for i in range(NUMBER_OF_BACKTESTS):\n",
+        "        train_cutoff_date = data_end - to_offset(offset_type)\n",
+        "        df_one = df_one.copy()\n",
+        "        df_one[\"backtest_iteration\"] = \"iteration_\" + str(train_cutoff_date)\n",
+        "        train = df_one[df_one[TIME_COLNAME] <= train_cutoff_date]\n",
+        "        test = df_one[\n",
+        "            (df_one[TIME_COLNAME] > train_cutoff_date)\n",
+        "            & (df_one[TIME_COLNAME] <= data_end)\n",
+        "        ]\n",
+        "        data_end = train[TIME_COLNAME].max()\n",
+        "        dfs_train.append(train)\n",
+        "        dfs_test.append(test)\n",
+        "\n",
+        "X_train = pd.concat(dfs_train, sort=False, ignore_index=True)\n",
+        "X_test = pd.concat(dfs_test, sort=False, ignore_index=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### 2.2 Create the Tabular Data Set.\n",
+        "\n",
+        "A Datastore is a place where data can be stored that is then made accessible to a compute either by means of mounting or copying the data to the compute target.\n",
+        "\n",
+        "Please refer to [Datastore](https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.core.datastore(class)?view=azure-ml-py) documentation on how to access data from Datastore.\n",
+        "\n",
+        "In this next step, we will upload the data and create a TabularDataset."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.data.dataset_factory import TabularDatasetFactory\n",
+        "\n",
+        "ds = ws.get_default_datastore()\n",
+        "# Upload saved data to the default data store.\n",
+        "train_data = TabularDatasetFactory.register_pandas_dataframe(\n",
+        "    X_train, target=(ds, \"data_mm\"), name=\"data_train\"\n",
+        ")\n",
+        "test_data = TabularDatasetFactory.register_pandas_dataframe(\n",
+        "    X_test, target=(ds, \"data_mm\"), name=\"data_test\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## 3.0 Build the training pipeline\n",
+        "Now that the dataset, WorkSpace, and datastore are set up, we can put together a pipeline for training.\n",
+        "\n",
+        "> Note that if you have an AzureML Data Scientist role, you will not have permission to create compute resources. Talk to your workspace or IT admin to create the compute targets described in this section, if they do not already exist."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Choose a compute target\n",
+        "\n",
+        "You will need to create a [compute target](https://docs.microsoft.com/en-us/azure/machine-learning/how-to-set-up-training-targets#amlcompute) for your AutoML run. In this tutorial, you create AmlCompute as your training compute resource.\n",
+        "\n",
+        "\\*\\*Creation of AmlCompute takes approximately 5 minutes.**\n",
+        "\n",
+        "If the AmlCompute with that name is already in your workspace this code will skip the creation process. As with other Azure services, there are limits on certain resources (e.g. AmlCompute) associated with the Azure Machine Learning service. Please read this [article](https://docs.microsoft.com/en-us/azure/machine-learning/how-to-manage-quotas) on the default limits and how to request more quota."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "gather": {
+          "logged": 1613007037308
+        }
+      },
+      "outputs": [],
+      "source": [
+        "from azureml.core.compute import ComputeTarget, AmlCompute\n",
+        "\n",
+        "# Name your cluster\n",
+        "compute_name = \"backtest-mm\"\n",
+        "\n",
+        "\n",
+        "if compute_name in ws.compute_targets:\n",
+        "    compute_target = ws.compute_targets[compute_name]\n",
+        "    if compute_target and type(compute_target) is AmlCompute:\n",
+        "        print(\"Found compute target: \" + compute_name)\n",
+        "else:\n",
+        "    print(\"Creating a new compute target...\")\n",
+        "    provisioning_config = AmlCompute.provisioning_configuration(\n",
+        "        vm_size=\"STANDARD_DS12_V2\", max_nodes=6\n",
+        "    )\n",
+        "    # Create the compute target\n",
+        "    compute_target = ComputeTarget.create(ws, compute_name, provisioning_config)\n",
+        "\n",
+        "    # Can poll for a minimum number of nodes and for a specific timeout.\n",
+        "    # If no min node count is provided it will use the scale settings for the cluster\n",
+        "    compute_target.wait_for_completion(\n",
+        "        show_output=True, min_node_count=None, timeout_in_minutes=20\n",
+        "    )\n",
+        "\n",
+        "    # For a more detailed view of current cluster status, use the 'status' property\n",
+        "    print(compute_target.status.serialize())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Set up training parameters\n",
+        "\n",
+        "This dictionary defines the AutoML and many models settings. For this forecasting task we need to define several settings including the name of the time column, the maximum forecast horizon, and the partition column name definition. Please note, that in this case we are setting grain_column_names to be the time series ID column plus iteration, because we want to train a separate model for each time series and iteration.\n",
+        "\n",
+        "| Property                           | Description|\n",
+        "| :---------------                   | :------------------- |\n",
+        "| **task**                           | forecasting |\n",
+        "| **primary_metric**                 | This is the metric that you want to optimize.<br> Forecasting supports the following primary metrics <br><i>normalized_root_mean_squared_error</i><br><i>normalized_mean_absolute_error</i> |\n",
+        "| **iteration_timeout_minutes**      | Maximum amount of time in minutes that the model can train. This is optional but provides customers with greater control on exit criteria. |\n",
+        "| **iterations**                     | Number of models to train. This is optional but provides customers with greater control on exit criteria. |\n",
+        "| **experiment_timeout_hours**       | Maximum amount of time in hours that the experiment can take before it terminates. This is optional but provides customers with greater control on exit criteria. |\n",
+        "| **label_column_name**              | The name of the label column. |\n",
+        "| **forecast_horizon**               | The forecast horizon is how many periods forward you would like to forecast. This integer horizon is in units of the timeseries frequency (e.g. daily, weekly). Periods are inferred from your data. |\n",
+        "| **n_cross_validations**            | Number of cross validation splits. Rolling Origin Validation is used to split time-series in a temporally consistent way. |\n",
+        "| **time_column_name**               | The name of your time column. |\n",
+        "| **time_series_id_column_names**     | The column names used to uniquely identify timeseries in data that has multiple rows with the same timestamp. |\n",
+        "| **track_child_runs**               | Flag to disable tracking of child runs. Only best run is tracked if the flag is set to False (this includes the model and metrics of the run). |\n",
+        "| **partition_column_names**         | The names of columns used to group your models. For timeseries, the groups must not split up individual time-series. That is, each group must contain one or more whole time-series. |"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "gather": {
+          "logged": 1613007061544
+        }
+      },
+      "outputs": [],
+      "source": [
+        "from azureml.train.automl.runtime._many_models.many_models_parameters import (\n",
+        "    ManyModelsTrainParameters,\n",
+        ")\n",
+        "\n",
+        "partition_column_names = [TIME_SERIES_ID_COLNAME, \"backtest_iteration\"]\n",
+        "automl_settings = {\n",
+        "    \"task\": \"forecasting\",\n",
+        "    \"primary_metric\": \"normalized_root_mean_squared_error\",\n",
+        "    \"iteration_timeout_minutes\": 10,  # This needs to be changed based on the dataset. We ask customer to explore how long training is taking before settings this value\n",
+        "    \"iterations\": 15,\n",
+        "    \"experiment_timeout_hours\": 0.25,  # This also needs to be changed based on the dataset. For larger data set this number needs to be bigger.\n",
+        "    \"label_column_name\": TARGET_COLNAME,\n",
+        "    \"n_cross_validations\": 3,\n",
+        "    \"time_column_name\": TIME_COLNAME,\n",
+        "    \"forecast_horizon\": 6,\n",
+        "    \"time_series_id_column_names\": partition_column_names,\n",
+        "    \"track_child_runs\": False,\n",
+        "}\n",
+        "\n",
+        "mm_paramters = ManyModelsTrainParameters(\n",
+        "    automl_settings=automl_settings, partition_column_names=partition_column_names\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Set up many models pipeline"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Parallel run step is leveraged to train multiple models at once. To configure the ParallelRunConfig you will need to determine the appropriate number of workers and nodes for your use case. The process_count_per_node is based off the number of cores of the compute VM. The node_count will determine the number of master nodes to use, increasing the node count will speed up the training process.\n",
+        "\n",
+        "| Property                           | Description|\n",
+        "| :---------------                   | :------------------- |\n",
+        "| **experiment**                     | The experiment used for training. |\n",
+        "| **train_data**                     | The file dataset to be used as input to the training run. |\n",
+        "| **node_count**                     | The number of compute nodes to be used for running the user script. We recommend to start with 3 and increase the node_count if the training time is taking too long. |\n",
+        "| **process_count_per_node**         | Process count per node, we recommend 2:1 ratio for number of cores: number of processes per node. eg. If node has 16 cores then configure 8 or less process count per node or optimal performance. |\n",
+        "| **train_pipeline_parameters**      | The set of configuration parameters defined in the previous section. |\n",
+        "\n",
+        "Calling this method will create a new aggregated dataset which is generated dynamically on pipeline execution."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.contrib.automl.pipeline.steps import AutoMLPipelineBuilder\n",
+        "\n",
+        "\n",
+        "training_pipeline_steps = AutoMLPipelineBuilder.get_many_models_train_steps(\n",
+        "    experiment=experiment,\n",
+        "    train_data=train_data,\n",
+        "    compute_target=compute_target,\n",
+        "    node_count=2,\n",
+        "    process_count_per_node=2,\n",
+        "    run_invocation_timeout=920,\n",
+        "    train_pipeline_parameters=mm_paramters,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.pipeline.core import Pipeline\n",
+        "\n",
+        "training_pipeline = Pipeline(ws, steps=training_pipeline_steps)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Submit the pipeline to run\n",
+        "Next we submit our pipeline to run. The whole training pipeline takes about 20 minutes using a STANDARD_DS12_V2 VM with our current ParallelRunConfig setting."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "training_run = experiment.submit(training_pipeline)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "training_run.wait_for_completion(show_output=False)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Check the run status, if training_run is in completed state, continue to next section. Otherwise, check the portal for failures."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## 4.0 Backtesting\n",
+        "Now that we selected the best AutoML model for each backtest fold, we will use these models to generate the forecasts and compare with the actuals."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Set up output dataset for inference data\n",
+        "Output of inference can be represented as [OutputFileDatasetConfig](https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.data.output_dataset_config.outputdatasetconfig?view=azure-ml-py) object and OutputFileDatasetConfig can be registered as a dataset. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.data import OutputFileDatasetConfig\n",
+        "\n",
+        "output_inference_data_ds = OutputFileDatasetConfig(\n",
+        "    name=\"many_models_inference_output\",\n",
+        "    destination=(dstore, \"backtesting/inference_data/\"),\n",
+        ").register_on_complete(name=\"backtesting_data_ds\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "For many models we need to provide the ManyModelsInferenceParameters object.\n",
+        "\n",
+        "#### ManyModelsInferenceParameters arguments\n",
+        "| Property                           | Description|\n",
+        "| :---------------                   | :------------------- |\n",
+        "| **partition_column_names**         | List of column names that identifies groups.                                           |\n",
+        "| **target_column_name**             | \\[Optional\\] Column name only if the inference dataset has the target. |\n",
+        "| **time_column_name**               | Column name only if it is timeseries. |\n",
+        "| **many_models_run_id**             | \\[Optional\\] Many models pipeline run id where models were trained. |\n",
+        "\n",
+        "#### get_many_models_batch_inference_steps arguments\n",
+        "| Property                           | Description|\n",
+        "| :---------------                   | :------------------- |\n",
+        "| **experiment**                     | The experiment used for inference run. |\n",
+        "| **inference_data**                 | The data to use for inferencing. It should be the same schema as used for training.\n",
+        "| **compute_target**                 | The compute target that runs the inference pipeline.|\n",
+        "| **node_count**                     | The number of compute nodes to be used for running the user script. We recommend to start with the number of cores per node (varies by compute sku). |\n",
+        "| **process_count_per_node**         | The number of processes per node.\n",
+        "| **train_run_id**                   | \\[Optional\\] The run id of the hierarchy training, by default it is the latest successful training many model run in the experiment. |\n",
+        "| **train_experiment_name**          | \\[Optional\\] The train experiment that contains the train pipeline. This one is only needed when the train pipeline is not in the same experiement as the inference pipeline. |\n",
+        "| **process_count_per_node**         | \\[Optional\\] The number of processes per node, by default it's 4. |"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.contrib.automl.pipeline.steps import AutoMLPipelineBuilder\n",
+        "from azureml.train.automl.runtime._many_models.many_models_parameters import (\n",
+        "    ManyModelsInferenceParameters,\n",
+        ")\n",
+        "\n",
+        "mm_parameters = ManyModelsInferenceParameters(\n",
+        "    partition_column_names=partition_column_names,\n",
+        "    time_column_name=TIME_COLNAME,\n",
+        "    target_column_name=TARGET_COLNAME,\n",
+        ")\n",
+        "\n",
+        "inference_steps = AutoMLPipelineBuilder.get_many_models_batch_inference_steps(\n",
+        "    experiment=experiment,\n",
+        "    inference_data=test_data,\n",
+        "    node_count=2,\n",
+        "    process_count_per_node=2,\n",
+        "    compute_target=compute_target,\n",
+        "    run_invocation_timeout=300,\n",
+        "    output_datastore=output_inference_data_ds,\n",
+        "    train_run_id=training_run.id,\n",
+        "    train_experiment_name=training_run.experiment.name,\n",
+        "    inference_pipeline_parameters=mm_parameters,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.pipeline.core import Pipeline\n",
+        "\n",
+        "inference_pipeline = Pipeline(ws, steps=inference_steps)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "inference_run = experiment.submit(inference_pipeline)\n",
+        "inference_run.wait_for_completion(show_output=False)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## 5.0 Retrieve results and calculate metrics\n",
+        "\n",
+        "The pipeline returns one file with the predictions for each times series ID and outputs the result to the forecasting_output Blob container. The details of the blob container is listed in 'forecasting_output.txt' under Outputs+logs. \n",
+        "\n",
+        "The next code snippet does the following:\n",
+        "1. Downloads the contents of the output folder that is passed in the parallel run step \n",
+        "2. Reads the parallel_run_step.txt file that has the predictions as pandas dataframe \n",
+        "3. Saves the table in csv format and \n",
+        "4. Displays the top 10 rows of the predictions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.contrib.automl.pipeline.steps.utilities import get_output_from_mm_pipeline\n",
+        "\n",
+        "forecasting_results_name = \"forecasting_results\"\n",
+        "forecasting_output_name = \"many_models_inference_output\"\n",
+        "forecast_file = get_output_from_mm_pipeline(\n",
+        "    inference_run, forecasting_results_name, forecasting_output_name\n",
+        ")\n",
+        "df = pd.read_csv(forecast_file, delimiter=\" \", header=None, parse_dates=[0])\n",
+        "df.columns = list(X_train.columns) + [\"predicted_level\"]\n",
+        "print(\n",
+        "    \"Prediction has \", df.shape[0], \" rows. Here the first 10 rows are being displayed.\"\n",
+        ")\n",
+        "# Save the scv file with header to read it in the next step.\n",
+        "df.rename(columns={TARGET_COLNAME: \"actual_level\"}, inplace=True)\n",
+        "df.to_csv(os.path.join(forecasting_results_name, \"forecast.csv\"), index=False)\n",
+        "df.head(10)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## View metrics\n",
+        "We will read in the obtained results and run the helper script, which will generate metrics and create the plots of predicted versus actual values."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from assets.score import calculate_scores_and_build_plots\n",
+        "\n",
+        "backtesting_results = \"backtesting_mm_results\"\n",
+        "os.makedirs(backtesting_results, exist_ok=True)\n",
+        "calculate_scores_and_build_plots(\n",
+        "    forecasting_results_name, backtesting_results, automl_settings\n",
+        ")\n",
+        "pd.DataFrame({\"File\": os.listdir(backtesting_results)})"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "The directory contains a set of files with results:\n",
+        "- forecast.csv contains forecasts for all backtest iterations. The backtest_iteration column contains iteration identifier with the last training date as a suffix\n",
+        "- scores.csv contains all metrics. If data set contains several time series, the metrics are given for all combinations of time series id and iterations, as well as scores for all iterations and time series ids, which are marked as \"all_sets\"\n",
+        "- plots_fcst_vs_actual.pdf contains the predictions vs forecast plots for each iteration and, eash time series is saved as separate plot.\n",
+        "\n",
+        "For demonstration purposes we will display the table of metrics for one of the time series with ID \"ts0\". We will create the utility function, which will build the table with metrics."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "def get_metrics_for_ts(all_metrics, ts):\n",
+        "    \"\"\"\n",
+        "    Get the metrics for the time series with ID ts and return it as pandas data frame.\n",
+        "\n",
+        "    :param all_metrics: The table with all the metrics.\n",
+        "    :param ts: The ID of a time series of interest.\n",
+        "    :return: The pandas DataFrame with metrics for one time series.\n",
+        "    \"\"\"\n",
+        "    results_df = None\n",
+        "    for ts_id, one_series in all_metrics.groupby(\"time_series_id\"):\n",
+        "        if not ts_id.startswith(ts):\n",
+        "            continue\n",
+        "        iteration = ts_id.split(\"|\")[-1]\n",
+        "        df = one_series[[\"metric_name\", \"metric\"]]\n",
+        "        df.rename({\"metric\": iteration}, axis=1, inplace=True)\n",
+        "        df.set_index(\"metric_name\", inplace=True)\n",
+        "        if results_df is None:\n",
+        "            results_df = df\n",
+        "        else:\n",
+        "            results_df = results_df.merge(\n",
+        "                df, how=\"inner\", left_index=True, right_index=True\n",
+        "            )\n",
+        "    results_df.sort_index(axis=1, inplace=True)\n",
+        "    return results_df\n",
+        "\n",
+        "\n",
+        "metrics_df = pd.read_csv(os.path.join(backtesting_results, \"scores.csv\"))\n",
+        "ts = \"ts_A\"\n",
+        "get_metrics_for_ts(metrics_df, ts)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Forecast vs actuals plots."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from IPython.display import IFrame\n",
+        "\n",
+        "IFrame(\"./backtesting_mm_results/plots_fcst_vs_actual.pdf\", width=800, height=300)"
+      ]
+    }
+  ],
+  "metadata": {
+    "authors": [
+      {
+        "name": "jialiu"
+      }
+    ],
+    "categories": [
+      "how-to-use-azureml",
+      "automated-machine-learning"
+    ],
+    "kernelspec": {
+      "display_name": "Python 3.6",
+      "language": "python",
+      "name": "python36"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.6.9"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 4
+}

how-to-use-azureml/automated-machine-learning/forecasting-backtest-many-models/auto-ml-forecasting-backtest-many-models.yml

@@ -0,0 +1,4 @@
+name: auto-ml-forecasting-backtest-many-models
+dependencies:
+- pip:
+  - azureml-sdk

how-to-use-azureml/automated-machine-learning/forecasting-backtest-single-model/assets/data_split.py

@@ -0,0 +1,45 @@
+import argparse
+import os
+
+import pandas as pd
+
+import azureml.train.automl.runtime._hts.hts_runtime_utilities as hru
+
+from azureml.core import Run
+from azureml.core.dataset import Dataset
+
+# Parse the arguments.
+args = {
+    "step_size": "--step-size",
+    "step_number": "--step-number",
+    "time_column_name": "--time-column-name",
+    "time_series_id_column_names": "--time-series-id-column-names",
+    "out_dir": "--output-dir",
+}
+parser = argparse.ArgumentParser("Parsing input arguments.")
+for argname, arg in args.items():
+    parser.add_argument(arg, dest=argname, required=True)
+parsed_args, _ = parser.parse_known_args()
+step_number = int(parsed_args.step_number)
+step_size = int(parsed_args.step_size)
+# Create the working dirrectory to store the temporary csv files.
+working_dir = parsed_args.out_dir
+os.makedirs(working_dir, exist_ok=True)
+# Set input and output
+script_run = Run.get_context()
+input_dataset = script_run.input_datasets["training_data"]
+X_train = input_dataset.to_pandas_dataframe()
+# Split the data.
+for i in range(step_number):
+    file_name = os.path.join(working_dir, "backtest_{}.csv".format(i))
+    if parsed_args.time_series_id_column_names:
+        dfs = []
+        for _, one_series in X_train.groupby([parsed_args.time_series_id_column_names]):
+            one_series = one_series.sort_values(
+                by=[parsed_args.time_column_name], inplace=False
+            )
+            dfs.append(one_series.iloc[: len(one_series) - step_size * i])
+        pd.concat(dfs, sort=False, ignore_index=True).to_csv(file_name, index=False)
+    else:
+        X_train.sort_values(by=[parsed_args.time_column_name], inplace=True)
+        X_train.iloc[: len(X_train) - step_size * i].to_csv(file_name, index=False)

how-to-use-azureml/automated-machine-learning/forecasting-backtest-single-model/assets/retrain_models.py

@@ -0,0 +1,173 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+"""The batch script needed for back testing of models using PRS."""
+import argparse
+import json
+import logging
+import os
+import pickle
+import re
+
+import pandas as pd
+
+from azureml.core.experiment import Experiment
+from azureml.core.model import Model
+from azureml.core.run import Run
+from azureml.automl.core.shared import constants
+from azureml.automl.runtime.shared.score import scoring
+from azureml.train.automl import AutoMLConfig
+
+RE_INVALID_SYMBOLS = re.compile(r"[:\s]")
+
+model_name = None
+target_column_name = None
+current_step_run = None
+output_dir = None
+
+logger = logging.getLogger(__name__)
+
+
+def _get_automl_settings():
+    with open(
+        os.path.join(
+            os.path.dirname(os.path.realpath(__file__)), "automl_settings.json"
+        )
+    ) as json_file:
+        return json.load(json_file)
+
+
+def init():
+    global model_name
+    global target_column_name
+    global output_dir
+    global automl_settings
+    global model_uid
+    logger.info("Initialization of the run.")
+    parser = argparse.ArgumentParser("Parsing input arguments.")
+    parser.add_argument("--output-dir", dest="out", required=True)
+    parser.add_argument("--model-name", dest="model", default=None)
+    parser.add_argument("--model-uid", dest="model_uid", default=None)
+
+    parsed_args, _ = parser.parse_known_args()
+    model_name = parsed_args.model
+    automl_settings = _get_automl_settings()
+    target_column_name = automl_settings.get("label_column_name")
+    output_dir = parsed_args.out
+    model_uid = parsed_args.model_uid
+    os.makedirs(output_dir, exist_ok=True)
+    os.environ["AUTOML_IGNORE_PACKAGE_VERSION_INCOMPATIBILITIES".lower()] = "True"
+
+
+def get_run():
+    global current_step_run
+    if current_step_run is None:
+        current_step_run = Run.get_context()
+    return current_step_run
+
+
+def run_backtest(data_input_name: str, file_name: str, experiment: Experiment):
+    """Re-train the model and return metrics."""
+    data_input = pd.read_csv(
+        data_input_name,
+        parse_dates=[automl_settings[constants.TimeSeries.TIME_COLUMN_NAME]],
+    )
+    print(data_input.head())
+    if not automl_settings.get(constants.TimeSeries.GRAIN_COLUMN_NAMES):
+        # There is no grains.
+        data_input.sort_values(
+            [automl_settings[constants.TimeSeries.TIME_COLUMN_NAME]], inplace=True
+        )
+        X_train = data_input.iloc[: -automl_settings["max_horizon"]]
+        y_train = X_train.pop(target_column_name).values
+        X_test = data_input.iloc[-automl_settings["max_horizon"] :]
+        y_test = X_test.pop(target_column_name).values
+    else:
+        # The data contain grains.
+        dfs_train = []
+        dfs_test = []
+        for _, one_series in data_input.groupby(
+            automl_settings.get(constants.TimeSeries.GRAIN_COLUMN_NAMES)
+        ):
+            one_series.sort_values(
+                [automl_settings[constants.TimeSeries.TIME_COLUMN_NAME]], inplace=True
+            )
+            dfs_train.append(one_series.iloc[: -automl_settings["max_horizon"]])
+            dfs_test.append(one_series.iloc[-automl_settings["max_horizon"] :])
+        X_train = pd.concat(dfs_train, sort=False, ignore_index=True)
+        y_train = X_train.pop(target_column_name).values
+        X_test = pd.concat(dfs_test, sort=False, ignore_index=True)
+        y_test = X_test.pop(target_column_name).values
+
+    last_training_date = str(
+        X_train[automl_settings[constants.TimeSeries.TIME_COLUMN_NAME]].max()
+    )
+
+    if file_name:
+        # If file name is provided, we will load model and retrain it on backtest data.
+        with open(file_name, "rb") as fp:
+            fitted_model = pickle.load(fp)
+        fitted_model.fit(X_train, y_train)
+    else:
+        # We will run the experiment and select the best model.
+        X_train[target_column_name] = y_train
+        automl_config = AutoMLConfig(training_data=X_train, **automl_settings)
+        automl_run = current_step_run.submit_child(automl_config, show_output=True)
+        best_run, fitted_model = automl_run.get_output()
+        # As we have generated models, we need to register them for the future use.
+        description = "Backtest model example"
+        tags = {"last_training_date": last_training_date, "experiment": experiment.name}
+        if model_uid:
+            tags["model_uid"] = model_uid
+        automl_run.register_model(
+            model_name=best_run.properties["model_name"],
+            description=description,
+            tags=tags,
+        )
+        print(f"The model {best_run.properties['model_name']} was registered.")
+
+    _, x_pred = fitted_model.forecast(X_test)
+    x_pred.reset_index(inplace=True, drop=False)
+    columns = [automl_settings[constants.TimeSeries.TIME_COLUMN_NAME]]
+    if automl_settings.get(constants.TimeSeries.GRAIN_COLUMN_NAMES):
+        # We know that fitted_model.grain_column_names is a list.
+        columns.extend(fitted_model.grain_column_names)
+    columns.append(constants.TimeSeriesInternal.DUMMY_TARGET_COLUMN)
+    # Remove featurized columns.
+    x_pred = x_pred[columns]
+    x_pred.rename(
+        {constants.TimeSeriesInternal.DUMMY_TARGET_COLUMN: "predicted_level"},
+        axis=1,
+        inplace=True,
+    )
+    x_pred["actual_level"] = y_test
+    x_pred["backtest_iteration"] = f"iteration_{last_training_date}"
+    date_safe = RE_INVALID_SYMBOLS.sub("_", last_training_date)
+    x_pred.to_csv(os.path.join(output_dir, f"iteration_{date_safe}.csv"), index=False)
+    return x_pred
+
+
+def run(input_files):
+    """Run the script"""
+    logger.info("Running mini batch.")
+    ws = get_run().experiment.workspace
+    file_name = None
+    if model_name:
+        models = Model.list(ws, name=model_name)
+        cloud_model = None
+        if models:
+            for one_mod in models:
+                if cloud_model is None or one_mod.version > cloud_model.version:
+                    logger.info(
+                        "Using existing model from the workspace. Model version: {}".format(
+                            one_mod.version
+                        )
+                    )
+                    cloud_model = one_mod
+        file_name = cloud_model.download(exist_ok=True)
+
+    forecasts = []
+    logger.info("Running backtest.")
+    for input_file in input_files:
+        forecasts.append(run_backtest(input_file, file_name, get_run().experiment))
+    return pd.concat(forecasts)

how-to-use-azureml/automated-machine-learning/forecasting-backtest-single-model/assets/score.py

@@ -0,0 +1,171 @@
+from typing import Any, Dict, Optional, List
+
+import argparse
+import json
+import os
+import re
+
+import numpy as np
+import pandas as pd
+
+from matplotlib import pyplot as plt
+from matplotlib.backends.backend_pdf import PdfPages
+
+from azureml.automl.core.shared import constants
+from azureml.automl.core.shared.types import GrainType
+from azureml.automl.runtime.shared.score import scoring
+
+GRAIN = "time_series_id"
+BACKTEST_ITER = "backtest_iteration"
+ACTUALS = "actual_level"
+PREDICTIONS = "predicted_level"
+ALL_GRAINS = "all_sets"
+
+FORECASTS_FILE = "forecast.csv"
+SCORES_FILE = "scores.csv"
+PLOTS_FILE = "plots_fcst_vs_actual.pdf"
+RE_INVALID_SYMBOLS = re.compile("[: ]")
+
+
+def _compute_metrics(df: pd.DataFrame, metrics: List[str]):
+    """
+    Compute metrics for one data frame.
+
+    :param df: The data frame which contains actual_level and predicted_level columns.
+    :return: The data frame with two columns - metric_name and metric.
+    """
+    scores = scoring.score_regression(
+        y_test=df[ACTUALS], y_pred=df[PREDICTIONS], metrics=metrics
+    )
+    metrics_df = pd.DataFrame(list(scores.items()), columns=["metric_name", "metric"])
+    metrics_df.sort_values(["metric_name"], inplace=True)
+    metrics_df.reset_index(drop=True, inplace=True)
+    return metrics_df
+
+
+def _format_grain_name(grain: GrainType) -> str:
+    """
+    Convert grain name to string.
+
+    :param grain: the grain name.
+    :return: the string representation of the given grain.
+    """
+    if not isinstance(grain, tuple) and not isinstance(grain, list):
+        return str(grain)
+    grain = list(map(str, grain))
+    return "|".join(grain)
+
+
+def compute_all_metrics(
+    fcst_df: pd.DataFrame,
+    ts_id_colnames: List[str],
+    metric_names: Optional[List[set]] = None,
+):
+    """
+    Calculate metrics per grain.
+
+    :param fcst_df: forecast data frame. Must contain 2 columns: 'actual_level' and 'predicted_level'
+    :param metric_names: (optional) the list of metric names to return
+    :param ts_id_colnames: (optional) list of grain column names
+    :return: dictionary of summary table for all tests and final decision on stationary vs nonstaionary
+    """
+    if not metric_names:
+        metric_names = list(constants.Metric.SCALAR_REGRESSION_SET)
+
+    if ts_id_colnames is None:
+        ts_id_colnames = []
+
+    metrics_list = []
+    if ts_id_colnames:
+        for grain, df in fcst_df.groupby(ts_id_colnames):
+            one_grain_metrics_df = _compute_metrics(df, metric_names)
+            one_grain_metrics_df[GRAIN] = _format_grain_name(grain)
+            metrics_list.append(one_grain_metrics_df)
+
+    # overall metrics
+    one_grain_metrics_df = _compute_metrics(fcst_df, metric_names)
+    one_grain_metrics_df[GRAIN] = ALL_GRAINS
+    metrics_list.append(one_grain_metrics_df)
+
+    # collect into a data frame
+    return pd.concat(metrics_list)
+
+
+def _draw_one_plot(
+    df: pd.DataFrame,
+    time_column_name: str,
+    grain_column_names: List[str],
+    pdf: PdfPages,
+) -> None:
+    """
+    Draw the single plot.
+
+    :param df: The data frame with the data to build plot.
+    :param time_column_name: The name of a time column.
+    :param grain_column_names: The name of grain columns.
+    :param pdf: The pdf backend used to render the plot.
+    """
+    fig, _ = plt.subplots(figsize=(20, 10))
+    df = df.set_index(time_column_name)
+    plt.plot(df[[ACTUALS, PREDICTIONS]])
+    plt.xticks(rotation=45)
+    iteration = df[BACKTEST_ITER].iloc[0]
+    if grain_column_names:
+        grain_name = [df[grain].iloc[0] for grain in grain_column_names]
+        plt.title(f"Time series ID: {_format_grain_name(grain_name)} {iteration}")
+    plt.legend(["actual", "forecast"])
+    plt.close(fig)
+    pdf.savefig(fig)
+
+
+def calculate_scores_and_build_plots(
+    input_dir: str, output_dir: str, automl_settings: Dict[str, Any]
+):
+    os.makedirs(output_dir, exist_ok=True)
+    grains = automl_settings.get(constants.TimeSeries.GRAIN_COLUMN_NAMES)
+    time_column_name = automl_settings.get(constants.TimeSeries.TIME_COLUMN_NAME)
+    if grains is None:
+        grains = []
+    if isinstance(grains, str):
+        grains = [grains]
+    while BACKTEST_ITER in grains:
+        grains.remove(BACKTEST_ITER)
+
+    dfs = []
+    for fle in os.listdir(input_dir):
+        file_path = os.path.join(input_dir, fle)
+        if os.path.isfile(file_path) and file_path.endswith(".csv"):
+            df_iter = pd.read_csv(file_path, parse_dates=[time_column_name])
+            for _, iteration in df_iter.groupby(BACKTEST_ITER):
+                dfs.append(iteration)
+    forecast_df = pd.concat(dfs, sort=False, ignore_index=True)
+    # To make sure plots are in order, sort the predictions by grain and iteration.
+    ts_index = grains + [BACKTEST_ITER]
+    forecast_df.sort_values(by=ts_index, inplace=True)
+    pdf = PdfPages(os.path.join(output_dir, PLOTS_FILE))
+    for _, one_forecast in forecast_df.groupby(ts_index):
+        _draw_one_plot(one_forecast, time_column_name, grains, pdf)
+    pdf.close()
+    forecast_df.to_csv(os.path.join(output_dir, FORECASTS_FILE), index=False)
+    # Remove np.NaN and np.inf from the prediction and actuals data.
+    forecast_df.replace([np.inf, -np.inf], np.nan, inplace=True)
+    forecast_df.dropna(subset=[ACTUALS, PREDICTIONS], inplace=True)
+    metrics = compute_all_metrics(forecast_df, grains + [BACKTEST_ITER])
+    metrics.to_csv(os.path.join(output_dir, SCORES_FILE), index=False)
+
+
+if __name__ == "__main__":
+    args = {"forecasts": "--forecasts", "scores_out": "--output-dir"}
+    parser = argparse.ArgumentParser("Parsing input arguments.")
+    for argname, arg in args.items():
+        parser.add_argument(arg, dest=argname, required=True)
+    parsed_args, _ = parser.parse_known_args()
+    input_dir = parsed_args.forecasts
+    output_dir = parsed_args.scores_out
+    with open(
+        os.path.join(
+            os.path.dirname(os.path.realpath(__file__)), "automl_settings.json"
+        )
+    ) as json_file:
+        automl_settings = json.load(json_file)
+    calculate_scores_and_build_plots(input_dir, output_dir, automl_settings)

how-to-use-azureml/automated-machine-learning/forecasting-backtest-single-model/auto-ml-forecasting-backtest-single-model.ipynb

@@ -0,0 +1,720 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
+        "\n",
+        "Licensed under the MIT License.\n",
+        "![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/automated-machine-learning/automl-forecasting-function.png)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# Automated MachineLearning\n",
+        "_**The model backtesting**_\n",
+        "\n",
+        "## Contents\n",
+        "1. [Introduction](#Introduction)\n",
+        "2. [Setup](#Setup)\n",
+        "3. [Data](#Data)\n",
+        "4. [Prepare remote compute and data.](#prepare_remote)\n",
+        "5. [Create the configuration for AutoML backtesting](#train)\n",
+        "6. [Backtest AutoML](#backtest_automl)\n",
+        "7. [View metrics](#Metrics)\n",
+        "8. [Backtest the best model](#backtest_model)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Introduction\n",
+        "Model backtesting is used to evaluate its performance on historical data. To do that we step back on the backtesting period by the data set several times and split the data to train and test sets. Then these data sets are used for training and evaluation of model.<br>\n",
+        "This notebook is intended to demonstrate backtesting on a single model, this is the best solution for small data sets with a few or one time series in it. For scenarios where we would like to choose the best AutoML model for every backtest iteration, please see [AutoML Forecasting Backtest Many Models Example](../forecasting-backtest-many-models/auto-ml-forecasting-backtest-many-models.ipynb) notebook.\n",
+        "![Backtesting](Backtesting.png)\n",
+        "This notebook demonstrates two ways of backtesting:\n",
+        "- AutoML backtesting: we will train separate AutoML models for historical data\n",
+        "- Model backtesting: from the first run we will select the best model trained on the most recent data, retrain it on the past data and evaluate."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "import numpy as np\n",
+        "import pandas as pd\n",
+        "import shutil\n",
+        "\n",
+        "import azureml.core\n",
+        "from azureml.core import Experiment, Model, Workspace"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "This notebook is compatible with Azure ML SDK version 1.35.1 or later."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "As part of the setup you have already created a <b>Workspace</b>."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "ws = Workspace.from_config()\n",
+        "\n",
+        "output = {}\n",
+        "output[\"Subscription ID\"] = ws.subscription_id\n",
+        "output[\"Workspace\"] = ws.name\n",
+        "output[\"SKU\"] = ws.sku\n",
+        "output[\"Resource Group\"] = ws.resource_group\n",
+        "output[\"Location\"] = ws.location\n",
+        "output[\"SDK Version\"] = azureml.core.VERSION\n",
+        "pd.set_option(\"display.max_colwidth\", None)\n",
+        "outputDf = pd.DataFrame(data=output, index=[\"\"])\n",
+        "outputDf.T"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Data\n",
+        "For the demonstration purposes we will simulate one year of daily data. To do this we need to specify the following parameters: time column name, time series ID column names and label column name. Our intention is to forecast for two weeks ahead."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "TIME_COLUMN_NAME = \"date\"\n",
+        "TIME_SERIES_ID_COLUMN_NAMES = \"time_series_id\"\n",
+        "LABEL_COLUMN_NAME = \"y\"\n",
+        "FORECAST_HORIZON = 14\n",
+        "FREQUENCY = \"D\"\n",
+        "\n",
+        "\n",
+        "def simulate_timeseries_data(\n",
+        "    train_len: int,\n",
+        "    test_len: int,\n",
+        "    time_column_name: str,\n",
+        "    target_column_name: str,\n",
+        "    time_series_id_column_name: str,\n",
+        "    time_series_number: int = 1,\n",
+        "    freq: str = \"H\",\n",
+        "):\n",
+        "    \"\"\"\n",
+        "    Return the time series of designed length.\n",
+        "\n",
+        "    :param train_len: The length of training data (one series).\n",
+        "    :type train_len: int\n",
+        "    :param test_len: The length of testing data (one series).\n",
+        "    :type test_len: int\n",
+        "    :param time_column_name: The desired name of a time column.\n",
+        "    :type time_column_name: str\n",
+        "    :param time_series_number: The number of time series in the data set.\n",
+        "    :type time_series_number: int\n",
+        "    :param freq: The frequency string representing pandas offset.\n",
+        "                 see https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html\n",
+        "    :type freq: str\n",
+        "    :returns: the tuple of train and test data sets.\n",
+        "    :rtype: tuple\n",
+        "\n",
+        "    \"\"\"\n",
+        "    data_train = []  # type: List[pd.DataFrame]\n",
+        "    data_test = []  # type: List[pd.DataFrame]\n",
+        "    data_length = train_len + test_len\n",
+        "    for i in range(time_series_number):\n",
+        "        X = pd.DataFrame(\n",
+        "            {\n",
+        "                time_column_name: pd.date_range(\n",
+        "                    start=\"2000-01-01\", periods=data_length, freq=freq\n",
+        "                ),\n",
+        "                target_column_name: np.arange(data_length).astype(float)\n",
+        "                + np.random.rand(data_length)\n",
+        "                + i * 5,\n",
+        "                \"ext_predictor\": np.asarray(range(42, 42 + data_length)),\n",
+        "                time_series_id_column_name: np.repeat(\"ts{}\".format(i), data_length),\n",
+        "            }\n",
+        "        )\n",
+        "        data_train.append(X[:train_len])\n",
+        "        data_test.append(X[train_len:])\n",
+        "    train = pd.concat(data_train)\n",
+        "    label_train = train.pop(target_column_name).values\n",
+        "    test = pd.concat(data_test)\n",
+        "    label_test = test.pop(target_column_name).values\n",
+        "    return train, label_train, test, label_test\n",
+        "\n",
+        "\n",
+        "n_test_periods = FORECAST_HORIZON\n",
+        "n_train_periods = 365\n",
+        "X_train, y_train, X_test, y_test = simulate_timeseries_data(\n",
+        "    train_len=n_train_periods,\n",
+        "    test_len=n_test_periods,\n",
+        "    time_column_name=TIME_COLUMN_NAME,\n",
+        "    target_column_name=LABEL_COLUMN_NAME,\n",
+        "    time_series_id_column_name=TIME_SERIES_ID_COLUMN_NAMES,\n",
+        "    time_series_number=2,\n",
+        "    freq=FREQUENCY,\n",
+        ")\n",
+        "X_train[LABEL_COLUMN_NAME] = y_train"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Let's see what the training data looks like."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "X_train.tail()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Prepare remote compute and data. <a id=\"prepare_remote\"></a>\n",
+        "The [Machine Learning service workspace](https://docs.microsoft.com/en-us/azure/machine-learning/service/concept-workspace), is paired with the storage account, which contains the default data store. We will use it to upload the artificial data and create [tabular dataset](https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.data.tabulardataset?view=azure-ml-py) for training. A tabular dataset defines a series of lazily-evaluated, immutable operations to load data from the data source into tabular representation."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.data.dataset_factory import TabularDatasetFactory\n",
+        "\n",
+        "ds = ws.get_default_datastore()\n",
+        "# Upload saved data to the default data store.\n",
+        "train_data = TabularDatasetFactory.register_pandas_dataframe(\n",
+        "    X_train, target=(ds, \"data\"), name=\"data_backtest\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "You will need to create a compute target for backtesting. In this [tutorial](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-set-up-training-targets#amlcompute), you create AmlCompute as your training compute resource.\n",
+        "\n",
+        "> Note that if you have an AzureML Data Scientist role, you will not have permission to create compute resources. Talk to your workspace or IT admin to create the compute targets described in this section, if they do not already exist."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.compute import ComputeTarget, AmlCompute\n",
+        "from azureml.core.compute_target import ComputeTargetException\n",
+        "\n",
+        "# Choose a name for your CPU cluster\n",
+        "amlcompute_cluster_name = \"backtest-cluster\"\n",
+        "\n",
+        "# Verify that cluster does not exist already\n",
+        "try:\n",
+        "    compute_target = ComputeTarget(workspace=ws, name=amlcompute_cluster_name)\n",
+        "    print(\"Found existing cluster, use it.\")\n",
+        "except ComputeTargetException:\n",
+        "    compute_config = AmlCompute.provisioning_configuration(\n",
+        "        vm_size=\"STANDARD_DS12_V2\", max_nodes=6\n",
+        "    )\n",
+        "    compute_target = ComputeTarget.create(ws, amlcompute_cluster_name, compute_config)\n",
+        "\n",
+        "compute_target.wait_for_completion(show_output=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Create the configuration for AutoML backtesting <a id=\"train\"></a>\n",
+        "\n",
+        "This dictionary defines the AutoML and many models settings. For this forecasting task we need to define several settings including the name of the time column, the maximum forecast horizon, and the partition column name definition.\n",
+        "\n",
+        "| Property                           | Description|\n",
+        "| :---------------                   | :------------------- |\n",
+        "| **task**                           | forecasting |\n",
+        "| **primary_metric**                 | This is the metric that you want to optimize.<br> Forecasting supports the following primary metrics <br><i>normalized_root_mean_squared_error</i><br><i>normalized_mean_absolute_error</i> |\n",
+        "| **iteration_timeout_minutes**      | Maximum amount of time in minutes that the model can train. This is optional but provides customers with greater control on exit criteria. |\n",
+        "| **iterations**                     | Number of models to train. This is optional but provides customers with greater control on exit criteria. |\n",
+        "| **experiment_timeout_hours**       | Maximum amount of time in hours that the experiment can take before it terminates. This is optional but provides customers with greater control on exit criteria. |\n",
+        "| **label_column_name**              | The name of the label column. |\n",
+        "| **max_horizon**               | The forecast horizon is how many periods forward you would like to forecast. This integer horizon is in units of the timeseries frequency (e.g. daily, weekly). Periods are inferred from your data. |\n",
+        "| **n_cross_validations**            | Number of cross validation splits. Rolling Origin Validation is used to split time-series in a temporally consistent way. |\n",
+        "| **time_column_name**               | The name of your time column. |\n",
+        "| **grain_column_names**     | The column names used to uniquely identify timeseries in data that has multiple rows with the same timestamp. |"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "automl_settings = {\n",
+        "    \"task\": \"forecasting\",\n",
+        "    \"primary_metric\": \"normalized_root_mean_squared_error\",\n",
+        "    \"iteration_timeout_minutes\": 10,  # This needs to be changed based on the dataset. We ask customer to explore how long training is taking before settings this value\n",
+        "    \"iterations\": 15,\n",
+        "    \"experiment_timeout_hours\": 1,  # This also needs to be changed based on the dataset. For larger data set this number needs to be bigger.\n",
+        "    \"label_column_name\": LABEL_COLUMN_NAME,\n",
+        "    \"n_cross_validations\": 3,\n",
+        "    \"time_column_name\": TIME_COLUMN_NAME,\n",
+        "    \"max_horizon\": FORECAST_HORIZON,\n",
+        "    \"track_child_runs\": False,\n",
+        "    \"grain_column_names\": TIME_SERIES_ID_COLUMN_NAMES,\n",
+        "}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Backtest AutoML <a id=\"backtest_automl\"></a>\n",
+        "First we set backtesting parameters: we will step back by 30 days and will make 5 such steps; for each step we will forecast for next two weeks."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# The number of periods to step back on each backtest iteration.\n",
+        "BACKTESTING_PERIOD = 30\n",
+        "# The number of times we will back test the model.\n",
+        "NUMBER_OF_BACKTESTS = 5"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "To train AutoML on backtesting folds we will use the [Azure Machine Learning pipeline](https://docs.microsoft.com/en-us/azure/machine-learning/concept-ml-pipelines). It will generate backtest folds, then train model for each of them and calculate the accuracy metrics. To run pipeline, you also need to create an <b>Experiment</b>. An Experiment corresponds to a prediction problem you are trying to solve (here, it is a forecasting), while a Run corresponds to a specific approach to the problem."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from uuid import uuid1\n",
+        "\n",
+        "from pipeline_helper import get_backtest_pipeline\n",
+        "\n",
+        "pipeline_exp = Experiment(ws, \"automl-backtesting\")\n",
+        "\n",
+        "# We will create the unique identifier to mark our models.\n",
+        "model_uid = str(uuid1())\n",
+        "\n",
+        "pipeline = get_backtest_pipeline(\n",
+        "    experiment=pipeline_exp,\n",
+        "    dataset=train_data,\n",
+        "    # The STANDARD_DS12_V2 has 4 vCPU per node, we will set 2 process per node to be safe.\n",
+        "    process_per_node=2,\n",
+        "    # The maximum number of nodes for our compute is 6.\n",
+        "    node_count=6,\n",
+        "    compute_target=compute_target,\n",
+        "    automl_settings=automl_settings,\n",
+        "    step_size=BACKTESTING_PERIOD,\n",
+        "    step_number=NUMBER_OF_BACKTESTS,\n",
+        "    model_uid=model_uid,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Run the pipeline and wait for results."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "pipeline_run = pipeline_exp.submit(pipeline)\n",
+        "pipeline_run.wait_for_completion(show_output=False)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "After the run is complete, we can download the results. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "metrics_output = pipeline_run.get_pipeline_output(\"results\")\n",
+        "metrics_output.download(\"backtest_metrics\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## View metrics<a id=\"Metrics\"></a>\n",
+        "To distinguish these metrics from the model backtest, which we will obtain in the next section, we will move the directory with metrics out of the backtest_metrics and will remove the parent folder. We will create the utility function for that."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "def copy_scoring_directory(new_name):\n",
+        "    scores_path = os.path.join(\"backtest_metrics\", \"azureml\")\n",
+        "    directory_list = [os.path.join(scores_path, d) for d in os.listdir(scores_path)]\n",
+        "    latest_file = max(directory_list, key=os.path.getctime)\n",
+        "    print(\n",
+        "        f\"The output directory {latest_file} was created on {pd.Timestamp(os.path.getctime(latest_file), unit='s')} GMT.\"\n",
+        "    )\n",
+        "    shutil.move(os.path.join(latest_file, \"results\"), new_name)\n",
+        "    shutil.rmtree(\"backtest_metrics\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Move the directory and list its contents."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "copy_scoring_directory(\"automl_backtest\")\n",
+        "pd.DataFrame({\"File\": os.listdir(\"automl_backtest\")})"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "The directory contains a set of files with results:\n",
+        "- forecast.csv contains forecasts for all backtest iterations. The backtest_iteration column contains iteration identifier with the last training date as a suffix\n",
+        "- scores.csv contains all metrics. If data set contains several time series, the metrics are given for all combinations of time series id and iterations, as well as scores for all iterations and time series id are marked as \"all_sets\"\n",
+        "- plots_fcst_vs_actual.pdf contains the predictions vs forecast plots for each iteration and time series.\n",
+        "\n",
+        "For demonstration purposes we will display the table of metrics for one of the time series with ID \"ts0\". Again, we will create the utility function, which will be re used in model backtesting."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "def get_metrics_for_ts(all_metrics, ts):\n",
+        "    \"\"\"\n",
+        "    Get the metrics for the time series with ID ts and return it as pandas data frame.\n",
+        "\n",
+        "    :param all_metrics: The table with all the metrics.\n",
+        "    :param ts: The ID of a time series of interest.\n",
+        "    :return: The pandas DataFrame with metrics for one time series.\n",
+        "    \"\"\"\n",
+        "    results_df = None\n",
+        "    for ts_id, one_series in all_metrics.groupby(\"time_series_id\"):\n",
+        "        if not ts_id.startswith(ts):\n",
+        "            continue\n",
+        "        iteration = ts_id.split(\"|\")[-1]\n",
+        "        df = one_series[[\"metric_name\", \"metric\"]]\n",
+        "        df.rename({\"metric\": iteration}, axis=1, inplace=True)\n",
+        "        df.set_index(\"metric_name\", inplace=True)\n",
+        "        if results_df is None:\n",
+        "            results_df = df\n",
+        "        else:\n",
+        "            results_df = results_df.merge(\n",
+        "                df, how=\"inner\", left_index=True, right_index=True\n",
+        "            )\n",
+        "    results_df.sort_index(axis=1, inplace=True)\n",
+        "    return results_df\n",
+        "\n",
+        "\n",
+        "metrics_df = pd.read_csv(os.path.join(\"automl_backtest\", \"scores.csv\"))\n",
+        "ts_id = \"ts0\"\n",
+        "get_metrics_for_ts(metrics_df, ts_id)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Forecast vs actuals plots."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from IPython.display import IFrame\n",
+        "\n",
+        "IFrame(\"./automl_backtest/plots_fcst_vs_actual.pdf\", width=800, height=300)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# <font color='blue'>Backtest the best model</font> <a id=\"backtest_model\"></a>\n",
+        "\n",
+        "For model backtesting we will use the same parameters we used to backtest AutoML. All the models, we have obtained in the previous run were registered in our workspace. To identify the model, each was assigned a tag with the last trainig date."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "model_list = Model.list(ws, tags={\"experiment\": \"automl-backtesting\"})\n",
+        "model_data = {\"name\": [], \"last_training_date\": []}\n",
+        "for model in model_list:\n",
+        "    if (\n",
+        "        \"last_training_date\" not in model.tags\n",
+        "        or \"model_uid\" not in model.tags\n",
+        "        or model.tags[\"model_uid\"] != model_uid\n",
+        "    ):\n",
+        "        continue\n",
+        "    model_data[\"name\"].append(model.name)\n",
+        "    model_data[\"last_training_date\"].append(\n",
+        "        pd.Timestamp(model.tags[\"last_training_date\"])\n",
+        "    )\n",
+        "df_models = pd.DataFrame(model_data)\n",
+        "df_models.sort_values([\"last_training_date\"], inplace=True)\n",
+        "df_models.reset_index(inplace=True, drop=True)\n",
+        "df_models"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "We will backtest the model trained on the most recet data."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "model_name = df_models[\"name\"].iloc[-1]\n",
+        "model_name"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Retrain the models.\n",
+        "Assemble the pipeline, which will retrain the best model from AutoML run on historical data."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "pipeline_exp = Experiment(ws, \"model-backtesting\")\n",
+        "\n",
+        "pipeline = get_backtest_pipeline(\n",
+        "    experiment=pipeline_exp,\n",
+        "    dataset=train_data,\n",
+        "    # The STANDARD_DS12_V2 has 4 vCPU per node, we will set 2 process per node to be safe.\n",
+        "    process_per_node=2,\n",
+        "    # The maximum number of nodes for our compute is 6.\n",
+        "    node_count=6,\n",
+        "    compute_target=compute_target,\n",
+        "    automl_settings=automl_settings,\n",
+        "    step_size=BACKTESTING_PERIOD,\n",
+        "    step_number=NUMBER_OF_BACKTESTS,\n",
+        "    model_name=model_name,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Launch the backtesting pipeline."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "pipeline_run = pipeline_exp.submit(pipeline)\n",
+        "pipeline_run.wait_for_completion(show_output=False)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "The metrics are stored in the pipeline output named \"score\". The next code will download the table with metrics."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "metrics_output = pipeline_run.get_pipeline_output(\"results\")\n",
+        "metrics_output.download(\"backtest_metrics\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Again, we will copy the data files from the downloaded directory, but in this case we will call the folder \"model_backtest\"; it will contain the same files as the one for AutoML backtesting."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "copy_scoring_directory(\"model_backtest\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Finally, we will display the metrics."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "model_metrics_df = pd.read_csv(os.path.join(\"model_backtest\", \"scores.csv\"))\n",
+        "get_metrics_for_ts(model_metrics_df, ts_id)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Forecast vs actuals plots."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from IPython.display import IFrame\n",
+        "\n",
+        "IFrame(\"./model_backtest/plots_fcst_vs_actual.pdf\", width=800, height=300)"
+      ]
+    }
+  ],
+  "metadata": {
+    "authors": [
+      {
+        "name": "jialiu"
+      }
+    ],
+    "category": "tutorial",
+    "compute": [
+      "Remote"
+    ],
+    "datasets": [
+      "None"
+    ],
+    "deployment": [
+      "None"
+    ],
+    "exclude_from_index": false,
+    "framework": [
+      "Azure ML AutoML"
+    ],
+    "kernelspec": {
+      "display_name": "Python 3.6",
+      "language": "python",
+      "name": "python36"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.6.9"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 4
+}

how-to-use-azureml/automated-machine-learning/forecasting-backtest-single-model/auto-ml-forecasting-backtest-single-model.yml

@@ -0,0 +1,4 @@
+name: auto-ml-forecasting-backtest-single-model
+dependencies:
+- pip:
+  - azureml-sdk

how-to-use-azureml/automated-machine-learning/forecasting-backtest-single-model/pipeline_helper.py

@@ -0,0 +1,166 @@
+from typing import Any, Dict, Optional
+
+import os
+
+import azureml.train.automl.runtime._hts.hts_runtime_utilities as hru
+
+from azureml._restclient.jasmine_client import JasmineClient
+from azureml.contrib.automl.pipeline.steps import utilities
+from azureml.core import RunConfiguration
+from azureml.core.compute import ComputeTarget
+from azureml.core.experiment import Experiment
+from azureml.data import LinkTabularOutputDatasetConfig, TabularDataset
+from azureml.pipeline.core import Pipeline, PipelineData, PipelineParameter
+from azureml.pipeline.steps import ParallelRunConfig, ParallelRunStep, PythonScriptStep
+from azureml.train.automl.constants import Scenarios
+from azureml.data.dataset_consumption_config import DatasetConsumptionConfig
+
+
+PROJECT_FOLDER = "assets"
+SETTINGS_FILE = "automl_settings.json"
+
+
+def get_backtest_pipeline(
+    experiment: Experiment,
+    dataset: TabularDataset,
+    process_per_node: int,
+    node_count: int,
+    compute_target: ComputeTarget,
+    automl_settings: Dict[str, Any],
+    step_size: int,
+    step_number: int,
+    model_name: Optional[str] = None,
+    model_uid: Optional[str] = None,
+) -> Pipeline:
+    """
+    :param experiment: The experiment used to run the pipeline.
+    :param dataset: Tabular data set to be used for model training.
+    :param process_per_node: The number of processes per node. Generally it should be the number of cores
+                             on the node divided by two.
+    :param node_count: The number of nodes to be used.
+    :param compute_target: The compute target to be used to run the pipeline.
+    :param model_name: The name of a model to be back tested.
+    :param automl_settings: The dictionary with automl settings.
+    :param step_size: The number of periods to step back in backtesting.
+    :param step_number: The number of backtesting iterations.
+    :param model_uid: The uid to mark models from this run of the experiment.
+    :return: The pipeline to be used for model retraining.
+             **Note:** The output will be uploaded in the pipeline output
+             called 'score'.
+    """
+    jasmine_client = JasmineClient(
+        service_context=experiment.workspace.service_context,
+        experiment_name=experiment.name,
+        experiment_id=experiment.id,
+    )
+    env = jasmine_client.get_curated_environment(
+        scenario=Scenarios.AUTOML,
+        enable_dnn=False,
+        enable_gpu=False,
+        compute=compute_target,
+        compute_sku=experiment.workspace.compute_targets.get(
+            compute_target.name
+        ).vm_size,
+    )
+    data_results = PipelineData(
+        name="results", datastore=None, pipeline_output_name="results"
+    )
+    ############################################################
+    # Split the data set using python script.
+    ############################################################
+    run_config = RunConfiguration()
+    run_config.docker.use_docker = True
+    run_config.environment = env
+
+    split_data = PipelineData(name="split_data_output", datastore=None).as_dataset()
+    split_step = PythonScriptStep(
+        name="split_data_for_backtest",
+        script_name="data_split.py",
+        inputs=[dataset.as_named_input("training_data")],
+        outputs=[split_data],
+        source_directory=PROJECT_FOLDER,
+        arguments=[
+            "--step-size",
+            step_size,
+            "--step-number",
+            step_number,
+            "--time-column-name",
+            automl_settings.get("time_column_name"),
+            "--time-series-id-column-names",
+            automl_settings.get("grain_column_names"),
+            "--output-dir",
+            split_data,
+        ],
+        runconfig=run_config,
+        compute_target=compute_target,
+        allow_reuse=False,
+    )
+    ############################################################
+    # We will do the backtest the parallel run step.
+    ############################################################
+    settings_path = os.path.join(PROJECT_FOLDER, SETTINGS_FILE)
+    hru.dump_object_to_json(automl_settings, settings_path)
+    mini_batch_size = PipelineParameter(name="batch_size_param", default_value=str(1))
+    back_test_config = ParallelRunConfig(
+        source_directory=PROJECT_FOLDER,
+        entry_script="retrain_models.py",
+        mini_batch_size=mini_batch_size,
+        error_threshold=-1,
+        output_action="append_row",
+        append_row_file_name="outputs.txt",
+        compute_target=compute_target,
+        environment=env,
+        process_count_per_node=process_per_node,
+        run_invocation_timeout=3600,
+        node_count=node_count,
+    )
+    forecasts = PipelineData(name="forecasts", datastore=None)
+    if model_name:
+        parallel_step_name = "{}-backtest".format(model_name.replace("_", "-"))
+    else:
+        parallel_step_name = "AutoML-backtest"
+
+    prs_args = [
+        "--target_column_name",
+        automl_settings.get("label_column_name"),
+        "--output-dir",
+        forecasts,
+    ]
+    if model_name is not None:
+        prs_args.append("--model-name")
+        prs_args.append(model_name)
+    if model_uid is not None:
+        prs_args.append("--model-uid")
+        prs_args.append(model_uid)
+    backtest_prs = ParallelRunStep(
+        name=parallel_step_name,
+        parallel_run_config=back_test_config,
+        arguments=prs_args,
+        inputs=[split_data],
+        output=forecasts,
+        allow_reuse=False,
+    )
+    ############################################################
+    # Then we collect the output and return it as scores output.
+    ############################################################
+    collection_step = PythonScriptStep(
+        name="score",
+        script_name="score.py",
+        inputs=[forecasts.as_mount()],
+        outputs=[data_results],
+        source_directory=PROJECT_FOLDER,
+        arguments=[
+            "--forecasts",
+            forecasts,
+            "--output-dir",
+            data_results,
+        ],
+        runconfig=run_config,
+        compute_target=compute_target,
+        allow_reuse=False,
+    )
+    # Build and return the pipeline.
+    return Pipeline(
+        workspace=experiment.workspace,
+        steps=[split_step, backtest_prs, collection_step],
+    )

how-to-use-azureml/automated-machine-learning/forecasting-beer-remote/Beer_no_valid_split_test.csv

@@ -1,20 +0,0 @@
-DATE,grain,BeerProduction
-2017-01-01,grain,9049
-2017-02-01,grain,10458
-2017-03-01,grain,12489
-2017-04-01,grain,11499
-2017-05-01,grain,13553
-2017-06-01,grain,14740
-2017-07-01,grain,11424
-2017-08-01,grain,13412
-2017-09-01,grain,11917
-2017-10-01,grain,12721
-2017-11-01,grain,13272
-2017-12-01,grain,14278
-2018-01-01,grain,9572
-2018-02-01,grain,10423
-2018-03-01,grain,12667
-2018-04-01,grain,11904
-2018-05-01,grain,14120
-2018-06-01,grain,14565
-2018-07-01,grain,12622

how-to-use-azureml/automated-machine-learning/forecasting-beer-remote/Beer_no_valid_split_train.csv

@@ -1,301 +0,0 @@
-DATE,grain,BeerProduction
-1992-01-01,grain,3459
-1992-02-01,grain,3458
-1992-03-01,grain,4002
-1992-04-01,grain,4564
-1992-05-01,grain,4221
-1992-06-01,grain,4529
-1992-07-01,grain,4466
-1992-08-01,grain,4137
-1992-09-01,grain,4126
-1992-10-01,grain,4259
-1992-11-01,grain,4240
-1992-12-01,grain,4936
-1993-01-01,grain,3031
-1993-02-01,grain,3261
-1993-03-01,grain,4160
-1993-04-01,grain,4377
-1993-05-01,grain,4307
-1993-06-01,grain,4696
-1993-07-01,grain,4458
-1993-08-01,grain,4457
-1993-09-01,grain,4364
-1993-10-01,grain,4236
-1993-11-01,grain,4500
-1993-12-01,grain,4974
-1994-01-01,grain,3075
-1994-02-01,grain,3377
-1994-03-01,grain,4443
-1994-04-01,grain,4261
-1994-05-01,grain,4460
-1994-06-01,grain,4985
-1994-07-01,grain,4324
-1994-08-01,grain,4719
-1994-09-01,grain,4374
-1994-10-01,grain,4248
-1994-11-01,grain,4784
-1994-12-01,grain,4971
-1995-01-01,grain,3370
-1995-02-01,grain,3484
-1995-03-01,grain,4269
-1995-04-01,grain,3994
-1995-05-01,grain,4715
-1995-06-01,grain,4974
-1995-07-01,grain,4223
-1995-08-01,grain,5000
-1995-09-01,grain,4235
-1995-10-01,grain,4554
-1995-11-01,grain,4851
-1995-12-01,grain,4826
-1996-01-01,grain,3699
-1996-02-01,grain,3983
-1996-03-01,grain,4262
-1996-04-01,grain,4619
-1996-05-01,grain,5219
-1996-06-01,grain,4836
-1996-07-01,grain,4941
-1996-08-01,grain,5062
-1996-09-01,grain,4365
-1996-10-01,grain,5012
-1996-11-01,grain,4850
-1996-12-01,grain,5097
-1997-01-01,grain,3758
-1997-02-01,grain,3825
-1997-03-01,grain,4454
-1997-04-01,grain,4635
-1997-05-01,grain,5210
-1997-06-01,grain,5057
-1997-07-01,grain,5231
-1997-08-01,grain,5034
-1997-09-01,grain,4970
-1997-10-01,grain,5342
-1997-11-01,grain,4831
-1997-12-01,grain,5965
-1998-01-01,grain,3796
-1998-02-01,grain,4019
-1998-03-01,grain,4898
-1998-04-01,grain,5090
-1998-05-01,grain,5237
-1998-06-01,grain,5447
-1998-07-01,grain,5435
-1998-08-01,grain,5107
-1998-09-01,grain,5515
-1998-10-01,grain,5583
-1998-11-01,grain,5346
-1998-12-01,grain,6286
-1999-01-01,grain,4032
-1999-02-01,grain,4435
-1999-03-01,grain,5479
-1999-04-01,grain,5483
-1999-05-01,grain,5587
-1999-06-01,grain,6176
-1999-07-01,grain,5621
-1999-08-01,grain,5889
-1999-09-01,grain,5828
-1999-10-01,grain,5849
-1999-11-01,grain,6180
-1999-12-01,grain,6771
-2000-01-01,grain,4243
-2000-02-01,grain,4952
-2000-03-01,grain,6008
-2000-04-01,grain,5353
-2000-05-01,grain,6435
-2000-06-01,grain,6673
-2000-07-01,grain,5636
-2000-08-01,grain,6630
-2000-09-01,grain,5887
-2000-10-01,grain,6322
-2000-11-01,grain,6520
-2000-12-01,grain,6678
-2001-01-01,grain,5082
-2001-02-01,grain,5216
-2001-03-01,grain,5893
-2001-04-01,grain,5894
-2001-05-01,grain,6799
-2001-06-01,grain,6667
-2001-07-01,grain,6374
-2001-08-01,grain,6840
-2001-09-01,grain,5575
-2001-10-01,grain,6545
-2001-11-01,grain,6789
-2001-12-01,grain,7180
-2002-01-01,grain,5117
-2002-02-01,grain,5442
-2002-03-01,grain,6337
-2002-04-01,grain,6525
-2002-05-01,grain,7216
-2002-06-01,grain,6761
-2002-07-01,grain,6958
-2002-08-01,grain,7070
-2002-09-01,grain,6148
-2002-10-01,grain,6924
-2002-11-01,grain,6716
-2002-12-01,grain,7975
-2003-01-01,grain,5326
-2003-02-01,grain,5609
-2003-03-01,grain,6414
-2003-04-01,grain,6741
-2003-05-01,grain,7144
-2003-06-01,grain,7133
-2003-07-01,grain,7568
-2003-08-01,grain,7266
-2003-09-01,grain,6634
-2003-10-01,grain,7626
-2003-11-01,grain,6843
-2003-12-01,grain,8540
-2004-01-01,grain,5629
-2004-02-01,grain,5898
-2004-03-01,grain,7045
-2004-04-01,grain,7094
-2004-05-01,grain,7333
-2004-06-01,grain,7918
-2004-07-01,grain,7289
-2004-08-01,grain,7396
-2004-09-01,grain,7259
-2004-10-01,grain,7268
-2004-11-01,grain,7731
-2004-12-01,grain,9058
-2005-01-01,grain,5557
-2005-02-01,grain,6237
-2005-03-01,grain,7723
-2005-04-01,grain,7262
-2005-05-01,grain,8241
-2005-06-01,grain,8757
-2005-07-01,grain,7352
-2005-08-01,grain,8496
-2005-09-01,grain,7741
-2005-10-01,grain,7710
-2005-11-01,grain,8247
-2005-12-01,grain,8902
-2006-01-01,grain,6066
-2006-02-01,grain,6590
-2006-03-01,grain,7923
-2006-04-01,grain,7335
-2006-05-01,grain,8843
-2006-06-01,grain,9327
-2006-07-01,grain,7792
-2006-08-01,grain,9156
-2006-09-01,grain,8037
-2006-10-01,grain,8640
-2006-11-01,grain,9128
-2006-12-01,grain,9545
-2007-01-01,grain,6627
-2007-02-01,grain,6743
-2007-03-01,grain,8195
-2007-04-01,grain,7828
-2007-05-01,grain,9570
-2007-06-01,grain,9484
-2007-07-01,grain,8608
-2007-08-01,grain,9543
-2007-09-01,grain,8123
-2007-10-01,grain,9649
-2007-11-01,grain,9390
-2007-12-01,grain,10065
-2008-01-01,grain,7093
-2008-02-01,grain,7483
-2008-03-01,grain,8365
-2008-04-01,grain,8895
-2008-05-01,grain,9794
-2008-06-01,grain,9977
-2008-07-01,grain,9553
-2008-08-01,grain,9375
-2008-09-01,grain,9225
-2008-10-01,grain,9948
-2008-11-01,grain,8758
-2008-12-01,grain,10839
-2009-01-01,grain,7266
-2009-02-01,grain,7578
-2009-03-01,grain,8688
-2009-04-01,grain,9162
-2009-05-01,grain,9369
-2009-06-01,grain,10167
-2009-07-01,grain,9507
-2009-08-01,grain,8923
-2009-09-01,grain,9272
-2009-10-01,grain,9075
-2009-11-01,grain,8949
-2009-12-01,grain,10843
-2010-01-01,grain,6558
-2010-02-01,grain,7481
-2010-03-01,grain,9475
-2010-04-01,grain,9424
-2010-05-01,grain,9351
-2010-06-01,grain,10552
-2010-07-01,grain,9077
-2010-08-01,grain,9273
-2010-09-01,grain,9420
-2010-10-01,grain,9413
-2010-11-01,grain,9866
-2010-12-01,grain,11455
-2011-01-01,grain,6901
-2011-02-01,grain,8014
-2011-03-01,grain,9832
-2011-04-01,grain,9281
-2011-05-01,grain,9967
-2011-06-01,grain,11344
-2011-07-01,grain,9106
-2011-08-01,grain,10469
-2011-09-01,grain,10085
-2011-10-01,grain,9612
-2011-11-01,grain,10328
-2011-12-01,grain,11483
-2012-01-01,grain,7486
-2012-02-01,grain,8641
-2012-03-01,grain,9709
-2012-04-01,grain,9423
-2012-05-01,grain,11342
-2012-06-01,grain,11274
-2012-07-01,grain,9845
-2012-08-01,grain,11163
-2012-09-01,grain,9532
-2012-10-01,grain,10754
-2012-11-01,grain,10953
-2012-12-01,grain,11922
-2013-01-01,grain,8395
-2013-02-01,grain,8888
-2013-03-01,grain,10110
-2013-04-01,grain,10493
-2013-05-01,grain,12218
-2013-06-01,grain,11385
-2013-07-01,grain,11186
-2013-08-01,grain,11462
-2013-09-01,grain,10494
-2013-10-01,grain,11540
-2013-11-01,grain,11138
-2013-12-01,grain,12709
-2014-01-01,grain,8557
-2014-02-01,grain,9059
-2014-03-01,grain,10055
-2014-04-01,grain,10977
-2014-05-01,grain,11792
-2014-06-01,grain,11904
-2014-07-01,grain,10965
-2014-08-01,grain,10981
-2014-09-01,grain,10828
-2014-10-01,grain,11817
-2014-11-01,grain,10470
-2014-12-01,grain,13310
-2015-01-01,grain,8400
-2015-02-01,grain,9062
-2015-03-01,grain,10722
-2015-04-01,grain,11107
-2015-05-01,grain,11508
-2015-06-01,grain,12904
-2015-07-01,grain,11869
-2015-08-01,grain,11224
-2015-09-01,grain,12022
-2015-10-01,grain,11983
-2015-11-01,grain,11506
-2015-12-01,grain,14183
-2016-01-01,grain,8650
-2016-02-01,grain,10323
-2016-03-01,grain,12110
-2016-04-01,grain,11424
-2016-05-01,grain,12243
-2016-06-01,grain,13686
-2016-07-01,grain,10956
-2016-08-01,grain,12706
-2016-09-01,grain,12279
-2016-10-01,grain,11914
-2016-11-01,grain,13025
-2016-12-01,grain,14431

how-to-use-azureml/automated-machine-learning/forecasting-beer-remote/auto-ml-forecasting-beer-remote.yml

@@ -1,4 +0,0 @@
-name: auto-ml-forecasting-beer-remote
-dependencies:
-- pip:
-  - azureml-sdk

how-to-use-azureml/automated-machine-learning/forecasting-bike-share/auto-ml-forecasting-bike-share.ipynb

@@ -64,22 +64,23 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "import azureml.core\n",
-        "import pandas as pd\n",
-        "import numpy as np\n",
+        "import json\n",
         "import logging\n",
-        "\n",
-        "from azureml.core import Workspace, Experiment, Dataset\n",
-        "from azureml.train.automl import AutoMLConfig\n",
         "from datetime import datetime\n",
-        "from azureml.automl.core.featurization import FeaturizationConfig"
+        "\n",
+        "import azureml.core\n",
+        "import numpy as np\n",
+        "import pandas as pd\n",
+        "from azureml.automl.core.featurization import FeaturizationConfig\n",
+        "from azureml.core import Dataset, Experiment, Workspace\n",
+        "from azureml.train.automl import AutoMLConfig"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "This sample notebook may use features that are not available in previous versions of the Azure ML SDK."
+        "This notebook is compatible with Azure ML SDK version 1.35.0 or later."
       ]
     },
     {
@@ -88,7 +89,6 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "print(\"This notebook was created using version 1.36.0 of the Azure ML SDK\")\n",
         "print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
       ]
     },
@@ -119,7 +119,8 @@
         "output[\"Resource Group\"] = ws.resource_group\n",
         "output[\"Location\"] = ws.location\n",
         "output[\"Run History Name\"] = experiment_name\n",
-        "pd.set_option(\"display.max_colwidth\", -1)\n",
+        "output[\"SDK Version\"] = azureml.core.VERSION\n",
+        "pd.set_option(\"display.max_colwidth\", None)\n",
         "outputDf = pd.DataFrame(data=output, index=[\"\"])\n",
         "outputDf.T"
       ]
@@ -398,8 +399,8 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "### Retrieve the Best Model\n",
-        "Below we select the best model from all the training iterations using get_output method."
+        "### Retrieve the Best Run details\n",
+        "Below we retrieve the best Run object from among all the runs in the experiment."
       ]
     },
     {
@@ -408,8 +409,8 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "best_run, fitted_model = remote_run.get_output()\n",
-        "fitted_model.steps"
+        "best_run = remote_run.get_best_child()\n",
+        "best_run"
       ]
     },
     {
@@ -418,7 +419,7 @@
       "source": [
         "## Featurization\n",
         "\n",
-        "You can access the engineered feature names generated in time-series featurization. Note that a number of named holiday periods are represented. We recommend that you have at least one year of data when using this feature to ensure that all yearly holidays are captured in the training featurization."
+        "We can look at the engineered feature names generated in time-series featurization via. the JSON file named 'engineered_feature_names.json' under the run outputs. Note that a number of named holiday periods are represented. We recommend that you have at least one year of data when using this feature to ensure that all yearly holidays are captured in the training featurization."
       ]
     },
     {
@@ -427,7 +428,14 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "fitted_model.named_steps[\"timeseriestransformer\"].get_engineered_feature_names()"
+        "# Download the JSON file locally\n",
+        "best_run.download_file(\n",
+        "    \"outputs/engineered_feature_names.json\", \"engineered_feature_names.json\"\n",
+        ")\n",
+        "with open(\"engineered_feature_names.json\", \"r\") as f:\n",
+        "    records = json.load(f)\n",
+        "\n",
+        "records"
       ]
     },
     {
@@ -451,12 +459,26 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "# Get the featurization summary as a list of JSON\n",
-        "featurization_summary = fitted_model.named_steps[\n",
-        "    \"timeseriestransformer\"\n",
-        "].get_featurization_summary()\n",
-        "# View the featurization summary as a pandas dataframe\n",
-        "pd.DataFrame.from_records(featurization_summary)"
+        "# Download the featurization summary JSON file locally\n",
+        "best_run.download_file(\n",
+        "    \"outputs/featurization_summary.json\", \"featurization_summary.json\"\n",
+        ")\n",
+        "\n",
+        "# Render the JSON as a pandas DataFrame\n",
+        "with open(\"featurization_summary.json\", \"r\") as f:\n",
+        "    records = json.load(f)\n",
+        "fs = pd.DataFrame.from_records(records)\n",
+        "\n",
+        "# View a summary of the featurization\n",
+        "fs[\n",
+        "    [\n",
+        "        \"RawFeatureName\",\n",
+        "        \"TypeDetected\",\n",
+        "        \"Dropped\",\n",
+        "        \"EngineeredFeatureCount\",\n",
+        "        \"Transformations\",\n",
+        "    ]\n",
+        "]"
       ]
     },
     {

how-to-use-azureml/automated-machine-learning/forecasting-energy-demand/auto-ml-forecasting-energy-demand.ipynb

@@ -68,6 +68,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
+        "import json\n",
         "import logging\n",
         "\n",
         "from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\n",
@@ -90,7 +91,7 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "This sample notebook may use features that are not available in previous versions of the Azure ML SDK."
+        "This notebook is compatible with Azure ML SDK version 1.35.0 or later."
       ]
     },
     {
@@ -99,7 +100,6 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "print(\"This notebook was created using version 1.36.0 of the Azure ML SDK\")\n",
         "print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
       ]
     },
@@ -132,7 +132,8 @@
         "output[\"Resource Group\"] = ws.resource_group\n",
         "output[\"Location\"] = ws.location\n",
         "output[\"Run History Name\"] = experiment_name\n",
-        "pd.set_option(\"display.max_colwidth\", -1)\n",
+        "output[\"SDK Version\"] = azureml.core.VERSION\n",
+        "pd.set_option(\"display.max_colwidth\", None)\n",
         "outputDf = pd.DataFrame(data=output, index=[\"\"])\n",
         "outputDf.T"
       ]
@@ -398,8 +399,8 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "## Retrieve the Best Model\n",
-        "Below we select the best model from all the training iterations using get_output method."
+        "## Retrieve the Best Run details\n",
+        "Below we retrieve the best Run object from among all the runs in the experiment."
       ]
     },
     {
@@ -408,8 +409,8 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "best_run, fitted_model = remote_run.get_output()\n",
-        "fitted_model.steps"
+        "best_run = remote_run.get_best_child()\n",
+        "best_run"
       ]
     },
     {
@@ -417,7 +418,7 @@
       "metadata": {},
       "source": [
         "## Featurization\n",
-        "You can access the engineered feature names generated in time-series featurization."
+        "We can look at the engineered feature names generated in time-series featurization via. the JSON file named 'engineered_feature_names.json' under the run outputs."
       ]
     },
     {
@@ -426,7 +427,14 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "fitted_model.named_steps[\"timeseriestransformer\"].get_engineered_feature_names()"
+        "# Download the JSON file locally\n",
+        "best_run.download_file(\n",
+        "    \"outputs/engineered_feature_names.json\", \"engineered_feature_names.json\"\n",
+        ")\n",
+        "with open(\"engineered_feature_names.json\", \"r\") as f:\n",
+        "    records = json.load(f)\n",
+        "\n",
+        "records"
       ]
     },
     {
@@ -449,12 +457,26 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "# Get the featurization summary as a list of JSON\n",
-        "featurization_summary = fitted_model.named_steps[\n",
-        "    \"timeseriestransformer\"\n",
-        "].get_featurization_summary()\n",
-        "# View the featurization summary as a pandas dataframe\n",
-        "pd.DataFrame.from_records(featurization_summary)"
+        "# Download the featurization summary JSON file locally\n",
+        "best_run.download_file(\n",
+        "    \"outputs/featurization_summary.json\", \"featurization_summary.json\"\n",
+        ")\n",
+        "\n",
+        "# Render the JSON as a pandas DataFrame\n",
+        "with open(\"featurization_summary.json\", \"r\") as f:\n",
+        "    records = json.load(f)\n",
+        "fs = pd.DataFrame.from_records(records)\n",
+        "\n",
+        "# View a summary of the featurization\n",
+        "fs[\n",
+        "    [\n",
+        "        \"RawFeatureName\",\n",
+        "        \"TypeDetected\",\n",
+        "        \"Dropped\",\n",
+        "        \"EngineeredFeatureCount\",\n",
+        "        \"Transformations\",\n",
+        "    ]\n",
+        "]"
       ]
     },
     {
@@ -481,7 +503,7 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "### Retreiving forecasts from the model\n",
+        "### Retrieving forecasts from the model\n",
         "We have created a function called `run_forecast` that submits the test data to the best model determined during the training run and retrieves forecasts. This function uses a helper script `forecasting_script` which is uploaded and expecuted on the remote compute."
       ]
     },
@@ -641,7 +663,7 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "### Retrieve the Best Model"
+        "### Retrieve the Best Run details"
       ]
     },
     {
@@ -650,7 +672,8 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "best_run_lags, fitted_model_lags = advanced_remote_run.get_output()"
+        "best_run_lags = remote_run.get_best_child()\n",
+        "best_run_lags"
       ]
     },
     {

how-to-use-azureml/automated-machine-learning/forecasting-forecast-function/auto-ml-forecasting-function.ipynb

@@ -85,7 +85,7 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "This sample notebook may use features that are not available in previous versions of the Azure ML SDK."
+        "This notebook is compatible with Azure ML SDK version 1.35.0 or later."
       ]
     },
     {
@@ -94,7 +94,6 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "print(\"This notebook was created using version 1.36.0 of the Azure ML SDK\")\n",
         "print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
       ]
     },
@@ -122,7 +121,8 @@
         "output[\"Resource Group\"] = ws.resource_group\n",
         "output[\"Location\"] = ws.location\n",
         "output[\"Run History Name\"] = experiment_name\n",
-        "pd.set_option(\"display.max_colwidth\", -1)\n",
+        "output[\"SDK Version\"] = azureml.core.VERSION\n",
+        "pd.set_option(\"display.max_colwidth\", None)\n",
         "outputDf = pd.DataFrame(data=output, index=[\"\"])\n",
         "outputDf.T"
       ]

how-to-use-azureml/automated-machine-learning/forecasting-github-dau/auto-ml-forecasting-github-dau.ipynb

@@ -30,7 +30,7 @@
       },
       "source": [
         "# Automated Machine Learning\n",
-        "**Beer Production Forecasting**\n",
+        "**Github DAU Forecasting**\n",
         "\n",
         "## Contents\n",
         "1. [Introduction](#Introduction)\n",
@@ -48,7 +48,7 @@
       },
       "source": [
         "## Introduction\n",
-        "This notebook demonstrates demand forecasting for Beer Production Dataset using AutoML.\n",
+        "This notebook demonstrates demand forecasting for Github Daily Active Users Dataset using AutoML.\n",
         "\n",
         "AutoML highlights here include using Deep Learning forecasts, Arima, Prophet,  Remote Execution and Remote Inferencing, and working with the `forecast` function. Please also look at the additional forecasting notebooks, which document lagging, rolling windows, forecast quantiles, other ways to use the forecast function, and forecaster deployment.\n",
         "\n",
@@ -57,7 +57,7 @@
         "Notebook synopsis:\n",
         "\n",
         "1. Creating an Experiment in an existing Workspace\n",
-        "2. Configuration and remote run of AutoML for a time-series model exploring Regression learners, Arima, Prophet and DNNs\n",
+        "2. Configuration and remote run of AutoML for a time-series model exploring DNNs\n",
         "4. Evaluating the fitted model using a rolling test "
       ]
     },
@@ -92,8 +92,7 @@
         "# Squash warning messages for cleaner output in the notebook\n",
         "warnings.showwarning = lambda *args, **kwargs: None\n",
         "\n",
-        "from azureml.core.workspace import Workspace\n",
-        "from azureml.core.experiment import Experiment\n",
+        "from azureml.core import Workspace, Experiment, Dataset\n",
         "from azureml.train.automl import AutoMLConfig\n",
         "from matplotlib import pyplot as plt\n",
         "from sklearn.metrics import mean_absolute_error, mean_squared_error\n",
@@ -104,7 +103,7 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "This sample notebook may use features that are not available in previous versions of the Azure ML SDK."
+        "This notebook is compatible with Azure ML SDK version 1.35.0 or later."
       ]
     },
     {
@@ -113,7 +112,6 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "print(\"This notebook was created using version 1.36.0 of the Azure ML SDK\")\n",
         "print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
       ]
     },
@@ -139,7 +137,7 @@
         "ws = Workspace.from_config()\n",
         "\n",
         "# choose a name for the run history container in the workspace\n",
-        "experiment_name = \"beer-remote-cpu\"\n",
+        "experiment_name = \"github-remote-cpu\"\n",
         "\n",
         "experiment = Experiment(ws, experiment_name)\n",
         "\n",
@@ -149,7 +147,8 @@
         "output[\"Resource Group\"] = ws.resource_group\n",
         "output[\"Location\"] = ws.location\n",
         "output[\"Run History Name\"] = experiment_name\n",
-        "pd.set_option(\"display.max_colwidth\", -1)\n",
+        "output[\"SDK Version\"] = azureml.core.VERSION\n",
+        "pd.set_option(\"display.max_colwidth\", None)\n",
         "outputDf = pd.DataFrame(data=output, index=[\"\"])\n",
         "outputDf.T"
       ]
@@ -180,7 +179,7 @@
         "from azureml.core.compute_target import ComputeTargetException\n",
         "\n",
         "# Choose a name for your CPU cluster\n",
-        "cpu_cluster_name = \"beer-cluster\"\n",
+        "cpu_cluster_name = \"github-cluster\"\n",
         "\n",
         "# Verify that cluster does not exist already\n",
         "try:\n",
@@ -203,7 +202,7 @@
       },
       "source": [
         "## Data\n",
-        "Read Beer demand data from file, and preview data."
+        "Read Github DAU data from file, and preview data."
       ]
     },
     {
@@ -246,21 +245,19 @@
         "plt.tight_layout()\n",
         "\n",
         "plt.subplot(2, 1, 1)\n",
-        "plt.title(\"Beer Production By Year\")\n",
-        "df = pd.read_csv(\n",
-        "    \"Beer_no_valid_split_train.csv\", parse_dates=True, index_col=\"DATE\"\n",
-        ").drop(columns=\"grain\")\n",
+        "plt.title(\"Github Daily Active User By Year\")\n",
+        "df = pd.read_csv(\"github_dau_2011-2018_train.csv\", parse_dates=True, index_col=\"date\")\n",
         "test_df = pd.read_csv(\n",
-        "    \"Beer_no_valid_split_test.csv\", parse_dates=True, index_col=\"DATE\"\n",
-        ").drop(columns=\"grain\")\n",
+        "    \"github_dau_2011-2018_test.csv\", parse_dates=True, index_col=\"date\"\n",
+        ")\n",
         "plt.plot(df)\n",
         "\n",
         "plt.subplot(2, 1, 2)\n",
-        "plt.title(\"Beer Production By Month\")\n",
+        "plt.title(\"Github Daily Active User By Month\")\n",
         "groups = df.groupby(df.index.month)\n",
         "months = concat([DataFrame(x[1].values) for x in groups], axis=1)\n",
         "months = DataFrame(months)\n",
-        "months.columns = range(1, 13)\n",
+        "months.columns = range(1, 49)\n",
         "months.boxplot()\n",
         "\n",
         "plt.show()"
@@ -275,10 +272,10 @@
       },
       "outputs": [],
       "source": [
-        "target_column_name = \"BeerProduction\"\n",
-        "time_column_name = \"DATE\"\n",
+        "target_column_name = \"count\"\n",
+        "time_column_name = \"date\"\n",
         "time_series_id_column_names = []\n",
-        "freq = \"M\"  # Monthly data"
+        "freq = \"D\"  # Daily data"
       ]
     },
     {
@@ -301,40 +298,21 @@
         "from helper import split_full_for_forecasting\n",
         "\n",
         "train, valid = split_full_for_forecasting(df, time_column_name)\n",
-        "train.to_csv(\"train.csv\")\n",
-        "valid.to_csv(\"valid.csv\")\n",
-        "test_df.to_csv(\"test.csv\")\n",
+        "\n",
+        "# Reset index to create a Tabualr Dataset.\n",
+        "train.reset_index(inplace=True)\n",
+        "valid.reset_index(inplace=True)\n",
+        "test_df.reset_index(inplace=True)\n",
         "\n",
         "datastore = ws.get_default_datastore()\n",
-        "datastore.upload_files(\n",
-        "    files=[\"./train.csv\"],\n",
-        "    target_path=\"beer-dataset/tabular/\",\n",
-        "    overwrite=True,\n",
-        "    show_progress=True,\n",
+        "train_dataset = Dataset.Tabular.register_pandas_dataframe(\n",
+        "    train, target=(datastore, \"dataset/\"), name=\"Github_DAU_train\"\n",
         ")\n",
-        "datastore.upload_files(\n",
-        "    files=[\"./valid.csv\"],\n",
-        "    target_path=\"beer-dataset/tabular/\",\n",
-        "    overwrite=True,\n",
-        "    show_progress=True,\n",
+        "valid_dataset = Dataset.Tabular.register_pandas_dataframe(\n",
+        "    valid, target=(datastore, \"dataset/\"), name=\"Github_DAU_valid\"\n",
         ")\n",
-        "datastore.upload_files(\n",
-        "    files=[\"./test.csv\"],\n",
-        "    target_path=\"beer-dataset/tabular/\",\n",
-        "    overwrite=True,\n",
-        "    show_progress=True,\n",
-        ")\n",
-        "\n",
-        "from azureml.core import Dataset\n",
-        "\n",
-        "train_dataset = Dataset.Tabular.from_delimited_files(\n",
-        "    path=[(datastore, \"beer-dataset/tabular/train.csv\")]\n",
-        ")\n",
-        "valid_dataset = Dataset.Tabular.from_delimited_files(\n",
-        "    path=[(datastore, \"beer-dataset/tabular/valid.csv\")]\n",
-        ")\n",
-        "test_dataset = Dataset.Tabular.from_delimited_files(\n",
-        "    path=[(datastore, \"beer-dataset/tabular/test.csv\")]\n",
+        "test_dataset = Dataset.Tabular.register_pandas_dataframe(\n",
+        "    test_df, target=(datastore, \"dataset/\"), name=\"Github_DAU_test\"\n",
         ")"
       ]
     },
@@ -397,10 +375,10 @@
         "forecasting_parameters = ForecastingParameters(\n",
         "    time_column_name=time_column_name,\n",
         "    forecast_horizon=forecast_horizon,\n",
-        "    freq=\"MS\",  # Set the forecast frequency to be monthly (start of the month)\n",
+        "    freq=\"D\",  # Set the forecast frequency to be daily\n",
         ")\n",
         "\n",
-        "# We will disable the enable_early_stopping flag to ensure the DNN model is recommended for demonstration purpose.\n",
+        "# To only allow the TCNForecaster we set the allowed_models parameter to reflect this.\n",
         "automl_config = AutoMLConfig(\n",
         "    task=\"forecasting\",\n",
         "    primary_metric=\"normalized_root_mean_squared_error\",\n",
@@ -413,7 +391,7 @@
         "    max_concurrent_iterations=4,\n",
         "    max_cores_per_iteration=-1,\n",
         "    enable_dnn=True,\n",
-        "    enable_early_stopping=False,\n",
+        "    allowed_models=[\"TCNForecaster\"],\n",
         "    forecasting_parameters=forecasting_parameters,\n",
         ")"
       ]
@@ -506,7 +484,9 @@
         "if not forecast_model in summary_df[\"run_id\"]:\n",
         "    forecast_model = \"ForecastTCN\"\n",
         "\n",
-        "best_dnn_run_id = summary_df[\"run_id\"][forecast_model]\n",
+        "best_dnn_run_id = summary_df[summary_df[\"Score\"] == summary_df[\"Score\"].min()][\n",
+        "    \"run_id\"\n",
+        "][forecast_model]\n",
         "best_dnn_run = Run(experiment, best_dnn_run_id)"
       ]
     },
@@ -567,11 +547,6 @@
       },
       "outputs": [],
       "source": [
-        "from azureml.core import Dataset\n",
-        "\n",
-        "test_dataset = Dataset.Tabular.from_delimited_files(\n",
-        "    path=[(datastore, \"beer-dataset/tabular/test.csv\")]\n",
-        ")\n",
         "# preview the first 3 rows of the dataset\n",
         "test_dataset.take(5).to_pandas_dataframe()"
       ]
@@ -582,7 +557,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "compute_target = ws.compute_targets[\"beer-cluster\"]\n",
+        "compute_target = ws.compute_targets[\"github-cluster\"]\n",
         "test_experiment = Experiment(ws, experiment_name + \"_test\")"
       ]
     },

how-to-use-azureml/automated-machine-learning/forecasting-github-dau/auto-ml-forecasting-github-dau.yml

@@ -0,0 +1,4 @@
+name: auto-ml-forecasting-github-dau
+dependencies:
+- pip:
+  - azureml-sdk

how-to-use-azureml/automated-machine-learning/forecasting-github-dau/github_dau_2011-2018_test.csv

@@ -0,0 +1,455 @@
+date,count,day_of_week,month_of_year,holiday
+2017-06-04,104663,6.0,5.0,0.0
+2017-06-05,155824,0.0,5.0,0.0
+2017-06-06,164908,1.0,5.0,0.0
+2017-06-07,170309,2.0,5.0,0.0
+2017-06-08,164256,3.0,5.0,0.0
+2017-06-09,153406,4.0,5.0,0.0
+2017-06-10,97024,5.0,5.0,0.0
+2017-06-11,103442,6.0,5.0,0.0
+2017-06-12,160768,0.0,5.0,0.0
+2017-06-13,166288,1.0,5.0,0.0
+2017-06-14,163819,2.0,5.0,0.0
+2017-06-15,157593,3.0,5.0,0.0
+2017-06-16,149259,4.0,5.0,0.0
+2017-06-17,95579,5.0,5.0,0.0
+2017-06-18,98723,6.0,5.0,0.0
+2017-06-19,159076,0.0,5.0,0.0
+2017-06-20,163340,1.0,5.0,0.0
+2017-06-21,163344,2.0,5.0,0.0
+2017-06-22,159528,3.0,5.0,0.0
+2017-06-23,146563,4.0,5.0,0.0
+2017-06-24,92631,5.0,5.0,0.0
+2017-06-25,96549,6.0,5.0,0.0
+2017-06-26,153249,0.0,5.0,0.0
+2017-06-27,160357,1.0,5.0,0.0
+2017-06-28,159941,2.0,5.0,0.0
+2017-06-29,156781,3.0,5.0,0.0
+2017-06-30,144709,4.0,5.0,0.0
+2017-07-01,89101,5.0,6.0,0.0
+2017-07-02,93046,6.0,6.0,0.0
+2017-07-03,144113,0.0,6.0,0.0
+2017-07-04,143061,1.0,6.0,1.0
+2017-07-05,154603,2.0,6.0,0.0
+2017-07-06,157200,3.0,6.0,0.0
+2017-07-07,147213,4.0,6.0,0.0
+2017-07-08,92348,5.0,6.0,0.0
+2017-07-09,97018,6.0,6.0,0.0
+2017-07-10,157192,0.0,6.0,0.0
+2017-07-11,161819,1.0,6.0,0.0
+2017-07-12,161998,2.0,6.0,0.0
+2017-07-13,160280,3.0,6.0,0.0
+2017-07-14,146818,4.0,6.0,0.0
+2017-07-15,93041,5.0,6.0,0.0
+2017-07-16,97505,6.0,6.0,0.0
+2017-07-17,156167,0.0,6.0,0.0
+2017-07-18,162855,1.0,6.0,0.0
+2017-07-19,162519,2.0,6.0,0.0
+2017-07-20,159941,3.0,6.0,0.0
+2017-07-21,148460,4.0,6.0,0.0
+2017-07-22,93431,5.0,6.0,0.0
+2017-07-23,98553,6.0,6.0,0.0
+2017-07-24,156202,0.0,6.0,0.0
+2017-07-25,162503,1.0,6.0,0.0
+2017-07-26,158479,2.0,6.0,0.0
+2017-07-27,158192,3.0,6.0,0.0
+2017-07-28,147108,4.0,6.0,0.0
+2017-07-29,93799,5.0,6.0,0.0
+2017-07-30,97920,6.0,6.0,0.0
+2017-07-31,152197,0.0,6.0,0.0
+2017-08-01,158477,1.0,7.0,0.0
+2017-08-02,159089,2.0,7.0,0.0
+2017-08-03,157182,3.0,7.0,0.0
+2017-08-04,146345,4.0,7.0,0.0
+2017-08-05,92534,5.0,7.0,0.0
+2017-08-06,97128,6.0,7.0,0.0
+2017-08-07,151359,0.0,7.0,0.0
+2017-08-08,159895,1.0,7.0,0.0
+2017-08-09,158329,2.0,7.0,0.0
+2017-08-10,155468,3.0,7.0,0.0
+2017-08-11,144914,4.0,7.0,0.0
+2017-08-12,92258,5.0,7.0,0.0
+2017-08-13,95933,6.0,7.0,0.0
+2017-08-14,147706,0.0,7.0,0.0
+2017-08-15,151115,1.0,7.0,0.0
+2017-08-16,157640,2.0,7.0,0.0
+2017-08-17,156600,3.0,7.0,0.0
+2017-08-18,146980,4.0,7.0,0.0
+2017-08-19,94592,5.0,7.0,0.0
+2017-08-20,99320,6.0,7.0,0.0
+2017-08-21,145727,0.0,7.0,0.0
+2017-08-22,160260,1.0,7.0,0.0
+2017-08-23,160440,2.0,7.0,0.0
+2017-08-24,157830,3.0,7.0,0.0
+2017-08-25,145822,4.0,7.0,0.0
+2017-08-26,94706,5.0,7.0,0.0
+2017-08-27,99047,6.0,7.0,0.0
+2017-08-28,152112,0.0,7.0,0.0
+2017-08-29,162440,1.0,7.0,0.0
+2017-08-30,162902,2.0,7.0,0.0
+2017-08-31,159498,3.0,7.0,0.0
+2017-09-01,145689,4.0,8.0,0.0
+2017-09-02,93589,5.0,8.0,0.0
+2017-09-03,100058,6.0,8.0,0.0
+2017-09-04,140865,0.0,8.0,1.0
+2017-09-05,165715,1.0,8.0,0.0
+2017-09-06,167463,2.0,8.0,0.0
+2017-09-07,164811,3.0,8.0,0.0
+2017-09-08,156157,4.0,8.0,0.0
+2017-09-09,101358,5.0,8.0,0.0
+2017-09-10,107915,6.0,8.0,0.0
+2017-09-11,167845,0.0,8.0,0.0
+2017-09-12,172756,1.0,8.0,0.0
+2017-09-13,172851,2.0,8.0,0.0
+2017-09-14,171675,3.0,8.0,0.0
+2017-09-15,159266,4.0,8.0,0.0
+2017-09-16,103547,5.0,8.0,0.0
+2017-09-17,110964,6.0,8.0,0.0
+2017-09-18,170976,0.0,8.0,0.0
+2017-09-19,177864,1.0,8.0,0.0
+2017-09-20,173567,2.0,8.0,0.0
+2017-09-21,172017,3.0,8.0,0.0
+2017-09-22,161357,4.0,8.0,0.0
+2017-09-23,104681,5.0,8.0,0.0
+2017-09-24,111711,6.0,8.0,0.0
+2017-09-25,173517,0.0,8.0,0.0
+2017-09-26,180049,1.0,8.0,0.0
+2017-09-27,178307,2.0,8.0,0.0
+2017-09-28,174157,3.0,8.0,0.0
+2017-09-29,161707,4.0,8.0,0.0
+2017-09-30,110536,5.0,8.0,0.0
+2017-10-01,106505,6.0,9.0,0.0
+2017-10-02,157565,0.0,9.0,0.0
+2017-10-03,164764,1.0,9.0,0.0
+2017-10-04,163383,2.0,9.0,0.0
+2017-10-05,162847,3.0,9.0,0.0
+2017-10-06,153575,4.0,9.0,0.0
+2017-10-07,107472,5.0,9.0,0.0
+2017-10-08,116127,6.0,9.0,0.0
+2017-10-09,174457,0.0,9.0,1.0
+2017-10-10,185217,1.0,9.0,0.0
+2017-10-11,185120,2.0,9.0,0.0
+2017-10-12,180844,3.0,9.0,0.0
+2017-10-13,170178,4.0,9.0,0.0
+2017-10-14,112754,5.0,9.0,0.0
+2017-10-15,121251,6.0,9.0,0.0
+2017-10-16,183906,0.0,9.0,0.0
+2017-10-17,188945,1.0,9.0,0.0
+2017-10-18,187297,2.0,9.0,0.0
+2017-10-19,183867,3.0,9.0,0.0
+2017-10-20,173021,4.0,9.0,0.0
+2017-10-21,115851,5.0,9.0,0.0
+2017-10-22,126088,6.0,9.0,0.0
+2017-10-23,189452,0.0,9.0,0.0
+2017-10-24,194412,1.0,9.0,0.0
+2017-10-25,192293,2.0,9.0,0.0
+2017-10-26,190163,3.0,9.0,0.0
+2017-10-27,177053,4.0,9.0,0.0
+2017-10-28,114934,5.0,9.0,0.0
+2017-10-29,125289,6.0,9.0,0.0
+2017-10-30,189245,0.0,9.0,0.0
+2017-10-31,191480,1.0,9.0,0.0
+2017-11-01,182281,2.0,10.0,0.0
+2017-11-02,186351,3.0,10.0,0.0
+2017-11-03,175422,4.0,10.0,0.0
+2017-11-04,118160,5.0,10.0,0.0
+2017-11-05,127602,6.0,10.0,0.0
+2017-11-06,191067,0.0,10.0,0.0
+2017-11-07,197083,1.0,10.0,0.0
+2017-11-08,194333,2.0,10.0,0.0
+2017-11-09,193914,3.0,10.0,0.0
+2017-11-10,179933,4.0,10.0,1.0
+2017-11-11,121346,5.0,10.0,0.0
+2017-11-12,131900,6.0,10.0,0.0
+2017-11-13,196969,0.0,10.0,0.0
+2017-11-14,201949,1.0,10.0,0.0
+2017-11-15,198424,2.0,10.0,0.0
+2017-11-16,196902,3.0,10.0,0.0
+2017-11-17,183893,4.0,10.0,0.0
+2017-11-18,122767,5.0,10.0,0.0
+2017-11-19,130890,6.0,10.0,0.0
+2017-11-20,194515,0.0,10.0,0.0
+2017-11-21,198601,1.0,10.0,0.0
+2017-11-22,191041,2.0,10.0,0.0
+2017-11-23,170321,3.0,10.0,1.0
+2017-11-24,155623,4.0,10.0,0.0
+2017-11-25,115759,5.0,10.0,0.0
+2017-11-26,128771,6.0,10.0,0.0
+2017-11-27,199419,0.0,10.0,0.0
+2017-11-28,207253,1.0,10.0,0.0
+2017-11-29,205406,2.0,10.0,0.0
+2017-11-30,200674,3.0,10.0,0.0
+2017-12-01,187017,4.0,11.0,0.0
+2017-12-02,129735,5.0,11.0,0.0
+2017-12-03,139120,6.0,11.0,0.0
+2017-12-04,205505,0.0,11.0,0.0
+2017-12-05,208218,1.0,11.0,0.0
+2017-12-06,202480,2.0,11.0,0.0
+2017-12-07,197822,3.0,11.0,0.0
+2017-12-08,180686,4.0,11.0,0.0
+2017-12-09,123667,5.0,11.0,0.0
+2017-12-10,130987,6.0,11.0,0.0
+2017-12-11,193901,0.0,11.0,0.0
+2017-12-12,194997,1.0,11.0,0.0
+2017-12-13,192063,2.0,11.0,0.0
+2017-12-14,186496,3.0,11.0,0.0
+2017-12-15,170812,4.0,11.0,0.0
+2017-12-16,110474,5.0,11.0,0.0
+2017-12-17,118165,6.0,11.0,0.0
+2017-12-18,176843,0.0,11.0,0.0
+2017-12-19,179550,1.0,11.0,0.0
+2017-12-20,173506,2.0,11.0,0.0
+2017-12-21,165910,3.0,11.0,0.0
+2017-12-22,145886,4.0,11.0,0.0
+2017-12-23,95246,5.0,11.0,0.0
+2017-12-24,88781,6.0,11.0,0.0
+2017-12-25,98189,0.0,11.0,1.0
+2017-12-26,121383,1.0,11.0,0.0
+2017-12-27,135300,2.0,11.0,0.0
+2017-12-28,136827,3.0,11.0,0.0
+2017-12-29,127700,4.0,11.0,0.0
+2017-12-30,93014,5.0,11.0,0.0
+2017-12-31,82878,6.0,11.0,0.0
+2018-01-01,86419,0.0,0.0,1.0
+2018-01-02,147428,1.0,0.0,0.0
+2018-01-03,162193,2.0,0.0,0.0
+2018-01-04,163784,3.0,0.0,0.0
+2018-01-05,158606,4.0,0.0,0.0
+2018-01-06,113467,5.0,0.0,0.0
+2018-01-07,118313,6.0,0.0,0.0
+2018-01-08,175623,0.0,0.0,0.0
+2018-01-09,183880,1.0,0.0,0.0
+2018-01-10,183945,2.0,0.0,0.0
+2018-01-11,181769,3.0,0.0,0.0
+2018-01-12,170552,4.0,0.0,0.0
+2018-01-13,115707,5.0,0.0,0.0
+2018-01-14,121191,6.0,0.0,0.0
+2018-01-15,176127,0.0,0.0,1.0
+2018-01-16,188032,1.0,0.0,0.0
+2018-01-17,189871,2.0,0.0,0.0
+2018-01-18,189348,3.0,0.0,0.0
+2018-01-19,177456,4.0,0.0,0.0
+2018-01-20,123321,5.0,0.0,0.0
+2018-01-21,128306,6.0,0.0,0.0
+2018-01-22,186132,0.0,0.0,0.0
+2018-01-23,197618,1.0,0.0,0.0
+2018-01-24,196402,2.0,0.0,0.0
+2018-01-25,192722,3.0,0.0,0.0
+2018-01-26,179415,4.0,0.0,0.0
+2018-01-27,125769,5.0,0.0,0.0
+2018-01-28,133306,6.0,0.0,0.0
+2018-01-29,194151,0.0,0.0,0.0
+2018-01-30,198680,1.0,0.0,0.0
+2018-01-31,198652,2.0,0.0,0.0
+2018-02-01,195472,3.0,1.0,0.0
+2018-02-02,183173,4.0,1.0,0.0
+2018-02-03,124276,5.0,1.0,0.0
+2018-02-04,129054,6.0,1.0,0.0
+2018-02-05,190024,0.0,1.0,0.0
+2018-02-06,198658,1.0,1.0,0.0
+2018-02-07,198272,2.0,1.0,0.0
+2018-02-08,195339,3.0,1.0,0.0
+2018-02-09,183086,4.0,1.0,0.0
+2018-02-10,122536,5.0,1.0,0.0
+2018-02-11,133033,6.0,1.0,0.0
+2018-02-12,185386,0.0,1.0,0.0
+2018-02-13,184789,1.0,1.0,0.0
+2018-02-14,176089,2.0,1.0,0.0
+2018-02-15,171317,3.0,1.0,0.0
+2018-02-16,162693,4.0,1.0,0.0
+2018-02-17,116342,5.0,1.0,0.0
+2018-02-18,122466,6.0,1.0,0.0
+2018-02-19,172364,0.0,1.0,1.0
+2018-02-20,185896,1.0,1.0,0.0
+2018-02-21,188166,2.0,1.0,0.0
+2018-02-22,189427,3.0,1.0,0.0
+2018-02-23,178732,4.0,1.0,0.0
+2018-02-24,132664,5.0,1.0,0.0
+2018-02-25,134008,6.0,1.0,0.0
+2018-02-26,200075,0.0,1.0,0.0
+2018-02-27,207996,1.0,1.0,0.0
+2018-02-28,204416,2.0,1.0,0.0
+2018-03-01,201320,3.0,2.0,0.0
+2018-03-02,188205,4.0,2.0,0.0
+2018-03-03,131162,5.0,2.0,0.0
+2018-03-04,138320,6.0,2.0,0.0
+2018-03-05,207326,0.0,2.0,0.0
+2018-03-06,212462,1.0,2.0,0.0
+2018-03-07,209357,2.0,2.0,0.0
+2018-03-08,194876,3.0,2.0,0.0
+2018-03-09,193761,4.0,2.0,0.0
+2018-03-10,133449,5.0,2.0,0.0
+2018-03-11,142258,6.0,2.0,0.0
+2018-03-12,208753,0.0,2.0,0.0
+2018-03-13,210602,1.0,2.0,0.0
+2018-03-14,214236,2.0,2.0,0.0
+2018-03-15,210761,3.0,2.0,0.0
+2018-03-16,196619,4.0,2.0,0.0
+2018-03-17,133056,5.0,2.0,0.0
+2018-03-18,141335,6.0,2.0,0.0
+2018-03-19,211580,0.0,2.0,0.0
+2018-03-20,219051,1.0,2.0,0.0
+2018-03-21,215435,2.0,2.0,0.0
+2018-03-22,211961,3.0,2.0,0.0
+2018-03-23,196009,4.0,2.0,0.0
+2018-03-24,132390,5.0,2.0,0.0
+2018-03-25,140021,6.0,2.0,0.0
+2018-03-26,205273,0.0,2.0,0.0
+2018-03-27,212686,1.0,2.0,0.0
+2018-03-28,210683,2.0,2.0,0.0
+2018-03-29,189044,3.0,2.0,0.0
+2018-03-30,170256,4.0,2.0,0.0
+2018-03-31,125999,5.0,2.0,0.0
+2018-04-01,126749,6.0,3.0,0.0
+2018-04-02,186546,0.0,3.0,0.0
+2018-04-03,207905,1.0,3.0,0.0
+2018-04-04,201528,2.0,3.0,0.0
+2018-04-05,188580,3.0,3.0,0.0
+2018-04-06,173714,4.0,3.0,0.0
+2018-04-07,125723,5.0,3.0,0.0
+2018-04-08,142545,6.0,3.0,0.0
+2018-04-09,204767,0.0,3.0,0.0
+2018-04-10,212048,1.0,3.0,0.0
+2018-04-11,210517,2.0,3.0,0.0
+2018-04-12,206924,3.0,3.0,0.0
+2018-04-13,191679,4.0,3.0,0.0
+2018-04-14,126394,5.0,3.0,0.0
+2018-04-15,137279,6.0,3.0,0.0
+2018-04-16,208085,0.0,3.0,0.0
+2018-04-17,213273,1.0,3.0,0.0
+2018-04-18,211580,2.0,3.0,0.0
+2018-04-19,206037,3.0,3.0,0.0
+2018-04-20,191211,4.0,3.0,0.0
+2018-04-21,125564,5.0,3.0,0.0
+2018-04-22,136469,6.0,3.0,0.0
+2018-04-23,206288,0.0,3.0,0.0
+2018-04-24,212115,1.0,3.0,0.0
+2018-04-25,207948,2.0,3.0,0.0
+2018-04-26,205759,3.0,3.0,0.0
+2018-04-27,181330,4.0,3.0,0.0
+2018-04-28,130046,5.0,3.0,0.0
+2018-04-29,120802,6.0,3.0,0.0
+2018-04-30,170390,0.0,3.0,0.0
+2018-05-01,169054,1.0,4.0,0.0
+2018-05-02,197891,2.0,4.0,0.0
+2018-05-03,199820,3.0,4.0,0.0
+2018-05-04,186783,4.0,4.0,0.0
+2018-05-05,124420,5.0,4.0,0.0
+2018-05-06,130666,6.0,4.0,0.0
+2018-05-07,196014,0.0,4.0,0.0
+2018-05-08,203058,1.0,4.0,0.0
+2018-05-09,198582,2.0,4.0,0.0
+2018-05-10,191321,3.0,4.0,0.0
+2018-05-11,183639,4.0,4.0,0.0
+2018-05-12,122023,5.0,4.0,0.0
+2018-05-13,128775,6.0,4.0,0.0
+2018-05-14,199104,0.0,4.0,0.0
+2018-05-15,200658,1.0,4.0,0.0
+2018-05-16,201541,2.0,4.0,0.0
+2018-05-17,196886,3.0,4.0,0.0
+2018-05-18,188597,4.0,4.0,0.0
+2018-05-19,121392,5.0,4.0,0.0
+2018-05-20,126981,6.0,4.0,0.0
+2018-05-21,189291,0.0,4.0,0.0
+2018-05-22,203038,1.0,4.0,0.0
+2018-05-23,205330,2.0,4.0,0.0
+2018-05-24,199208,3.0,4.0,0.0
+2018-05-25,187768,4.0,4.0,0.0
+2018-05-26,117635,5.0,4.0,0.0
+2018-05-27,124352,6.0,4.0,0.0
+2018-05-28,180398,0.0,4.0,1.0
+2018-05-29,194170,1.0,4.0,0.0
+2018-05-30,200281,2.0,4.0,0.0
+2018-05-31,197244,3.0,4.0,0.0
+2018-06-01,184037,4.0,5.0,0.0
+2018-06-02,121135,5.0,5.0,0.0
+2018-06-03,129389,6.0,5.0,0.0
+2018-06-04,200331,0.0,5.0,0.0
+2018-06-05,207735,1.0,5.0,0.0
+2018-06-06,203354,2.0,5.0,0.0
+2018-06-07,200520,3.0,5.0,0.0
+2018-06-08,182038,4.0,5.0,0.0
+2018-06-09,120164,5.0,5.0,0.0
+2018-06-10,125256,6.0,5.0,0.0
+2018-06-11,194786,0.0,5.0,0.0
+2018-06-12,200815,1.0,5.0,0.0
+2018-06-13,197740,2.0,5.0,0.0
+2018-06-14,192294,3.0,5.0,0.0
+2018-06-15,173587,4.0,5.0,0.0
+2018-06-16,105955,5.0,5.0,0.0
+2018-06-17,110780,6.0,5.0,0.0
+2018-06-18,174582,0.0,5.0,0.0
+2018-06-19,193310,1.0,5.0,0.0
+2018-06-20,193062,2.0,5.0,0.0
+2018-06-21,187986,3.0,5.0,0.0
+2018-06-22,173606,4.0,5.0,0.0
+2018-06-23,111795,5.0,5.0,0.0
+2018-06-24,116134,6.0,5.0,0.0
+2018-06-25,185919,0.0,5.0,0.0
+2018-06-26,193142,1.0,5.0,0.0
+2018-06-27,188114,2.0,5.0,0.0
+2018-06-28,183737,3.0,5.0,0.0
+2018-06-29,171496,4.0,5.0,0.0
+2018-06-30,107210,5.0,5.0,0.0
+2018-07-01,111053,6.0,6.0,0.0
+2018-07-02,176198,0.0,6.0,0.0
+2018-07-03,184040,1.0,6.0,0.0
+2018-07-04,169783,2.0,6.0,1.0
+2018-07-05,177996,3.0,6.0,0.0
+2018-07-06,167378,4.0,6.0,0.0
+2018-07-07,106401,5.0,6.0,0.0
+2018-07-08,112327,6.0,6.0,0.0
+2018-07-09,182835,0.0,6.0,0.0
+2018-07-10,187694,1.0,6.0,0.0
+2018-07-11,185762,2.0,6.0,0.0
+2018-07-12,184099,3.0,6.0,0.0
+2018-07-13,170860,4.0,6.0,0.0
+2018-07-14,106799,5.0,6.0,0.0
+2018-07-15,108475,6.0,6.0,0.0
+2018-07-16,175704,0.0,6.0,0.0
+2018-07-17,183596,1.0,6.0,0.0
+2018-07-18,179897,2.0,6.0,0.0
+2018-07-19,183373,3.0,6.0,0.0
+2018-07-20,169626,4.0,6.0,0.0
+2018-07-21,106785,5.0,6.0,0.0
+2018-07-22,112387,6.0,6.0,0.0
+2018-07-23,180572,0.0,6.0,0.0
+2018-07-24,186943,1.0,6.0,0.0
+2018-07-25,185744,2.0,6.0,0.0
+2018-07-26,183117,3.0,6.0,0.0
+2018-07-27,168526,4.0,6.0,0.0
+2018-07-28,105936,5.0,6.0,0.0
+2018-07-29,111708,6.0,6.0,0.0
+2018-07-30,179950,0.0,6.0,0.0
+2018-07-31,185930,1.0,6.0,0.0
+2018-08-01,183366,2.0,7.0,0.0
+2018-08-02,182412,3.0,7.0,0.0
+2018-08-03,173429,4.0,7.0,0.0
+2018-08-04,106108,5.0,7.0,0.0
+2018-08-05,110059,6.0,7.0,0.0
+2018-08-06,178355,0.0,7.0,0.0
+2018-08-07,185518,1.0,7.0,0.0
+2018-08-08,183204,2.0,7.0,0.0
+2018-08-09,181276,3.0,7.0,0.0
+2018-08-10,168297,4.0,7.0,0.0
+2018-08-11,106488,5.0,7.0,0.0
+2018-08-12,111786,6.0,7.0,0.0
+2018-08-13,178620,0.0,7.0,0.0
+2018-08-14,181922,1.0,7.0,0.0
+2018-08-15,172198,2.0,7.0,0.0
+2018-08-16,177367,3.0,7.0,0.0
+2018-08-17,166550,4.0,7.0,0.0
+2018-08-18,107011,5.0,7.0,0.0
+2018-08-19,112299,6.0,7.0,0.0
+2018-08-20,176718,0.0,7.0,0.0
+2018-08-21,182562,1.0,7.0,0.0
+2018-08-22,181484,2.0,7.0,0.0
+2018-08-23,180317,3.0,7.0,0.0
+2018-08-24,170197,4.0,7.0,0.0
+2018-08-25,109383,5.0,7.0,0.0
+2018-08-26,113373,6.0,7.0,0.0
+2018-08-27,180142,0.0,7.0,0.0
+2018-08-28,191628,1.0,7.0,0.0
+2018-08-29,191149,2.0,7.0,0.0
+2018-08-30,187503,3.0,7.0,0.0
+2018-08-31,172280,4.0,7.0,0.0

how-to-use-azureml/automated-machine-learning/forecasting-github-dau/helper.py

@@ -79,9 +79,7 @@ def get_result_df(remote_run):
             if "goal" in run.properties:
                 goal_minimize = run.properties["goal"].split("_")[-1] == "min"
 
-    summary_df = summary_df.T.sort_values(
-        "Score", ascending=goal_minimize
-    ).drop_duplicates(["run_algorithm"])
+    summary_df = summary_df.T.sort_values("Score", ascending=goal_minimize)
     summary_df = summary_df.set_index("run_algorithm")
     return summary_df
 
@@ -105,13 +103,8 @@ def run_inference(
     train_run.download_file(
         "outputs/{}".format(model_base_name), "inference/{}".format(model_base_name)
     )
-    train_run.download_file("outputs/conda_env_v_1_0_0.yml", "inference/condafile.yml")
 
-    inference_env = Environment("myenv")
-    inference_env.docker.enabled = True
-    inference_env.python.conda_dependencies = CondaDependencies(
-        conda_dependencies_file_path="inference/condafile.yml"
-    )
+    inference_env = train_run.get_environment()
 
     est = Estimator(
         source_directory=script_folder,

how-to-use-azureml/automated-machine-learning/forecasting-hierarchical-timeseries/auto-ml-forecasting-hierarchical-timeseries.ipynb

@@ -78,7 +78,8 @@
         "output[\"Resource Group\"] = ws.resource_group\n",
         "output[\"Location\"] = ws.location\n",
         "output[\"Default datastore name\"] = dstore.name\n",
-        "pd.set_option(\"display.max_colwidth\", -1)\n",
+        "output[\"SDK Version\"] = azureml.core.VERSION\n",
+        "pd.set_option(\"display.max_colwidth\", None)\n",
         "outputDf = pd.DataFrame(data=output, index=[\"\"])\n",
         "outputDf.T"
       ]
@@ -150,37 +151,13 @@
         "datastore"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "gather": {
-          "logged": 1613005886349
-        },
-        "jupyter": {
-          "outputs_hidden": false,
-          "source_hidden": false
-        },
-        "nteract": {
-          "transient": {
-            "deleting": false
-          }
-        }
-      },
-      "outputs": [],
-      "source": [
-        "datastore.upload(\n",
-        "    src_dir=\"./Data/\", target_path=datastore_path, overwrite=True, show_progress=True\n",
-        ")"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
         "### Create the TabularDatasets \n",
         "\n",
-        "Datasets in Azure Machine Learning are references to specific data in a Datastore. The data can be retrieved as a [TabularDatasets](https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.data.tabulardataset?view=azure-ml-py)."
+        "Datasets in Azure Machine Learning are references to specific data in a Datastore. The data can be retrieved as a [TabularDatasets](https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.data.tabulardataset?view=azure-ml-py). We will read in the data as a pandas DataFrame, upload to the data store and register them to your Workspace using ```register_pandas_dataframe``` so they can be called as an input into the training pipeline. We will use the inference dataset as part of the forecasting pipeline. The step need only be completed once."
       ]
     },
     {
@@ -193,34 +170,20 @@
       },
       "outputs": [],
       "source": [
-        "from azureml.core.dataset import Dataset\n",
+        "from azureml.data.dataset_factory import TabularDatasetFactory\n",
         "\n",
-        "train_ds = Dataset.Tabular.from_delimited_files(\n",
-        "    path=datastore.path(\"hts-sample/hts-sample-train.csv\"), validate=False\n",
+        "registered_train = TabularDatasetFactory.register_pandas_dataframe(\n",
+        "    pd.read_csv(\"Data/hts-sample-train.csv\"),\n",
+        "    target=(datastore, \"hts-sample\"),\n",
+        "    name=\"hts-sales-train\",\n",
         ")\n",
-        "inference_ds = Dataset.Tabular.from_delimited_files(\n",
-        "    path=datastore.path(\"hts-sample/hts-sample-test.csv\"), validate=False\n",
+        "registered_inference = TabularDatasetFactory.register_pandas_dataframe(\n",
+        "    pd.read_csv(\"Data/hts-sample-test.csv\"),\n",
+        "    target=(datastore, \"hts-sample\"),\n",
+        "    name=\"hts-sales-test\",\n",
         ")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Register the TabularDatasets to the Workspace \n",
-        "Finally, register the dataset to your Workspace so it can be called as an input into the training pipeline in the next notebook. We will use the inference dataset as part of the forecasting pipeline. The step need only be completed once."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "registered_train = train_ds.register(ws, \"hts-sales-train\")\n",
-        "registered_inference = inference_ds.register(ws, \"hts-sales-test\")"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {},
@@ -419,7 +382,7 @@
       "metadata": {},
       "source": [
         "### Submit the pipeline to run\n",
-        "Next we submit our pipeline to run. The whole training pipeline takes about 1h 11m using a Standard_D12_V2 VM with our current ParallelRunConfig setting."
+        "Next we submit our pipeline to run. The whole training pipeline takes about 1h using a Standard_D16_V3 VM with our current ParallelRunConfig setting."
       ]
     },
     {
@@ -609,7 +572,7 @@
       "source": [
         "## Retrieve results\n",
         "\n",
-        "Forecast results can be retrieved through the following code. The prediction results summary and the actual predictions are downloaded the \"forecast_results\" folder"
+        "Forecast results can be retrieved through the following code. The prediction results summary and the actual predictions are downloaded in forecast_results folder"
       ]
     },
     {

how-to-use-azureml/automated-machine-learning/forecasting-many-models/auto-ml-forecasting-many-models.ipynb

@@ -78,7 +78,8 @@
         "output[\"Resource Group\"] = ws.resource_group\n",
         "output[\"Location\"] = ws.location\n",
         "output[\"Default datastore name\"] = dstore.name\n",
-        "pd.set_option(\"display.max_colwidth\", -1)\n",
+        "output[\"SDK Version\"] = azureml.core.VERSION\n",
+        "pd.set_option(\"display.max_colwidth\", None)\n",
         "outputDf = pd.DataFrame(data=output, index=[\"\"])\n",
         "outputDf.T"
       ]
@@ -324,7 +325,7 @@
         "| **enable_early_stopping**          | Flag to enable early termination if the score is not improving in the short term. |\n",
         "| **time_column_name**               | The name of your time column. |\n",
         "| **enable_engineered_explanations** | Engineered feature explanations will be downloaded if enable_engineered_explanations flag is set to True. By default it is set to False to save storage space. |\n",
-        "| **time_series_id_column_name**     | The column names used to uniquely identify timeseries in data that has multiple rows with the same timestamp. |\n",
+        "| **time_series_id_column_names**     | The column names used to uniquely identify timeseries in data that has multiple rows with the same timestamp. |\n",
         "| **track_child_runs**               | Flag to disable tracking of child runs. Only best run is tracked if the flag is set to False (this includes the model and metrics of the run). |\n",
         "| **pipeline_fetch_max_batch_size**  | Determines how many pipelines (training algorithms) to fetch at a time for training, this helps reduce throttling when training at large scale. |\n",
         "| **partition_column_names**         | The names of columns used to group your models. For timeseries, the groups must not split up individual time-series. That is, each group must contain one or more whole time-series. |"
@@ -355,8 +356,8 @@
         "    \"n_cross_validations\": 3,\n",
         "    \"time_column_name\": \"WeekStarting\",\n",
         "    \"drop_column_names\": \"Revenue\",\n",
-        "    \"max_horizon\": 6,\n",
-        "    \"grain_column_names\": partition_column_names,\n",
+        "    \"forecast_horizon\": 6,\n",
+        "    \"time_series_id_column_names\": partition_column_names,\n",
         "    \"track_child_runs\": False,\n",
         "}\n",
         "\n",

how-to-use-azureml/automated-machine-learning/forecasting-orange-juice-sales/auto-ml-forecasting-orange-juice-sales.ipynb

@@ -58,21 +58,22 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "import azureml.core\n",
-        "import pandas as pd\n",
+        "import json\n",
         "import logging\n",
         "\n",
-        "from azureml.core.workspace import Workspace\n",
+        "import azureml.core\n",
+        "import pandas as pd\n",
+        "from azureml.automl.core.featurization import FeaturizationConfig\n",
         "from azureml.core.experiment import Experiment\n",
-        "from azureml.train.automl import AutoMLConfig\n",
-        "from azureml.automl.core.featurization import FeaturizationConfig"
+        "from azureml.core.workspace import Workspace\n",
+        "from azureml.train.automl import AutoMLConfig"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "This sample notebook may use features that are not available in previous versions of the Azure ML SDK."
+        "This notebook is compatible with Azure ML SDK version 1.35.0 or later."
       ]
     },
     {
@@ -81,7 +82,6 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "print(\"This notebook was created using version 1.36.0 of the Azure ML SDK\")\n",
         "print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
       ]
     },
@@ -112,7 +112,8 @@
         "output[\"Resource Group\"] = ws.resource_group\n",
         "output[\"Location\"] = ws.location\n",
         "output[\"Run History Name\"] = experiment_name\n",
-        "pd.set_option(\"display.max_colwidth\", -1)\n",
+        "output[\"SDK Version\"] = azureml.core.VERSION\n",
+        "pd.set_option(\"display.max_colwidth\", None)\n",
         "outputDf = pd.DataFrame(data=output, index=[\"\"])\n",
         "outputDf.T"
       ]
@@ -262,22 +263,14 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "train.to_csv(r\"./dominicks_OJ_train.csv\", index=None, header=True)\n",
-        "test.to_csv(r\"./dominicks_OJ_test.csv\", index=None, header=True)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
+        "from azureml.data.dataset_factory import TabularDatasetFactory\n",
+        "\n",
         "datastore = ws.get_default_datastore()\n",
-        "datastore.upload_files(\n",
-        "    files=[\"./dominicks_OJ_train.csv\", \"./dominicks_OJ_test.csv\"],\n",
-        "    target_path=\"dataset/\",\n",
-        "    overwrite=True,\n",
-        "    show_progress=True,\n",
+        "train_dataset = TabularDatasetFactory.register_pandas_dataframe(\n",
+        "    train, target=(datastore, \"dataset/\"), name=\"dominicks_OJ_train\"\n",
+        ")\n",
+        "test_dataset = TabularDatasetFactory.register_pandas_dataframe(\n",
+        "    test, target=(datastore, \"dataset/\"), name=\"dominicks_OJ_test\"\n",
         ")"
       ]
     },
@@ -288,22 +281,6 @@
         "### Create dataset for training"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "from azureml.core.dataset import Dataset\n",
-        "\n",
-        "train_dataset = Dataset.Tabular.from_delimited_files(\n",
-        "    path=datastore.path(\"dataset/dominicks_OJ_train.csv\")\n",
-        ")\n",
-        "test_dataset = Dataset.Tabular.from_delimited_files(\n",
-        "    path=datastore.path(\"dataset/dominicks_OJ_test.csv\")\n",
-        ")"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -496,8 +473,8 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "### Retrieve the Best Model\n",
-        "Each run within an Experiment stores serialized (i.e. pickled) pipelines from the AutoML iterations. We can now retrieve the pipeline with the best performance on the validation dataset:"
+        "### Retrieve the Best Run details\n",
+        "Below we retrieve the best Run object from among all the runs in the experiment."
       ]
     },
     {
@@ -506,9 +483,9 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "best_run, fitted_model = remote_run.get_output()\n",
-        "print(fitted_model.steps)\n",
-        "model_name = best_run.properties[\"model_name\"]"
+        "best_run = remote_run.get_best_child()\n",
+        "model_name = best_run.properties[\"model_name\"]\n",
+        "best_run"
       ]
     },
     {
@@ -526,16 +503,26 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "custom_featurizer = fitted_model.named_steps[\"timeseriestransformer\"]"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "custom_featurizer.get_featurization_summary()"
+        "# Download the featurization summary JSON file locally\n",
+        "best_run.download_file(\n",
+        "    \"outputs/featurization_summary.json\", \"featurization_summary.json\"\n",
+        ")\n",
+        "\n",
+        "# Render the JSON as a pandas DataFrame\n",
+        "with open(\"featurization_summary.json\", \"r\") as f:\n",
+        "    records = json.load(f)\n",
+        "fs = pd.DataFrame.from_records(records)\n",
+        "\n",
+        "# View a summary of the featurization\n",
+        "fs[\n",
+        "    [\n",
+        "        \"RawFeatureName\",\n",
+        "        \"TypeDetected\",\n",
+        "        \"Dropped\",\n",
+        "        \"EngineeredFeatureCount\",\n",
+        "        \"Transformations\",\n",
+        "    ]\n",
+        "]"
       ]
     },
     {
@@ -562,7 +549,7 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "### Retreiving forecasts from the model\n",
+        "### Retrieving forecasts from the model\n",
         "We have created a function called `run_forecast` that submits the test data to the best model determined during the training run and retrieves forecasts. This function uses a helper script `forecasting_script` which is uploaded and expecuted on the remote compute."
       ]
     },

how-to-use-azureml/automated-machine-learning/forecasting-recipes-univariate/auto-ml-forecasting-univariate-recipe-run-experiment.ipynb

@@ -229,7 +229,7 @@
         "output[\"Resource Group\"] = ws.resource_group\n",
         "output[\"Location\"] = ws.location\n",
         "output[\"Run History Name\"] = experiment_name\n",
-        "pd.set_option(\"display.max_colwidth\", -1)\n",
+        "pd.set_option(\"display.max_colwidth\", None)\n",
         "outputDf = pd.DataFrame(data=output, index=[\"\"])\n",
         "print(outputDf.T)"
       ]
@@ -387,8 +387,8 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "### Retrieve the best model\n",
-        "Below we select the best model from all the training iterations using get_output method."
+        "### Retrieve the Best Run details\n",
+        "Below we retrieve the best Run object from among all the runs in the experiment."
       ]
     },
     {
@@ -397,8 +397,8 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "best_run, fitted_model = remote_run.get_output()\n",
-        "fitted_model.steps"
+        "best_run = remote_run.get_best_child()\n",
+        "best_run"
       ]
     },
     {

how-to-use-azureml/automated-machine-learning/forecasting-recipes-univariate/helper_functions.py

@@ -46,11 +46,11 @@ def kpss_test(series, **kw):
     """
     if kw["store"]:
         statistic, p_value, critical_values, rstore = stattools.kpss(
-            series, regression=kw["reg_type"], lags=kw["lags"], store=kw["store"]
+            series, regression=kw["reg_type"], nlags=kw["lags"], store=kw["store"]
         )
     else:
         statistic, p_value, lags, critical_values = stattools.kpss(
-            series, regression=kw["reg_type"], lags=kw["lags"]
+            series, regression=kw["reg_type"], nlags=kw["lags"]
         )
     output = {
         "statistic": statistic,

how-to-use-azureml/automated-machine-learning/local-run-classification-credit-card-fraud/auto-ml-classification-credit-card-fraud-local.ipynb

@@ -1,21 +1,5 @@
 {
   "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
-        "\n",
-        "Licensed under the MIT License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/automated-machine-learning/classification-credit-card-fraud/auto-ml-classification-credit-card-fraud.png)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {},
@@ -90,16 +74,6 @@
         "This sample notebook may use features that are not available in previous versions of the Azure ML SDK."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "print(\"This notebook was created using version 1.36.0 of the Azure ML SDK\")\n",
-        "print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -109,18 +83,19 @@
         "ws = Workspace.from_config()\n",
         "\n",
         "# choose a name for experiment\n",
-        "experiment_name = 'automl-classification-ccard-local'\n",
+        "experiment_name = \"automl-classification-ccard-local\"\n",
         "\n",
-        "experiment=Experiment(ws, experiment_name)\n",
+        "experiment = Experiment(ws, experiment_name)\n",
         "\n",
         "output = {}\n",
-        "output['Subscription ID'] = ws.subscription_id\n",
-        "output['Workspace'] = ws.name\n",
-        "output['Resource Group'] = ws.resource_group\n",
-        "output['Location'] = ws.location\n",
-        "output['Experiment Name'] = experiment.name\n",
-        "pd.set_option('display.max_colwidth', -1)\n",
-        "outputDf = pd.DataFrame(data = output, index = [''])\n",
+        "output[\"Subscription ID\"] = ws.subscription_id\n",
+        "output[\"Workspace\"] = ws.name\n",
+        "output[\"Resource Group\"] = ws.resource_group\n",
+        "output[\"Location\"] = ws.location\n",
+        "output[\"Experiment Name\"] = experiment.name\n",
+        "output[\"SDK Version\"] = azureml.core.VERSION\n",
+        "pd.set_option(\"display.max_colwidth\", None)\n",
+        "outputDf = pd.DataFrame(data=output, index=[\"\"])\n",
         "outputDf.T"
       ]
     },
@@ -142,7 +117,7 @@
         "data = \"https://automlsamplenotebookdata.blob.core.windows.net/automl-sample-notebook-data/creditcard.csv\"\n",
         "dataset = Dataset.Tabular.from_delimited_files(data)\n",
         "training_data, validation_data = dataset.random_split(percentage=0.8, seed=223)\n",
-        "label_column_name = 'Class'"
+        "label_column_name = \"Class\""
       ]
     },
     {
@@ -168,23 +143,26 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "name": "enable-ensemble"
+      },
       "outputs": [],
       "source": [
         "automl_settings = {\n",
         "    \"n_cross_validations\": 3,\n",
-        "    \"primary_metric\": 'AUC_weighted',\n",
-        "    \"experiment_timeout_hours\": 0.25, # This is a time limit for testing purposes, remove it for real use cases, this will drastically limit ability to find the best model possible\n",
+        "    \"primary_metric\": \"average_precision_score_weighted\",\n",
+        "    \"experiment_timeout_hours\": 0.25,  # This is a time limit for testing purposes, remove it for real use cases, this will drastically limit ability to find the best model possible\n",
         "    \"verbosity\": logging.INFO,\n",
-        "    \"enable_stack_ensemble\": False\n",
+        "    \"enable_stack_ensemble\": False,\n",
         "}\n",
         "\n",
-        "automl_config = AutoMLConfig(task = 'classification',\n",
-        "                             debug_log = 'automl_errors.log',\n",
-        "                             training_data = training_data,\n",
-        "                             label_column_name = label_column_name,\n",
-        "                             **automl_settings\n",
-        "                            )"
+        "automl_config = AutoMLConfig(\n",
+        "    task=\"classification\",\n",
+        "    debug_log=\"automl_errors.log\",\n",
+        "    training_data=training_data,\n",
+        "    label_column_name=label_column_name,\n",
+        "    **automl_settings,\n",
+        ")"
       ]
     },
     {
@@ -201,7 +179,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "local_run = experiment.submit(automl_config, show_output = True)"
+        "local_run = experiment.submit(automl_config, show_output=True)"
       ]
     },
     {
@@ -211,8 +189,8 @@
       "outputs": [],
       "source": [
         "# If you need to retrieve a run that already started, use the following code\n",
-        "#from azureml.train.automl.run import AutoMLRun\n",
-        "#local_run = AutoMLRun(experiment = experiment, run_id = '<replace with your run id>')"
+        "# from azureml.train.automl.run import AutoMLRun\n",
+        "# local_run = AutoMLRun(experiment = experiment, run_id = '<replace with your run id>')"
       ]
     },
     {
@@ -240,6 +218,7 @@
       "outputs": [],
       "source": [
         "from azureml.widgets import RunDetails\n",
+        "\n",
         "RunDetails(local_run).show()"
       ]
     },
@@ -288,8 +267,12 @@
       "outputs": [],
       "source": [
         "# convert the test data to dataframe\n",
-        "X_test_df = validation_data.drop_columns(columns=[label_column_name]).to_pandas_dataframe()\n",
-        "y_test_df = validation_data.keep_columns(columns=[label_column_name], validate=True).to_pandas_dataframe()"
+        "X_test_df = validation_data.drop_columns(\n",
+        "    columns=[label_column_name]\n",
+        ").to_pandas_dataframe()\n",
+        "y_test_df = validation_data.keep_columns(\n",
+        "    columns=[label_column_name], validate=True\n",
+        ").to_pandas_dataframe()"
       ]
     },
     {
@@ -323,20 +306,26 @@
         "import numpy as np\n",
         "import itertools\n",
         "\n",
-        "cf =confusion_matrix(y_test_df.values,y_pred)\n",
-        "plt.imshow(cf,cmap=plt.cm.Blues,interpolation='nearest')\n",
+        "cf = confusion_matrix(y_test_df.values, y_pred)\n",
+        "plt.imshow(cf, cmap=plt.cm.Blues, interpolation=\"nearest\")\n",
         "plt.colorbar()\n",
-        "plt.title('Confusion Matrix')\n",
-        "plt.xlabel('Predicted')\n",
-        "plt.ylabel('Actual')\n",
-        "class_labels = ['False','True']\n",
+        "plt.title(\"Confusion Matrix\")\n",
+        "plt.xlabel(\"Predicted\")\n",
+        "plt.ylabel(\"Actual\")\n",
+        "class_labels = [\"False\", \"True\"]\n",
         "tick_marks = np.arange(len(class_labels))\n",
-        "plt.xticks(tick_marks,class_labels)\n",
-        "plt.yticks([-0.5,0,1,1.5],['','False','True',''])\n",
+        "plt.xticks(tick_marks, class_labels)\n",
+        "plt.yticks([-0.5, 0, 1, 1.5], [\"\", \"False\", \"True\", \"\"])\n",
         "# plotting text value inside cells\n",
-        "thresh = cf.max() / 2.\n",
-        "for i,j in itertools.product(range(cf.shape[0]),range(cf.shape[1])):\n",
-        "    plt.text(j,i,format(cf[i,j],'d'),horizontalalignment='center',color='white' if cf[i,j] >thresh else 'black')\n",
+        "thresh = cf.max() / 2.0\n",
+        "for i, j in itertools.product(range(cf.shape[0]), range(cf.shape[1])):\n",
+        "    plt.text(\n",
+        "        j,\n",
+        "        i,\n",
+        "        format(cf[i, j], \"d\"),\n",
+        "        horizontalalignment=\"center\",\n",
+        "        color=\"white\" if cf[i, j] > thresh else \"black\",\n",
+        "    )\n",
         "plt.show()"
       ]
     },
@@ -363,7 +352,10 @@
         "client = ExplanationClient.from_run(best_run)\n",
         "engineered_explanations = client.download_model_explanation(raw=False)\n",
         "print(engineered_explanations.get_feature_importance_dict())\n",
-        "print(\"You can visualize the engineered explanations under the 'Explanations (preview)' tab in the AutoML run at:-\\n\" + best_run.get_portal_url())"
+        "print(\n",
+        "    \"You can visualize the engineered explanations under the 'Explanations (preview)' tab in the AutoML run at:-\\n\"\n",
+        "    + best_run.get_portal_url()\n",
+        ")"
       ]
     },
     {
@@ -382,7 +374,10 @@
       "source": [
         "raw_explanations = client.download_model_explanation(raw=True)\n",
         "print(raw_explanations.get_feature_importance_dict())\n",
-        "print(\"You can visualize the raw explanations under the 'Explanations (preview)' tab in the AutoML run at:-\\n\" + best_run.get_portal_url())"
+        "print(\n",
+        "    \"You can visualize the raw explanations under the 'Explanations (preview)' tab in the AutoML run at:-\\n\"\n",
+        "    + best_run.get_portal_url()\n",
+        ")"
       ]
     },
     {
@@ -398,7 +393,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "automl_run, fitted_model = local_run.get_output(metric='accuracy')"
+        "automl_run, fitted_model = local_run.get_output(metric=\"accuracy\")"
       ]
     },
     {
@@ -432,12 +427,18 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "from azureml.train.automl.runtime.automl_explain_utilities import automl_setup_model_explanations\n",
+        "from azureml.train.automl.runtime.automl_explain_utilities import (\n",
+        "    automl_setup_model_explanations,\n",
+        ")\n",
         "\n",
-        "automl_explainer_setup_obj = automl_setup_model_explanations(fitted_model, X=X_train, \n",
-        "                                                             X_test=X_test, y=y_train, \n",
-        "                                                             task='classification',\n",
-        "                                                             automl_run=automl_run)"
+        "automl_explainer_setup_obj = automl_setup_model_explanations(\n",
+        "    fitted_model,\n",
+        "    X=X_train,\n",
+        "    X_test=X_test,\n",
+        "    y=y_train,\n",
+        "    task=\"classification\",\n",
+        "    automl_run=automl_run,\n",
+        ")"
       ]
     },
     {
@@ -455,13 +456,18 @@
       "outputs": [],
       "source": [
         "from azureml.interpret.mimic_wrapper import MimicWrapper\n",
-        "explainer = MimicWrapper(ws, automl_explainer_setup_obj.automl_estimator,\n",
-        "                         explainable_model=automl_explainer_setup_obj.surrogate_model, \n",
-        "                         init_dataset=automl_explainer_setup_obj.X_transform, run=automl_explainer_setup_obj.automl_run,\n",
-        "                         features=automl_explainer_setup_obj.engineered_feature_names, \n",
-        "                         feature_maps=[automl_explainer_setup_obj.feature_map],\n",
-        "                         classes=automl_explainer_setup_obj.classes,\n",
-        "                         explainer_kwargs=automl_explainer_setup_obj.surrogate_model_params)"
+        "\n",
+        "explainer = MimicWrapper(\n",
+        "    ws,\n",
+        "    automl_explainer_setup_obj.automl_estimator,\n",
+        "    explainable_model=automl_explainer_setup_obj.surrogate_model,\n",
+        "    init_dataset=automl_explainer_setup_obj.X_transform,\n",
+        "    run=automl_explainer_setup_obj.automl_run,\n",
+        "    features=automl_explainer_setup_obj.engineered_feature_names,\n",
+        "    feature_maps=[automl_explainer_setup_obj.feature_map],\n",
+        "    classes=automl_explainer_setup_obj.classes,\n",
+        "    explainer_kwargs=automl_explainer_setup_obj.surrogate_model_params,\n",
+        ")"
       ]
     },
     {
@@ -479,9 +485,14 @@
       "outputs": [],
       "source": [
         "# Compute the engineered explanations\n",
-        "engineered_explanations = explainer.explain(['local', 'global'], eval_dataset=automl_explainer_setup_obj.X_test_transform)\n",
+        "engineered_explanations = explainer.explain(\n",
+        "    [\"local\", \"global\"], eval_dataset=automl_explainer_setup_obj.X_test_transform\n",
+        ")\n",
         "print(engineered_explanations.get_feature_importance_dict())\n",
-        "print(\"You can visualize the engineered explanations under the 'Explanations (preview)' tab in the AutoML run at:-\\n\" + automl_run.get_portal_url())"
+        "print(\n",
+        "    \"You can visualize the engineered explanations under the 'Explanations (preview)' tab in the AutoML run at:-\\n\"\n",
+        "    + automl_run.get_portal_url()\n",
+        ")"
       ]
     },
     {
@@ -499,12 +510,18 @@
       "outputs": [],
       "source": [
         "# Compute the raw explanations\n",
-        "raw_explanations = explainer.explain(['local', 'global'], get_raw=True,\n",
-        "                                     raw_feature_names=automl_explainer_setup_obj.raw_feature_names,\n",
-        "                                     eval_dataset=automl_explainer_setup_obj.X_test_transform,\n",
-        "                                     raw_eval_dataset=automl_explainer_setup_obj.X_test_raw)\n",
+        "raw_explanations = explainer.explain(\n",
+        "    [\"local\", \"global\"],\n",
+        "    get_raw=True,\n",
+        "    raw_feature_names=automl_explainer_setup_obj.raw_feature_names,\n",
+        "    eval_dataset=automl_explainer_setup_obj.X_test_transform,\n",
+        "    raw_eval_dataset=automl_explainer_setup_obj.X_test_raw,\n",
+        ")\n",
         "print(raw_explanations.get_feature_importance_dict())\n",
-        "print(\"You can visualize the raw explanations under the 'Explanations (preview)' tab in the AutoML run at:-\\n\" + automl_run.get_portal_url())"
+        "print(\n",
+        "    \"You can visualize the raw explanations under the 'Explanations (preview)' tab in the AutoML run at:-\\n\"\n",
+        "    + automl_run.get_portal_url()\n",
+        ")"
       ]
     },
     {
@@ -524,15 +541,17 @@
         "import joblib\n",
         "\n",
         "# Initialize the ScoringExplainer\n",
-        "scoring_explainer = TreeScoringExplainer(explainer.explainer, feature_maps=[automl_explainer_setup_obj.feature_map])\n",
+        "scoring_explainer = TreeScoringExplainer(\n",
+        "    explainer.explainer, feature_maps=[automl_explainer_setup_obj.feature_map]\n",
+        ")\n",
         "\n",
         "# Pickle scoring explainer locally to './scoring_explainer.pkl'\n",
-        "scoring_explainer_file_name = 'scoring_explainer.pkl'\n",
-        "with open(scoring_explainer_file_name, 'wb') as stream:\n",
+        "scoring_explainer_file_name = \"scoring_explainer.pkl\"\n",
+        "with open(scoring_explainer_file_name, \"wb\") as stream:\n",
         "    joblib.dump(scoring_explainer, stream)\n",
         "\n",
         "# Upload the scoring explainer to the automl run\n",
-        "automl_run.upload_file('outputs/scoring_explainer.pkl', scoring_explainer_file_name)"
+        "automl_run.upload_file(\"outputs/scoring_explainer.pkl\", scoring_explainer_file_name)"
       ]
     },
     {
@@ -551,10 +570,12 @@
       "outputs": [],
       "source": [
         "# Register trained automl model present in the 'outputs' folder in the artifacts\n",
-        "original_model = automl_run.register_model(model_name='automl_model', \n",
-        "                                           model_path='outputs/model.pkl')\n",
-        "scoring_explainer_model = automl_run.register_model(model_name='scoring_explainer',\n",
-        "                                                    model_path='outputs/scoring_explainer.pkl')"
+        "original_model = automl_run.register_model(\n",
+        "    model_name=\"automl_model\", model_path=\"outputs/model.pkl\"\n",
+        ")\n",
+        "scoring_explainer_model = automl_run.register_model(\n",
+        "    model_name=\"scoring_explainer\", model_path=\"outputs/scoring_explainer.pkl\"\n",
+        ")"
       ]
     },
     {
@@ -575,7 +596,7 @@
         "from azureml.automl.core.shared import constants\n",
         "from azureml.core.environment import Environment\n",
         "\n",
-        "automl_run.download_file(constants.CONDA_ENV_FILE_PATH, 'myenv.yml')\n",
+        "automl_run.download_file(constants.CONDA_ENV_FILE_PATH, \"myenv.yml\")\n",
         "myenv = Environment.from_conda_specification(name=\"myenv\", file_path=\"myenv.yml\")\n",
         "myenv"
       ]
@@ -598,7 +619,9 @@
         "import joblib\n",
         "import pandas as pd\n",
         "from azureml.core.model import Model\n",
-        "from azureml.train.automl.runtime.automl_explain_utilities import automl_setup_model_explanations\n",
+        "from azureml.train.automl.runtime.automl_explain_utilities import (\n",
+        "    automl_setup_model_explanations,\n",
+        ")\n",
         "\n",
         "\n",
         "def init():\n",
@@ -607,28 +630,35 @@
         "\n",
         "    # Retrieve the path to the model file using the model name\n",
         "    # Assume original model is named original_prediction_model\n",
-        "    automl_model_path = Model.get_model_path('automl_model')\n",
-        "    scoring_explainer_path = Model.get_model_path('scoring_explainer')\n",
+        "    automl_model_path = Model.get_model_path(\"automl_model\")\n",
+        "    scoring_explainer_path = Model.get_model_path(\"scoring_explainer\")\n",
         "\n",
         "    automl_model = joblib.load(automl_model_path)\n",
         "    scoring_explainer = joblib.load(scoring_explainer_path)\n",
         "\n",
         "\n",
         "def run(raw_data):\n",
-        "    data = pd.read_json(raw_data, orient='records')    \n",
+        "    data = pd.read_json(raw_data, orient=\"records\")\n",
         "    # Make prediction\n",
         "    predictions = automl_model.predict(data)\n",
         "    # Setup for inferencing explanations\n",
-        "    automl_explainer_setup_obj = automl_setup_model_explanations(automl_model,\n",
-        "                                                                 X_test=data, task='classification')\n",
+        "    automl_explainer_setup_obj = automl_setup_model_explanations(\n",
+        "        automl_model, X_test=data, task=\"classification\"\n",
+        "    )\n",
         "    # Retrieve model explanations for engineered explanations\n",
-        "    engineered_local_importance_values = scoring_explainer.explain(automl_explainer_setup_obj.X_test_transform)\n",
+        "    engineered_local_importance_values = scoring_explainer.explain(\n",
+        "        automl_explainer_setup_obj.X_test_transform\n",
+        "    )\n",
         "    # Retrieve model explanations for raw explanations\n",
-        "    raw_local_importance_values = scoring_explainer.explain(automl_explainer_setup_obj.X_test_transform, get_raw=True)\n",
+        "    raw_local_importance_values = scoring_explainer.explain(\n",
+        "        automl_explainer_setup_obj.X_test_transform, get_raw=True\n",
+        "    )\n",
         "    # You can return any data type as long as it is JSON-serializable\n",
-        "    return {'predictions': predictions.tolist(),\n",
-        "            'engineered_local_importance_values': engineered_local_importance_values,\n",
-        "            'raw_local_importance_values': raw_local_importance_values}\n"
+        "    return {\n",
+        "        \"predictions\": predictions.tolist(),\n",
+        "        \"engineered_local_importance_values\": engineered_local_importance_values,\n",
+        "        \"raw_local_importance_values\": raw_local_importance_values,\n",
+        "    }"
       ]
     },
     {
@@ -647,7 +677,7 @@
       "source": [
         "from azureml.core.model import InferenceConfig\n",
         "\n",
-        "inf_config = InferenceConfig(entry_script='score.py', environment=myenv)"
+        "inf_config = InferenceConfig(entry_script=\"score.py\", environment=myenv)"
       ]
     },
     {
@@ -668,17 +698,17 @@
         "from azureml.core.compute_target import ComputeTargetException\n",
         "\n",
         "# Choose a name for your cluster.\n",
-        "aks_name = 'scoring-explain'\n",
+        "aks_name = \"scoring-explain\"\n",
         "\n",
         "# Verify that cluster does not exist already\n",
         "try:\n",
         "    aks_target = ComputeTarget(workspace=ws, name=aks_name)\n",
-        "    print('Found existing cluster, use it.')\n",
+        "    print(\"Found existing cluster, use it.\")\n",
         "except ComputeTargetException:\n",
-        "    prov_config = AksCompute.provisioning_configuration(vm_size='STANDARD_D3_V2')\n",
-        "    aks_target = ComputeTarget.create(workspace=ws, \n",
-        "                                      name=aks_name,\n",
-        "                                      provisioning_configuration=prov_config)\n",
+        "    prov_config = AksCompute.provisioning_configuration(vm_size=\"STANDARD_D3_V2\")\n",
+        "    aks_target = ComputeTarget.create(\n",
+        "        workspace=ws, name=aks_name, provisioning_configuration=prov_config\n",
+        "    )\n",
         "aks_target.wait_for_completion(show_output=True)"
       ]
     },
@@ -708,16 +738,18 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "aks_service_name ='model-scoring-local-aks'\n",
+        "aks_service_name = \"model-scoring-local-aks\"\n",
         "\n",
-        "aks_service = Model.deploy(workspace=ws,\n",
-        "                           name=aks_service_name,\n",
-        "                           models=[scoring_explainer_model, original_model],\n",
-        "                           inference_config=inf_config,\n",
-        "                           deployment_config=aks_config,\n",
-        "                           deployment_target=aks_target)\n",
+        "aks_service = Model.deploy(\n",
+        "    workspace=ws,\n",
+        "    name=aks_service_name,\n",
+        "    models=[scoring_explainer_model, original_model],\n",
+        "    inference_config=inf_config,\n",
+        "    deployment_config=aks_config,\n",
+        "    deployment_target=aks_target,\n",
+        ")\n",
         "\n",
-        "aks_service.wait_for_deployment(show_output = True)\n",
+        "aks_service.wait_for_deployment(show_output=True)\n",
         "print(aks_service.state)"
       ]
     },
@@ -752,18 +784,24 @@
       "outputs": [],
       "source": [
         "# Serialize the first row of the test data into json\n",
-        "X_test_json = X_test_df[:1].to_json(orient='records')\n",
+        "X_test_json = X_test_df[:1].to_json(orient=\"records\")\n",
         "print(X_test_json)\n",
         "\n",
         "# Call the service to get the predictions and the engineered and raw explanations\n",
         "output = aks_service.run(X_test_json)\n",
         "\n",
         "# Print the predicted value\n",
-        "print('predictions:\\n{}\\n'.format(output['predictions']))\n",
+        "print(\"predictions:\\n{}\\n\".format(output[\"predictions\"]))\n",
         "# Print the engineered feature importances for the predicted value\n",
-        "print('engineered_local_importance_values:\\n{}\\n'.format(output['engineered_local_importance_values']))\n",
+        "print(\n",
+        "    \"engineered_local_importance_values:\\n{}\\n\".format(\n",
+        "        output[\"engineered_local_importance_values\"]\n",
+        "    )\n",
+        ")\n",
         "# Print the raw feature importances for the predicted value\n",
-        "print('raw_local_importance_values:\\n{}\\n'.format(output['raw_local_importance_values']))\n"
+        "print(\n",
+        "    \"raw_local_importance_values:\\n{}\\n\".format(output[\"raw_local_importance_values\"])\n",
+        ")"
       ]
     },
     {

how-to-use-azureml/automated-machine-learning/regression-explanation-featurization/auto-ml-regression-explanation-featurization.ipynb

@@ -1,21 +1,5 @@
 {
   "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
-        "\n",
-        "Licensed under the MIT License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/automated-machine-learning/regression-car-price-model-explaination-and-featurization/auto-ml-regression.png)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {},
@@ -68,6 +52,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
+        "import json\n",
         "import logging\n",
         "\n",
         "from matplotlib import pyplot as plt\n",
@@ -77,6 +62,7 @@
         "import azureml.core\n",
         "from azureml.core.experiment import Experiment\n",
         "from azureml.core.workspace import Workspace\n",
+        "\n",
         "from azureml.automl.core.featurization import FeaturizationConfig\n",
         "from azureml.train.automl import AutoMLConfig\n",
         "from azureml.core.dataset import Dataset"
@@ -89,16 +75,6 @@
         "This sample notebook may use features that are not available in previous versions of the Azure ML SDK."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "print(\"This notebook was created using version 1.36.0 of the Azure ML SDK\")\n",
-        "print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -108,17 +84,18 @@
         "ws = Workspace.from_config()\n",
         "\n",
         "# Choose a name for the experiment.\n",
-        "experiment_name = 'automl-regression-hardware-explain'\n",
+        "experiment_name = \"automl-regression-hardware-explain\"\n",
         "experiment = Experiment(ws, experiment_name)\n",
         "\n",
         "output = {}\n",
-        "output['Subscription ID'] = ws.subscription_id\n",
-        "output['Workspace Name'] = ws.name\n",
-        "output['Resource Group'] = ws.resource_group\n",
-        "output['Location'] = ws.location\n",
-        "output['Experiment Name'] = experiment.name\n",
-        "pd.set_option('display.max_colwidth', -1)\n",
-        "outputDf = pd.DataFrame(data = output, index = [''])\n",
+        "output[\"Subscription ID\"] = ws.subscription_id\n",
+        "output[\"Workspace Name\"] = ws.name\n",
+        "output[\"Resource Group\"] = ws.resource_group\n",
+        "output[\"Location\"] = ws.location\n",
+        "output[\"Experiment Name\"] = experiment.name\n",
+        "output[\"SDK Version\"] = azureml.core.VERSION\n",
+        "pd.set_option(\"display.max_colwidth\", None)\n",
+        "outputDf = pd.DataFrame(data=output, index=[\"\"])\n",
         "outputDf.T"
       ]
     },
@@ -151,12 +128,12 @@
         "# Verify that cluster does not exist already\n",
         "try:\n",
         "    compute_target = ComputeTarget(workspace=ws, name=amlcompute_cluster_name)\n",
-        "    print('Found existing cluster, use it.')\n",
+        "    print(\"Found existing cluster, use it.\")\n",
         "except ComputeTargetException:\n",
-        "    compute_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_DS12_V2',\n",
-        "                                                           max_nodes=4)\n",
+        "    compute_config = AmlCompute.provisioning_configuration(\n",
+        "        vm_size=\"STANDARD_DS12_V2\", max_nodes=4\n",
+        "    )\n",
         "    compute_target = ComputeTarget.create(ws, amlcompute_cluster_name, compute_config)\n",
-        "\n",
         "compute_target.wait_for_completion(show_output=True)"
       ]
     },
@@ -175,7 +152,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "data = 'https://automlsamplenotebookdata.blob.core.windows.net/automl-sample-notebook-data/machineData.csv'\n",
+        "data = \"https://automlsamplenotebookdata.blob.core.windows.net/automl-sample-notebook-data/machineData.csv\"\n",
         "\n",
         "dataset = Dataset.Tabular.from_delimited_files(data)\n",
         "\n",
@@ -184,14 +161,22 @@
         "\n",
         "\n",
         "# Register the train dataset with your workspace\n",
-        "train_data.register(workspace = ws, name = 'machineData_train_dataset',\n",
-        "                       description = 'hardware performance training data',\n",
-        "                      create_new_version=True)\n",
+        "train_data.register(\n",
+        "    workspace=ws,\n",
+        "    name=\"machineData_train_dataset\",\n",
+        "    description=\"hardware performance training data\",\n",
+        "    create_new_version=True,\n",
+        ")\n",
         "\n",
         "# Register the test dataset with your workspace\n",
-        "test_data.register(workspace = ws, name = 'machineData_test_dataset', description = 'hardware performance test data', create_new_version=True)\n",
+        "test_data.register(\n",
+        "    workspace=ws,\n",
+        "    name=\"machineData_test_dataset\",\n",
+        "    description=\"hardware performance test data\",\n",
+        "    create_new_version=True,\n",
+        ")\n",
         "\n",
-        "label =\"ERP\"\n",
+        "label = \"ERP\"\n",
         "\n",
         "train_data.to_pandas_dataframe().head()"
       ]
@@ -248,15 +233,19 @@
       "outputs": [],
       "source": [
         "featurization_config = FeaturizationConfig()\n",
-        "featurization_config.blocked_transformers = ['LabelEncoder']\n",
-        "#featurization_config.drop_columns = ['MMIN']\n",
-        "featurization_config.add_column_purpose('MYCT', 'Numeric')\n",
-        "featurization_config.add_column_purpose('VendorName', 'CategoricalHash')\n",
-        "#default strategy mean, add transformer param for for 3 columns\n",
-        "featurization_config.add_transformer_params('Imputer', ['CACH'], {\"strategy\": \"median\"})\n",
-        "featurization_config.add_transformer_params('Imputer', ['CHMIN'], {\"strategy\": \"median\"})\n",
-        "featurization_config.add_transformer_params('Imputer', ['PRP'], {\"strategy\": \"most_frequent\"})\n",
-        "#featurization_config.add_transformer_params('HashOneHotEncoder', [], {\"number_of_bits\": 3})"
+        "featurization_config.blocked_transformers = [\"LabelEncoder\"]\n",
+        "# featurization_config.drop_columns = ['MMIN']\n",
+        "featurization_config.add_column_purpose(\"MYCT\", \"Numeric\")\n",
+        "featurization_config.add_column_purpose(\"VendorName\", \"CategoricalHash\")\n",
+        "# default strategy mean, add transformer param for for 3 columns\n",
+        "featurization_config.add_transformer_params(\"Imputer\", [\"CACH\"], {\"strategy\": \"median\"})\n",
+        "featurization_config.add_transformer_params(\n",
+        "    \"Imputer\", [\"CHMIN\"], {\"strategy\": \"median\"}\n",
+        ")\n",
+        "featurization_config.add_transformer_params(\n",
+        "    \"Imputer\", [\"PRP\"], {\"strategy\": \"most_frequent\"}\n",
+        ")\n",
+        "# featurization_config.add_transformer_params('HashOneHotEncoder', [], {\"number_of_bits\": 3})"
       ]
     },
     {
@@ -270,23 +259,24 @@
       "outputs": [],
       "source": [
         "automl_settings = {\n",
-        "    \"enable_early_stopping\": True, \n",
-        "    \"experiment_timeout_hours\" : 0.25,\n",
+        "    \"enable_early_stopping\": True,\n",
+        "    \"experiment_timeout_hours\": 0.25,\n",
         "    \"max_concurrent_iterations\": 4,\n",
         "    \"max_cores_per_iteration\": -1,\n",
         "    \"n_cross_validations\": 5,\n",
-        "    \"primary_metric\": 'normalized_root_mean_squared_error',\n",
-        "    \"verbosity\": logging.INFO\n",
+        "    \"primary_metric\": \"normalized_root_mean_squared_error\",\n",
+        "    \"verbosity\": logging.INFO,\n",
         "}\n",
         "\n",
-        "automl_config = AutoMLConfig(task = 'regression',\n",
-        "                             debug_log = 'automl_errors.log',\n",
-        "                             compute_target=compute_target,\n",
-        "                             featurization=featurization_config,\n",
-        "                             training_data = train_data,\n",
-        "                             label_column_name = label,\n",
-        "                             **automl_settings\n",
-        "                            )"
+        "automl_config = AutoMLConfig(\n",
+        "    task=\"regression\",\n",
+        "    debug_log=\"automl_errors.log\",\n",
+        "    compute_target=compute_target,\n",
+        "    featurization=featurization_config,\n",
+        "    training_data=train_data,\n",
+        "    label_column_name=label,\n",
+        "    **automl_settings,\n",
+        ")"
       ]
     },
     {
@@ -303,7 +293,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "remote_run = experiment.submit(automl_config, show_output = False)"
+        "remote_run = experiment.submit(automl_config, show_output=False)"
       ]
     },
     {
@@ -319,9 +309,9 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "#from azureml.train.automl.run import AutoMLRun\n",
-        "#remote_run = AutoMLRun(experiment=experiment, run_id='<run_ID_goes_here')\n",
-        "#remote_run"
+        "# from azureml.train.automl.run import AutoMLRun\n",
+        "# remote_run = AutoMLRun(experiment=experiment, run_id='<run_ID_goes_here')\n",
+        "# remote_run"
       ]
     },
     {
@@ -339,16 +329,8 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "best_run, fitted_model = remote_run.get_output()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "best_run_customized, fitted_model_customized = remote_run.get_output()"
+        "# Retrieve the best Run object\n",
+        "best_run = remote_run.get_best_child()"
       ]
     },
     {
@@ -357,7 +339,7 @@
       "source": [
         "## Transparency\n",
         "\n",
-        "View updated featurization summary"
+        "View featurization summary for the best model - to study how different features were transformed. This is stored as a JSON file in the outputs directory for the run."
       ]
     },
     {
@@ -366,41 +348,16 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "custom_featurizer = fitted_model_customized.named_steps['datatransformer']"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "custom_featurizer.get_featurization_summary()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "is_user_friendly=False allows for more detailed summary for transforms being applied"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "custom_featurizer.get_featurization_summary(is_user_friendly=False)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "custom_featurizer.get_stats_feature_type_summary()"
+        "# Download the featurization summary JSON file locally\n",
+        "best_run.download_file(\n",
+        "    \"outputs/featurization_summary.json\", \"featurization_summary.json\"\n",
+        ")\n",
+        "\n",
+        "# Render the JSON as a pandas DataFrame\n",
+        "with open(\"featurization_summary.json\", \"r\") as f:\n",
+        "    records = json.load(f)\n",
+        "\n",
+        "pd.DataFrame.from_records(records)"
       ]
     },
     {
@@ -428,7 +385,8 @@
       "outputs": [],
       "source": [
         "from azureml.widgets import RunDetails\n",
-        "RunDetails(remote_run).show() "
+        "\n",
+        "RunDetails(remote_run).show()"
       ]
     },
     {
@@ -449,7 +407,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "#automl_run, fitted_model = remote_run.get_output(metric='r2_score')\n",
+        "# automl_run, fitted_model = remote_run.get_output(metric='r2_score')\n",
         "automl_run, fitted_model = remote_run.get_output(iteration=2)"
       ]
     },
@@ -475,7 +433,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "with open('train_explainer.py', 'r') as cefr:\n",
+        "with open(\"train_explainer.py\", \"r\") as cefr:\n",
         "    print(cefr.read())"
       ]
     },
@@ -497,32 +455,36 @@
         "import os\n",
         "\n",
         "# create script folder\n",
-        "script_folder = './sample_projects/automl-regression-hardware'\n",
+        "script_folder = \"./sample_projects/automl-regression-hardware\"\n",
         "if not os.path.exists(script_folder):\n",
         "    os.makedirs(script_folder)\n",
         "\n",
         "# Copy the sample script to script folder.\n",
-        "shutil.copy('train_explainer.py', script_folder)\n",
+        "shutil.copy(\"train_explainer.py\", script_folder)\n",
         "\n",
         "# Create the explainer script that will run on the remote compute.\n",
-        "script_file_name = script_folder + '/train_explainer.py'\n",
+        "script_file_name = script_folder + \"/train_explainer.py\"\n",
         "\n",
         "# Open the sample script for modification\n",
-        "with open(script_file_name, 'r') as cefr:\n",
+        "with open(script_file_name, \"r\") as cefr:\n",
         "    content = cefr.read()\n",
         "\n",
         "# Replace the values in train_explainer.py file with the appropriate values\n",
-        "content = content.replace('<<experiment_name>>', automl_run.experiment.name) # your experiment name.\n",
-        "content = content.replace('<<run_id>>', automl_run.id) # Run-id of the AutoML run for which you want to explain the model.\n",
-        "content = content.replace('<<target_column_name>>', 'ERP') # Your target column name\n",
-        "content = content.replace('<<task>>', 'regression') # Training task type\n",
+        "content = content.replace(\n",
+        "    \"<<experiment_name>>\", automl_run.experiment.name\n",
+        ")  # your experiment name.\n",
+        "content = content.replace(\n",
+        "    \"<<run_id>>\", automl_run.id\n",
+        ")  # Run-id of the AutoML run for which you want to explain the model.\n",
+        "content = content.replace(\"<<target_column_name>>\", \"ERP\")  # Your target column name\n",
+        "content = content.replace(\"<<task>>\", \"regression\")  # Training task type\n",
         "# Name of your training dataset register with your workspace\n",
-        "content = content.replace('<<train_dataset_name>>', 'machineData_train_dataset') \n",
+        "content = content.replace(\"<<train_dataset_name>>\", \"machineData_train_dataset\")\n",
         "# Name of your test dataset register with your workspace\n",
-        "content = content.replace('<<test_dataset_name>>', 'machineData_test_dataset')\n",
+        "content = content.replace(\"<<test_dataset_name>>\", \"machineData_test_dataset\")\n",
         "\n",
         "# Write sample file into your script folder.\n",
-        "with open(script_file_name, 'w') as cefw:\n",
+        "with open(script_file_name, \"w\") as cefw:\n",
         "    cefw.write(content)"
       ]
     },
@@ -540,6 +502,8 @@
       "outputs": [],
       "source": [
         "from azureml.core.runconfig import RunConfiguration\n",
+        "from azureml.core.conda_dependencies import CondaDependencies\n",
+        "import pkg_resources\n",
         "\n",
         "# create a new RunConfig object\n",
         "conda_run_config = RunConfiguration(framework=\"python\")\n",
@@ -549,7 +513,9 @@
         "conda_run_config.environment.docker.enabled = True\n",
         "\n",
         "# specify CondaDependencies obj\n",
-        "conda_run_config.environment.python.conda_dependencies = automl_run.get_environment().python.conda_dependencies"
+        "conda_run_config.environment.python.conda_dependencies = (\n",
+        "    automl_run.get_environment().python.conda_dependencies\n",
+        ")"
       ]
     },
     {
@@ -569,9 +535,11 @@
         "# Now submit a run on AmlCompute for model explanations\n",
         "from azureml.core.script_run_config import ScriptRunConfig\n",
         "\n",
-        "script_run_config = ScriptRunConfig(source_directory=script_folder,\n",
-        "                                    script='train_explainer.py',\n",
-        "                                    run_config=conda_run_config)\n",
+        "script_run_config = ScriptRunConfig(\n",
+        "    source_directory=script_folder,\n",
+        "    script=\"train_explainer.py\",\n",
+        "    run_config=conda_run_config,\n",
+        ")\n",
         "\n",
         "run = experiment.submit(script_run_config)\n",
         "\n",
@@ -613,10 +581,16 @@
       "outputs": [],
       "source": [
         "from azureml.interpret import ExplanationClient\n",
+        "\n",
         "client = ExplanationClient.from_run(automl_run)\n",
-        "engineered_explanations = client.download_model_explanation(raw=False, comment='engineered explanations')\n",
+        "engineered_explanations = client.download_model_explanation(\n",
+        "    raw=False, comment=\"engineered explanations\"\n",
+        ")\n",
         "print(engineered_explanations.get_feature_importance_dict())\n",
-        "print(\"You can visualize the engineered explanations under the 'Explanations (preview)' tab in the AutoML run at:-\\n\" + automl_run.get_portal_url())"
+        "print(\n",
+        "    \"You can visualize the engineered explanations under the 'Explanations (preview)' tab in the AutoML run at:-\\n\"\n",
+        "    + automl_run.get_portal_url()\n",
+        ")"
       ]
     },
     {
@@ -633,9 +607,14 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "raw_explanations = client.download_model_explanation(raw=True, comment='raw explanations')\n",
+        "raw_explanations = client.download_model_explanation(\n",
+        "    raw=True, comment=\"raw explanations\"\n",
+        ")\n",
         "print(raw_explanations.get_feature_importance_dict())\n",
-        "print(\"You can visualize the raw explanations under the 'Explanations (preview)' tab in the AutoML run at:-\\n\" + automl_run.get_portal_url())"
+        "print(\n",
+        "    \"You can visualize the raw explanations under the 'Explanations (preview)' tab in the AutoML run at:-\\n\"\n",
+        "    + automl_run.get_portal_url()\n",
+        ")"
       ]
     },
     {
@@ -657,10 +636,12 @@
       "outputs": [],
       "source": [
         "# Register trained automl model present in the 'outputs' folder in the artifacts\n",
-        "original_model = automl_run.register_model(model_name='automl_model', \n",
-        "                                           model_path='outputs/model.pkl')\n",
-        "scoring_explainer_model = automl_run.register_model(model_name='scoring_explainer',\n",
-        "                                                    model_path='outputs/scoring_explainer.pkl')"
+        "original_model = automl_run.register_model(\n",
+        "    model_name=\"automl_model\", model_path=\"outputs/model.pkl\"\n",
+        ")\n",
+        "scoring_explainer_model = automl_run.register_model(\n",
+        "    model_name=\"scoring_explainer\", model_path=\"outputs/scoring_explainer.pkl\"\n",
+        ")"
       ]
     },
     {
@@ -679,10 +660,9 @@
       "source": [
         "conda_dep = automl_run.get_environment().python.conda_dependencies\n",
         "\n",
-        "with open(\"myenv.yml\",\"w\") as f:\n",
+        "with open(\"myenv.yml\", \"w\") as f:\n",
         "    f.write(conda_dep.serialize_to_string())\n",
-        "\n",
-        "with open(\"myenv.yml\",\"r\") as f:\n",
+        "with open(\"myenv.yml\", \"r\") as f:\n",
         "    print(f.read())"
       ]
     },
@@ -699,7 +679,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "with open(\"score_explain.py\",\"r\") as f:\n",
+        "with open(\"score_explain.py\", \"r\") as f:\n",
         "    print(f.read())"
       ]
     },
@@ -717,22 +697,30 @@
       "metadata": {},
       "outputs": [],
       "source": [
+        "from azureml.core.webservice import Webservice\n",
         "from azureml.core.model import InferenceConfig\n",
         "from azureml.core.webservice import AciWebservice\n",
         "from azureml.core.model import Model\n",
         "from azureml.core.environment import Environment\n",
         "\n",
-        "aciconfig = AciWebservice.deploy_configuration(cpu_cores=2, \n",
-        "                                               memory_gb=2, \n",
-        "                                               tags={\"data\": \"Machine Data\",  \n",
-        "                                                     \"method\" : \"local_explanation\"}, \n",
-        "                                               description='Get local explanations for Machine test data')\n",
+        "aciconfig = AciWebservice.deploy_configuration(\n",
+        "    cpu_cores=2,\n",
+        "    memory_gb=2,\n",
+        "    tags={\"data\": \"Machine Data\", \"method\": \"local_explanation\"},\n",
+        "    description=\"Get local explanations for Machine test data\",\n",
+        ")\n",
         "\n",
         "myenv = Environment.from_conda_specification(name=\"myenv\", file_path=\"myenv.yml\")\n",
         "inference_config = InferenceConfig(entry_script=\"score_explain.py\", environment=myenv)\n",
         "\n",
         "# Use configs and models generated above\n",
-        "service = Model.deploy(ws, 'model-scoring', [scoring_explainer_model, original_model], inference_config, aciconfig)\n",
+        "service = Model.deploy(\n",
+        "    ws,\n",
+        "    \"model-scoring\",\n",
+        "    [scoring_explainer_model, original_model],\n",
+        "    inference_config,\n",
+        "    aciconfig,\n",
+        ")\n",
         "service.wait_for_deployment(show_output=True)"
       ]
     },
@@ -766,19 +754,19 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "if service.state == 'Healthy':\n",
+        "if service.state == \"Healthy\":\n",
         "    X_test = test_data.drop_columns([label]).to_pandas_dataframe()\n",
         "    # Serialize the first row of the test data into json\n",
-        "    X_test_json = X_test[:1].to_json(orient='records')\n",
+        "    X_test_json = X_test[:1].to_json(orient=\"records\")\n",
         "    print(X_test_json)\n",
         "    # Call the service to get the predictions and the engineered and raw explanations\n",
         "    output = service.run(X_test_json)\n",
         "    # Print the predicted value\n",
-        "    print(output['predictions'])\n",
+        "    print(output[\"predictions\"])\n",
         "    # Print the engineered feature importances for the predicted value\n",
-        "    print(output['engineered_local_importance_values'])\n",
+        "    print(output[\"engineered_local_importance_values\"])\n",
         "    # Print the raw feature importances for the predicted value\n",
-        "    print(output['raw_local_importance_values'])"
+        "    print(output[\"raw_local_importance_values\"])"
       ]
     },
     {
@@ -814,14 +802,14 @@
         "# preview the first 3 rows of the dataset\n",
         "\n",
         "test_data = test_data.to_pandas_dataframe()\n",
-        "y_test = test_data['ERP'].fillna(0)\n",
-        "test_data = test_data.drop('ERP', 1)\n",
+        "y_test = test_data[\"ERP\"].fillna(0)\n",
+        "test_data = test_data.drop(\"ERP\", 1)\n",
         "test_data = test_data.fillna(0)\n",
         "\n",
         "\n",
         "train_data = train_data.to_pandas_dataframe()\n",
-        "y_train = train_data['ERP'].fillna(0)\n",
-        "train_data = train_data.drop('ERP', 1)\n",
+        "y_train = train_data[\"ERP\"].fillna(0)\n",
+        "train_data = train_data.drop(\"ERP\", 1)\n",
         "train_data = train_data.fillna(0)"
       ]
     },
@@ -848,27 +836,41 @@
         "from sklearn.metrics import mean_squared_error, r2_score\n",
         "\n",
         "# Set up a multi-plot chart.\n",
-        "f, (a0, a1) = plt.subplots(1, 2, gridspec_kw = {'width_ratios':[1, 1], 'wspace':0, 'hspace': 0})\n",
-        "f.suptitle('Regression Residual Values', fontsize = 18)\n",
+        "f, (a0, a1) = plt.subplots(\n",
+        "    1, 2, gridspec_kw={\"width_ratios\": [1, 1], \"wspace\": 0, \"hspace\": 0}\n",
+        ")\n",
+        "f.suptitle(\"Regression Residual Values\", fontsize=18)\n",
         "f.set_figheight(6)\n",
         "f.set_figwidth(16)\n",
         "\n",
         "# Plot residual values of training set.\n",
         "a0.axis([0, 360, -100, 100])\n",
-        "a0.plot(y_residual_train, 'bo', alpha = 0.5)\n",
-        "a0.plot([-10,360],[0,0], 'r-', lw = 3)\n",
-        "a0.text(16,170,'RMSE = {0:.2f}'.format(np.sqrt(mean_squared_error(y_train, y_pred_train))), fontsize = 12)\n",
-        "a0.text(16,140,'R2 score = {0:.2f}'.format(r2_score(y_train, y_pred_train)),fontsize = 12)\n",
-        "a0.set_xlabel('Training samples', fontsize = 12)\n",
-        "a0.set_ylabel('Residual Values', fontsize = 12)\n",
+        "a0.plot(y_residual_train, \"bo\", alpha=0.5)\n",
+        "a0.plot([-10, 360], [0, 0], \"r-\", lw=3)\n",
+        "a0.text(\n",
+        "    16,\n",
+        "    170,\n",
+        "    \"RMSE = {0:.2f}\".format(np.sqrt(mean_squared_error(y_train, y_pred_train))),\n",
+        "    fontsize=12,\n",
+        ")\n",
+        "a0.text(\n",
+        "    16, 140, \"R2 score = {0:.2f}\".format(r2_score(y_train, y_pred_train)), fontsize=12\n",
+        ")\n",
+        "a0.set_xlabel(\"Training samples\", fontsize=12)\n",
+        "a0.set_ylabel(\"Residual Values\", fontsize=12)\n",
         "\n",
         "# Plot residual values of test set.\n",
         "a1.axis([0, 90, -100, 100])\n",
-        "a1.plot(y_residual_test, 'bo', alpha = 0.5)\n",
-        "a1.plot([-10,360],[0,0], 'r-', lw = 3)\n",
-        "a1.text(5,170,'RMSE = {0:.2f}'.format(np.sqrt(mean_squared_error(y_test, y_pred_test))), fontsize = 12)\n",
-        "a1.text(5,140,'R2 score = {0:.2f}'.format(r2_score(y_test, y_pred_test)),fontsize = 12)\n",
-        "a1.set_xlabel('Test samples', fontsize = 12)\n",
+        "a1.plot(y_residual_test, \"bo\", alpha=0.5)\n",
+        "a1.plot([-10, 360], [0, 0], \"r-\", lw=3)\n",
+        "a1.text(\n",
+        "    5,\n",
+        "    170,\n",
+        "    \"RMSE = {0:.2f}\".format(np.sqrt(mean_squared_error(y_test, y_pred_test))),\n",
+        "    fontsize=12,\n",
+        ")\n",
+        "a1.text(5, 140, \"R2 score = {0:.2f}\".format(r2_score(y_test, y_pred_test)), fontsize=12)\n",
+        "a1.set_xlabel(\"Test samples\", fontsize=12)\n",
         "a1.set_yticklabels([])\n",
         "\n",
         "plt.show()"
@@ -881,9 +883,11 @@
       "outputs": [],
       "source": [
         "%matplotlib inline\n",
-        "test_pred = plt.scatter(y_test, y_pred_test, color='')\n",
-        "test_test = plt.scatter(y_test, y_test, color='g')\n",
-        "plt.legend((test_pred, test_test), ('prediction', 'truth'), loc='upper left', fontsize=8)\n",
+        "test_pred = plt.scatter(y_test, y_pred_test, color=\"\")\n",
+        "test_test = plt.scatter(y_test, y_test, color=\"g\")\n",
+        "plt.legend(\n",
+        "    (test_pred, test_test), (\"prediction\", \"truth\"), loc=\"upper left\", fontsize=8\n",
+        ")\n",
         "plt.show()"
       ]
     }

how-to-use-azureml/automated-machine-learning/regression-explanation-featurization/score_explain.py

@@ -1,7 +1,10 @@
 import pandas as pd
 import joblib
 from azureml.core.model import Model
-from azureml.train.automl.runtime.automl_explain_utilities import automl_setup_model_explanations
+from azureml.train.automl.runtime.automl_explain_utilities import (
+    automl_setup_model_explanations,
+)
+import scipy as sp
 
 
 def init():
@@ -11,26 +14,55 @@ def init():
 
     # Retrieve the path to the model file using the model name
     # Assume original model is named original_prediction_model
-    automl_model_path = Model.get_model_path('automl_model')
-    scoring_explainer_path = Model.get_model_path('scoring_explainer')
+    automl_model_path = Model.get_model_path("automl_model")
+    scoring_explainer_path = Model.get_model_path("scoring_explainer")
 
     automl_model = joblib.load(automl_model_path)
     scoring_explainer = joblib.load(scoring_explainer_path)
 
 
+def is_multi_dimensional(matrix):
+    if hasattr(matrix, "ndim") and matrix.ndim > 1:
+        return True
+    if hasattr(matrix, "shape") and matrix.shape[1]:
+        return True
+    return False
+
+
+def convert_matrix(matrix):
+    if sp.sparse.issparse(matrix):
+        matrix = matrix.todense()
+    if is_multi_dimensional(matrix):
+        matrix = matrix.tolist()
+    return matrix
+
+
 def run(raw_data):
     # Get predictions and explanations for each data point
-    data = pd.read_json(raw_data, orient='records')
+    data = pd.read_json(raw_data, orient="records")
     # Make prediction
     predictions = automl_model.predict(data)
     # Setup for inferencing explanations
-    automl_explainer_setup_obj = automl_setup_model_explanations(automl_model,
-                                                                 X_test=data, task='regression')
+    automl_explainer_setup_obj = automl_setup_model_explanations(
+        automl_model, X_test=data, task="regression"
+    )
     # Retrieve model explanations for engineered explanations
-    engineered_local_importance_values = scoring_explainer.explain(automl_explainer_setup_obj.X_test_transform)
+    engineered_local_importance_values = scoring_explainer.explain(
+        automl_explainer_setup_obj.X_test_transform
+    )
+    engineered_local_importance_values = convert_matrix(
+        engineered_local_importance_values
+    )
+
     # Retrieve model explanations for raw explanations
-    raw_local_importance_values = scoring_explainer.explain(automl_explainer_setup_obj.X_test_transform, get_raw=True)
+    raw_local_importance_values = scoring_explainer.explain(
+        automl_explainer_setup_obj.X_test_transform, get_raw=True
+    )
+    raw_local_importance_values = convert_matrix(raw_local_importance_values)
+
     # You can return any data type as long as it is JSON-serializable
-    return {'predictions': predictions.tolist(),
-            'engineered_local_importance_values': engineered_local_importance_values,
-            'raw_local_importance_values': raw_local_importance_values}
+    return {
+        "predictions": predictions.tolist(),
+        "engineered_local_importance_values": engineered_local_importance_values,
+        "raw_local_importance_values": raw_local_importance_values,
+    }

how-to-use-azureml/automated-machine-learning/regression-explanation-featurization/train_explainer.py

@@ -10,11 +10,13 @@ from azureml.core.dataset import Dataset
 from azureml.core.run import Run
 from azureml.interpret.mimic_wrapper import MimicWrapper
 from azureml.interpret.scoring.scoring_explainer import TreeScoringExplainer
-from azureml.train.automl.runtime.automl_explain_utilities import automl_setup_model_explanations, \
-    automl_check_model_if_explainable
+from azureml.train.automl.runtime.automl_explain_utilities import (
+    automl_setup_model_explanations,
+    automl_check_model_if_explainable,
+)
 
 
-OUTPUT_DIR = './outputs/'
+OUTPUT_DIR = "./outputs/"
 os.makedirs(OUTPUT_DIR, exist_ok=True)
 
 # Get workspace from the run context
@@ -22,63 +24,77 @@ run = Run.get_context()
 ws = run.experiment.workspace
 
 # Get the AutoML run object from the experiment name and the workspace
-experiment = Experiment(ws, '<<experiment_name>>')
-automl_run = Run(experiment=experiment, run_id='<<run_id>>')
+experiment = Experiment(ws, "<<experiment_name>>")
+automl_run = Run(experiment=experiment, run_id="<<run_id>>")
 
 # Check if this AutoML model is explainable
 if not automl_check_model_if_explainable(automl_run):
-    raise Exception("Model explanations are currently not supported for " + automl_run.get_properties().get(
-        'run_algorithm'))
+    raise Exception(
+        "Model explanations are currently not supported for "
+        + automl_run.get_properties().get("run_algorithm")
+    )
 
 # Download the best model from the artifact store
-automl_run.download_file(name=MODEL_PATH, output_file_path='model.pkl')
+automl_run.download_file(name=MODEL_PATH, output_file_path="model.pkl")
 
 # Load the AutoML model into memory
-fitted_model = joblib.load('model.pkl')
+fitted_model = joblib.load("model.pkl")
 
 # Get the train dataset from the workspace
-train_dataset = Dataset.get_by_name(workspace=ws, name='<<train_dataset_name>>')
+train_dataset = Dataset.get_by_name(workspace=ws, name="<<train_dataset_name>>")
 # Drop the labeled column to get the training set.
-X_train = train_dataset.drop_columns(columns=['<<target_column_name>>'])
-y_train = train_dataset.keep_columns(columns=['<<target_column_name>>'], validate=True)
+X_train = train_dataset.drop_columns(columns=["<<target_column_name>>"])
+y_train = train_dataset.keep_columns(columns=["<<target_column_name>>"], validate=True)
 
 # Get the test dataset from the workspace
-test_dataset = Dataset.get_by_name(workspace=ws, name='<<test_dataset_name>>')
+test_dataset = Dataset.get_by_name(workspace=ws, name="<<test_dataset_name>>")
 # Drop the labeled column to get the testing set.
-X_test = test_dataset.drop_columns(columns=['<<target_column_name>>'])
+X_test = test_dataset.drop_columns(columns=["<<target_column_name>>"])
 
 # Setup the class for explaining the AutoML models
-automl_explainer_setup_obj = automl_setup_model_explanations(fitted_model, '<<task>>',
-                                                             X=X_train, X_test=X_test,
-                                                             y=y_train,
-                                                             automl_run=automl_run)
+automl_explainer_setup_obj = automl_setup_model_explanations(
+    fitted_model, "<<task>>", X=X_train, X_test=X_test, y=y_train, automl_run=automl_run
+)
 
 # Initialize the Mimic Explainer
-explainer = MimicWrapper(ws, automl_explainer_setup_obj.automl_estimator, LGBMExplainableModel,
-                         init_dataset=automl_explainer_setup_obj.X_transform,
-                         run=automl_explainer_setup_obj.automl_run,
-                         features=automl_explainer_setup_obj.engineered_feature_names,
-                         feature_maps=[automl_explainer_setup_obj.feature_map],
-                         classes=automl_explainer_setup_obj.classes)
+explainer = MimicWrapper(
+    ws,
+    automl_explainer_setup_obj.automl_estimator,
+    LGBMExplainableModel,
+    init_dataset=automl_explainer_setup_obj.X_transform,
+    run=automl_explainer_setup_obj.automl_run,
+    features=automl_explainer_setup_obj.engineered_feature_names,
+    feature_maps=[automl_explainer_setup_obj.feature_map],
+    classes=automl_explainer_setup_obj.classes,
+)
 
 # Compute the engineered explanations
-engineered_explanations = explainer.explain(['local', 'global'], tag='engineered explanations',
-                                            eval_dataset=automl_explainer_setup_obj.X_test_transform)
+engineered_explanations = explainer.explain(
+    ["local", "global"],
+    tag="engineered explanations",
+    eval_dataset=automl_explainer_setup_obj.X_test_transform,
+)
 
 # Compute the raw explanations
-raw_explanations = explainer.explain(['local', 'global'], get_raw=True, tag='raw explanations',
-                                     raw_feature_names=automl_explainer_setup_obj.raw_feature_names,
-                                     eval_dataset=automl_explainer_setup_obj.X_test_transform,
-                                     raw_eval_dataset=automl_explainer_setup_obj.X_test_raw)
+raw_explanations = explainer.explain(
+    ["local", "global"],
+    get_raw=True,
+    tag="raw explanations",
+    raw_feature_names=automl_explainer_setup_obj.raw_feature_names,
+    eval_dataset=automl_explainer_setup_obj.X_test_transform,
+    raw_eval_dataset=automl_explainer_setup_obj.X_test_raw,
+)
 
 print("Engineered and raw explanations computed successfully")
 
 # Initialize the ScoringExplainer
-scoring_explainer = TreeScoringExplainer(explainer.explainer, feature_maps=[automl_explainer_setup_obj.feature_map])
+scoring_explainer = TreeScoringExplainer(
+    explainer.explainer, feature_maps=[automl_explainer_setup_obj.feature_map]
+)
 
 # Pickle scoring explainer locally
-with open('scoring_explainer.pkl', 'wb') as stream:
+with open("scoring_explainer.pkl", "wb") as stream:
     joblib.dump(scoring_explainer, stream)
 
 # Upload the scoring explainer to the automl run
-automl_run.upload_file('outputs/scoring_explainer.pkl', 'scoring_explainer.pkl')
+automl_run.upload_file("outputs/scoring_explainer.pkl", "scoring_explainer.pkl")

how-to-use-azureml/automated-machine-learning/regression/auto-ml-regression.ipynb

@@ -1,21 +1,5 @@
 {
   "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
-        "\n",
-        "Licensed under the MIT License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/automated-machine-learning/regression/auto-ml-regression.png)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {},
@@ -70,7 +54,7 @@
         "from matplotlib import pyplot as plt\n",
         "import numpy as np\n",
         "import pandas as pd\n",
-        " \n",
+        "\n",
         "\n",
         "import azureml.core\n",
         "from azureml.core.experiment import Experiment\n",
@@ -86,16 +70,6 @@
         "This sample notebook may use features that are not available in previous versions of the Azure ML SDK."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "print(\"This notebook was created using version 1.36.0 of the Azure ML SDK\")\n",
-        "print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -105,18 +79,19 @@
         "ws = Workspace.from_config()\n",
         "\n",
         "# Choose a name for the experiment.\n",
-        "experiment_name = 'automl-regression'\n",
+        "experiment_name = \"automl-regression\"\n",
         "\n",
         "experiment = Experiment(ws, experiment_name)\n",
         "\n",
         "output = {}\n",
-        "output['Subscription ID'] = ws.subscription_id\n",
-        "output['Workspace'] = ws.name\n",
-        "output['Resource Group'] = ws.resource_group\n",
-        "output['Location'] = ws.location\n",
-        "output['Run History Name'] = experiment_name\n",
-        "pd.set_option('display.max_colwidth', -1)\n",
-        "outputDf = pd.DataFrame(data = output, index = [''])\n",
+        "output[\"Subscription ID\"] = ws.subscription_id\n",
+        "output[\"Workspace\"] = ws.name\n",
+        "output[\"Resource Group\"] = ws.resource_group\n",
+        "output[\"Location\"] = ws.location\n",
+        "output[\"Run History Name\"] = experiment_name\n",
+        "output[\"SDK Version\"] = azureml.core.VERSION\n",
+        "pd.set_option(\"display.max_colwidth\", None)\n",
+        "outputDf = pd.DataFrame(data=output, index=[\"\"])\n",
         "outputDf.T"
       ]
     },
@@ -143,10 +118,11 @@
         "# Verify that cluster does not exist already\n",
         "try:\n",
         "    compute_target = ComputeTarget(workspace=ws, name=cpu_cluster_name)\n",
-        "    print('Found existing cluster, use it.')\n",
+        "    print(\"Found existing cluster, use it.\")\n",
         "except ComputeTargetException:\n",
-        "    compute_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_DS12_V2',\n",
-        "                                                           max_nodes=4)\n",
+        "    compute_config = AmlCompute.provisioning_configuration(\n",
+        "        vm_size=\"STANDARD_DS12_V2\", max_nodes=4\n",
+        "    )\n",
         "    compute_target = ComputeTarget.create(ws, cpu_cluster_name, compute_config)\n",
         "\n",
         "compute_target.wait_for_completion(show_output=True)"
@@ -179,7 +155,7 @@
         "# Split the dataset into train and test datasets\n",
         "train_data, test_data = dataset.random_split(percentage=0.8, seed=223)\n",
         "\n",
-        "label = \"ERP\"\n"
+        "label = \"ERP\""
       ]
     },
     {
@@ -213,20 +189,21 @@
       "source": [
         "automl_settings = {\n",
         "    \"n_cross_validations\": 3,\n",
-        "    \"primary_metric\": 'normalized_root_mean_squared_error',\n",
-        "    \"enable_early_stopping\": True, \n",
-        "    \"experiment_timeout_hours\": 0.3, #for real scenarios we reccommend a timeout of at least one hour \n",
+        "    \"primary_metric\": \"r2_score\",\n",
+        "    \"enable_early_stopping\": True,\n",
+        "    \"experiment_timeout_hours\": 0.3,  # for real scenarios we reccommend a timeout of at least one hour\n",
         "    \"max_concurrent_iterations\": 4,\n",
         "    \"max_cores_per_iteration\": -1,\n",
         "    \"verbosity\": logging.INFO,\n",
         "}\n",
         "\n",
-        "automl_config = AutoMLConfig(task = 'regression',\n",
-        "                             compute_target = compute_target,\n",
-        "                             training_data = train_data,\n",
-        "                             label_column_name = label,\n",
-        "                             **automl_settings\n",
-        "                            )"
+        "automl_config = AutoMLConfig(\n",
+        "    task=\"regression\",\n",
+        "    compute_target=compute_target,\n",
+        "    training_data=train_data,\n",
+        "    label_column_name=label,\n",
+        "    **automl_settings,\n",
+        ")"
       ]
     },
     {
@@ -242,7 +219,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "remote_run = experiment.submit(automl_config, show_output = False)"
+        "remote_run = experiment.submit(automl_config, show_output=False)"
       ]
     },
     {
@@ -252,8 +229,8 @@
       "outputs": [],
       "source": [
         "# If you need to retrieve a run that already started, use the following code\n",
-        "#from azureml.train.automl.run import AutoMLRun\n",
-        "#remote_run = AutoMLRun(experiment = experiment, run_id = '<replace with your run id>')"
+        "# from azureml.train.automl.run import AutoMLRun\n",
+        "# remote_run = AutoMLRun(experiment = experiment, run_id = '<replace with your run id>')"
       ]
     },
     {
@@ -281,7 +258,8 @@
       "outputs": [],
       "source": [
         "from azureml.widgets import RunDetails\n",
-        "RunDetails(remote_run).show() "
+        "\n",
+        "RunDetails(remote_run).show()"
       ]
     },
     {
@@ -328,7 +306,7 @@
       "outputs": [],
       "source": [
         "lookup_metric = \"root_mean_squared_error\"\n",
-        "best_run, fitted_model = remote_run.get_output(metric = lookup_metric)\n",
+        "best_run, fitted_model = remote_run.get_output(metric=lookup_metric)\n",
         "print(best_run)\n",
         "print(fitted_model)"
       ]
@@ -348,7 +326,7 @@
       "outputs": [],
       "source": [
         "iteration = 3\n",
-        "third_run, third_model = remote_run.get_output(iteration = iteration)\n",
+        "third_run, third_model = remote_run.get_output(iteration=iteration)\n",
         "print(third_run)\n",
         "print(third_model)"
       ]
@@ -366,12 +344,12 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "y_test = test_data.keep_columns('ERP').to_pandas_dataframe()\n",
-        "test_data = test_data.drop_columns('ERP').to_pandas_dataframe()\n",
+        "y_test = test_data.keep_columns(\"ERP\").to_pandas_dataframe()\n",
+        "test_data = test_data.drop_columns(\"ERP\").to_pandas_dataframe()\n",
         "\n",
         "\n",
-        "y_train = train_data.keep_columns('ERP').to_pandas_dataframe()\n",
-        "train_data = train_data.drop_columns('ERP').to_pandas_dataframe()\n"
+        "y_train = train_data.keep_columns(\"ERP\").to_pandas_dataframe()\n",
+        "train_data = train_data.drop_columns(\"ERP\").to_pandas_dataframe()"
       ]
     },
     {
@@ -397,27 +375,41 @@
         "from sklearn.metrics import mean_squared_error, r2_score\n",
         "\n",
         "# Set up a multi-plot chart.\n",
-        "f, (a0, a1) = plt.subplots(1, 2, gridspec_kw = {'width_ratios':[1, 1], 'wspace':0, 'hspace': 0})\n",
-        "f.suptitle('Regression Residual Values', fontsize = 18)\n",
+        "f, (a0, a1) = plt.subplots(\n",
+        "    1, 2, gridspec_kw={\"width_ratios\": [1, 1], \"wspace\": 0, \"hspace\": 0}\n",
+        ")\n",
+        "f.suptitle(\"Regression Residual Values\", fontsize=18)\n",
         "f.set_figheight(6)\n",
         "f.set_figwidth(16)\n",
         "\n",
         "# Plot residual values of training set.\n",
         "a0.axis([0, 360, -100, 100])\n",
-        "a0.plot(y_residual_train, 'bo', alpha = 0.5)\n",
-        "a0.plot([-10,360],[0,0], 'r-', lw = 3)\n",
-        "a0.text(16,170,'RMSE = {0:.2f}'.format(np.sqrt(mean_squared_error(y_train, y_pred_train))), fontsize = 12)\n",
-        "a0.text(16,140,'R2 score = {0:.2f}'.format(r2_score(y_train, y_pred_train)),fontsize = 12)\n",
-        "a0.set_xlabel('Training samples', fontsize = 12)\n",
-        "a0.set_ylabel('Residual Values', fontsize = 12)\n",
+        "a0.plot(y_residual_train, \"bo\", alpha=0.5)\n",
+        "a0.plot([-10, 360], [0, 0], \"r-\", lw=3)\n",
+        "a0.text(\n",
+        "    16,\n",
+        "    170,\n",
+        "    \"RMSE = {0:.2f}\".format(np.sqrt(mean_squared_error(y_train, y_pred_train))),\n",
+        "    fontsize=12,\n",
+        ")\n",
+        "a0.text(\n",
+        "    16, 140, \"R2 score = {0:.2f}\".format(r2_score(y_train, y_pred_train)), fontsize=12\n",
+        ")\n",
+        "a0.set_xlabel(\"Training samples\", fontsize=12)\n",
+        "a0.set_ylabel(\"Residual Values\", fontsize=12)\n",
         "\n",
         "# Plot residual values of test set.\n",
         "a1.axis([0, 90, -100, 100])\n",
-        "a1.plot(y_residual_test, 'bo', alpha = 0.5)\n",
-        "a1.plot([-10,360],[0,0], 'r-', lw = 3)\n",
-        "a1.text(5,170,'RMSE = {0:.2f}'.format(np.sqrt(mean_squared_error(y_test, y_pred_test))), fontsize = 12)\n",
-        "a1.text(5,140,'R2 score = {0:.2f}'.format(r2_score(y_test, y_pred_test)),fontsize = 12)\n",
-        "a1.set_xlabel('Test samples', fontsize = 12)\n",
+        "a1.plot(y_residual_test, \"bo\", alpha=0.5)\n",
+        "a1.plot([-10, 360], [0, 0], \"r-\", lw=3)\n",
+        "a1.text(\n",
+        "    5,\n",
+        "    170,\n",
+        "    \"RMSE = {0:.2f}\".format(np.sqrt(mean_squared_error(y_test, y_pred_test))),\n",
+        "    fontsize=12,\n",
+        ")\n",
+        "a1.text(5, 140, \"R2 score = {0:.2f}\".format(r2_score(y_test, y_pred_test)), fontsize=12)\n",
+        "a1.set_xlabel(\"Test samples\", fontsize=12)\n",
         "a1.set_yticklabels([])\n",
         "\n",
         "plt.show()"
@@ -430,9 +422,11 @@
       "outputs": [],
       "source": [
         "%matplotlib inline\n",
-        "test_pred = plt.scatter(y_test, y_pred_test, color='')\n",
-        "test_test = plt.scatter(y_test, y_test, color='g')\n",
-        "plt.legend((test_pred, test_test), ('prediction', 'truth'), loc='upper left', fontsize=8)\n",
+        "test_pred = plt.scatter(y_test, y_pred_test, color=\"\")\n",
+        "test_test = plt.scatter(y_test, y_test, color=\"g\")\n",
+        "plt.legend(\n",
+        "    (test_pred, test_test), (\"prediction\", \"truth\"), loc=\"upper left\", fontsize=8\n",
+        ")\n",
         "plt.show()"
       ]
     },

how-to-use-azureml/deployment/deploy-to-cloud/model-register-and-deploy.ipynb

@@ -82,7 +82,7 @@
       "source": [
         "## Create trained model\n",
         "\n",
-        "For this example, we will train a small model on scikit-learn's [diabetes dataset](https://scikit-learn.org/stable/datasets/index.html#diabetes-dataset). "
+        "For this example, we will train a small model on scikit-learn's [diabetes dataset](https://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_diabetes.html). "
       ]
     },
     {
@@ -279,7 +279,9 @@
         "\n",
         "\n",
         "environment = Environment('my-sklearn-environment')\n",
-        "environment.python.conda_dependencies = CondaDependencies.create(pip_packages=[\n",
+        "environment.python.conda_dependencies = CondaDependencies.create(conda_packages=[\n",
+        "    'pip==20.2.4'],\n",
+        "    pip_packages=[\n",
         "    'azureml-defaults',\n",
         "    'inference-schema[numpy-support]',\n",
         "    'joblib',\n",
@@ -478,7 +480,9 @@
         "\n",
         "\n",
         "environment = Environment('my-sklearn-environment')\n",
-        "environment.python.conda_dependencies = CondaDependencies.create(pip_packages=[\n",
+        "environment.python.conda_dependencies = CondaDependencies.create(conda_packages=[\n",
+        "    'pip==20.2.4'],\n",
+        "    pip_packages=[\n",
         "    'azureml-defaults',\n",
         "    'inference-schema[numpy-support]',\n",
         "    'joblib',\n",

how-to-use-azureml/deployment/deploy-to-local/register-model-deploy-local-advanced.ipynb

@@ -81,7 +81,7 @@
       "source": [
         "## Create trained model\n",
         "\n",
-        "For this example, we will train a small model on scikit-learn's [diabetes dataset](https://scikit-learn.org/stable/datasets/index.html#diabetes-dataset). "
+        "For this example, we will train a small model on scikit-learn's [diabetes dataset](https://scikit-learn.org/stable/datasets/toy_dataset.html#diabetes-dataset). "
       ]
     },
     {
@@ -263,7 +263,7 @@
         "\n",
         "# explicitly set base_image to None when setting base_dockerfile\n",
         "myenv.docker.base_image = None\n",
-        "myenv.docker.base_dockerfile = \"FROM mcr.microsoft.com/azureml/base:intelmpi2018.3-ubuntu16.04\\nRUN echo \\\"this is test\\\"\"\n",
+        "myenv.docker.base_dockerfile = \"FROM mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu20.04\\nRUN echo \\\"this is test\\\"\"\n",
         "myenv.inferencing_stack_version = \"latest\"\n",
         "\n",
         "inference_config = InferenceConfig(source_directory=source_directory,\n",

how-to-use-azureml/deployment/deploy-with-controlled-rollout/deploy-aks-with-controlled-rollout.ipynb

@@ -105,7 +105,9 @@
         "from azureml.core.conda_dependencies import CondaDependencies\n",
         "\n",
         "environment=Environment('my-sklearn-environment')\n",
-        "environment.python.conda_dependencies = CondaDependencies.create(pip_packages=[\n",
+        "environment.python.conda_dependencies = CondaDependencies.create(conda_packages=[\n",
+        "    'pip==20.2.4'],\n",
+        "    pip_packages=[\n",
         "    'azureml-defaults',\n",
         "    'inference-schema[numpy-support]',\n",
         "    'numpy',\n",

how-to-use-azureml/deployment/onnx/onnx-inference-facial-expression-recognition-deploy.ipynb

@@ -70,7 +70,7 @@
         "\n",
         "import urllib.request\n",
         "\n",
-        "onnx_model_url = \"https://github.com/onnx/models/blob/master/vision/body_analysis/emotion_ferplus/model/emotion-ferplus-7.tar.gz?raw=true\"\n",
+        "onnx_model_url = \"https://github.com/onnx/models/blob/main/vision/body_analysis/emotion_ferplus/model/emotion-ferplus-7.tar.gz?raw=true\"\n",
         "\n",
         "urllib.request.urlretrieve(onnx_model_url, filename=\"emotion-ferplus-7.tar.gz\")\n",
         "\n",

how-to-use-azureml/deployment/onnx/onnx-inference-mnist-deploy.ipynb

@@ -70,7 +70,7 @@
         "\n",
         "import urllib.request\n",
         "\n",
-        "onnx_model_url = \"https://github.com/onnx/models/blob/master/vision/classification/mnist/model/mnist-7.tar.gz?raw=true\"\n",
+        "onnx_model_url = \"https://github.com/onnx/models/blob/main/vision/classification/mnist/model/mnist-7.tar.gz?raw=true\"\n",
         "\n",
         "urllib.request.urlretrieve(onnx_model_url, filename=\"mnist-7.tar.gz\")"
       ]

how-to-use-azureml/explain-model/azure-integration/gpu-explanation/train-explain-model-gpu-tree-explainer.ipynb

@@ -106,7 +106,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "print(\"This notebook was created using version 1.36.0 of the Azure ML SDK\")\n",
+        "print(\"This notebook was created using version 1.41.0 of the Azure ML SDK\")\n",
         "print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
       ]
     },

how-to-use-azureml/explain-model/azure-integration/remote-explanation/explain-model-on-amlcompute.ipynb

@@ -249,6 +249,7 @@
       "source": [
         "from azureml.core.runconfig import RunConfiguration\n",
         "from azureml.core.conda_dependencies import CondaDependencies\n",
+        "import sys\n",
         "\n",
         "# Create a new RunConfig object\n",
         "run_config = RunConfiguration(framework=\"python\")\n",
@@ -260,6 +261,8 @@
         "    'azureml-defaults', 'azureml-telemetry', 'azureml-interpret'\n",
         "]\n",
         "\n",
+        "python_version = '{0}.{1}'.format(sys.version_info[0], sys.version_info[1])\n",
+        "\n",
         "# Note: this is to pin the scikit-learn and pandas versions to be same as notebook.\n",
         "# In production scenario user would choose their dependencies\n",
         "import pkg_resources\n",
@@ -283,7 +286,7 @@
         "# environment, otherwise if a model is trained or deployed in a different environment this can\n",
         "# cause errors.  Please take extra care when specifying your dependencies in a production environment.\n",
         "azureml_pip_packages.extend([sklearn_dep, pandas_dep])\n",
-        "run_config.environment.python.conda_dependencies = CondaDependencies.create(pip_packages=azureml_pip_packages)\n",
+        "run_config.environment.python.conda_dependencies = CondaDependencies.create(pip_packages=azureml_pip_packages,  python_version=python_version)\n",
         "\n",
         "from azureml.core import ScriptRunConfig\n",
         "\n",

how-to-use-azureml/explain-model/azure-integration/remote-explanation/explain-model-on-amlcompute.yml

@@ -11,4 +11,6 @@ dependencies:
   - matplotlib
   - azureml-dataset-runtime
   - ipywidgets
-  - raiwidgets~=0.10.0
+  - raiwidgets~=0.17.0
+  - itsdangerous==2.0.1
+  - markupsafe<2.1.0

how-to-use-azureml/explain-model/azure-integration/run-history/save-retrieve-explanations-run-history.yml

@@ -10,4 +10,7 @@ dependencies:
   - ipython
   - matplotlib
   - ipywidgets
-  - raiwidgets~=0.10.0
+  - raiwidgets~=0.17.0
+  - packaging>=20.9
+  - itsdangerous==2.0.1
+  - markupsafe<2.1.0

how-to-use-azureml/explain-model/azure-integration/scoring-time/train-explain-model-locally-and-deploy.ipynb

@@ -324,13 +324,15 @@
       "outputs": [],
       "source": [
         "from azureml.core.conda_dependencies import CondaDependencies \n",
+        "import sys\n",
         "\n",
         "# azureml-defaults is required to host the model as a web service.\n",
         "azureml_pip_packages = [\n",
         "    'azureml-defaults', 'azureml-core', 'azureml-telemetry',\n",
         "    'azureml-interpret'\n",
         "]\n",
-        " \n",
+        "\n",
+        "python_version = '{0}.{1}'.format(sys.version_info[0], sys.version_info[1])\n",
         "\n",
         "# Note: this is to pin the scikit-learn and pandas versions to be same as notebook.\n",
         "# In production scenario user would choose their dependencies\n",
@@ -354,7 +356,10 @@
         "# the submitted job is run in.  Note the remote environment(s) needs to be similar to the local\n",
         "# environment, otherwise if a model is trained or deployed in a different environment this can\n",
         "# cause errors.  Please take extra care when specifying your dependencies in a production environment.\n",
-        "myenv = CondaDependencies.create(pip_packages=['pyyaml', sklearn_dep, pandas_dep] + azureml_pip_packages)\n",
+        "myenv = CondaDependencies.create(\n",
+        "    python_version=python_version,\n",
+        "    conda_packages=['pip==20.2.4'],\n",
+        "    pip_packages=['pyyaml', sklearn_dep, pandas_dep] + azureml_pip_packages)\n",
         "\n",
         "with open(\"myenv.yml\",\"w\") as f:\n",
         "    f.write(myenv.serialize_to_string())\n",
@@ -387,7 +392,7 @@
         "\n",
         "\n",
         "aciconfig = AciWebservice.deploy_configuration(cpu_cores=1, \n",
-        "                                               memory_gb=1, \n",
+        "                                               memory_gb=2, \n",
         "                                               tags={\"data\": \"IBM_Attrition\",  \n",
         "                                                     \"method\" : \"local_explanation\"}, \n",
         "                                               description='Get local explanations for IBM Employee Attrition data')\n",
@@ -411,8 +416,8 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "import requests\n",
         "import json\n",
+        "from raiutils.webservice import post_with_retries\n",
         "\n",
         "\n",
         "# Create data to test service with\n",
@@ -424,7 +429,7 @@
         "\n",
         "# Send request to service\n",
         "print(\"POST to url\", service.scoring_uri)\n",
-        "resp = requests.post(service.scoring_uri, sample_data, headers=headers)\n",
+        "resp = post_with_retries(service.scoring_uri, sample_data, headers)\n",
         "\n",
         "# Can covert back to Python objects from json string if desired\n",
         "print(\"prediction:\", resp.text)\n",

how-to-use-azureml/explain-model/azure-integration/scoring-time/train-explain-model-locally-and-deploy.yml

@@ -10,4 +10,8 @@ dependencies:
   - ipython
   - matplotlib
   - ipywidgets
-  - raiwidgets~=0.10.0
+  - raiwidgets~=0.17.0
+  - packaging>=20.9
+  - itsdangerous==2.0.1
+  - markupsafe<2.1.0
+  - raiutils

how-to-use-azureml/explain-model/azure-integration/scoring-time/train-explain-model-on-amlcompute-and-deploy.ipynb

@@ -251,6 +251,7 @@
         "from azureml.core.runconfig import RunConfiguration\n",
         "from azureml.core.conda_dependencies import CondaDependencies\n",
         "from azureml.core.runconfig import DEFAULT_CPU_IMAGE\n",
+        "import sys\n",
         "\n",
         "# Create a new runconfig object\n",
         "run_config = RunConfiguration()\n",
@@ -268,7 +269,7 @@
         "    'azureml-defaults', 'azureml-telemetry', 'azureml-interpret'\n",
         "]\n",
         " \n",
-        "\n",
+        "python_version = '{0}.{1}'.format(sys.version_info[0], sys.version_info[1])\n",
         "\n",
         "# Note: this is to pin the scikit-learn version to be same as notebook.\n",
         "# In production scenario user would choose their dependencies\n",
@@ -293,7 +294,10 @@
         "# environment, otherwise if a model is trained or deployed in a different environment this can\n",
         "# cause errors.  Please take extra care when specifying your dependencies in a production environment.\n",
         "azureml_pip_packages.extend(['pyyaml', sklearn_dep, pandas_dep])\n",
-        "run_config.environment.python.conda_dependencies = CondaDependencies.create(pip_packages=azureml_pip_packages)\n",
+        "run_config.environment.python.conda_dependencies = CondaDependencies.create(\n",
+        "    python_version=python_version,\n",
+        "    pip_packages=azureml_pip_packages)\n",
+        "\n",
         "# Now submit a run on AmlCompute\n",
         "from azureml.core.script_run_config import ScriptRunConfig\n",
         "\n",
@@ -453,7 +457,7 @@
         "# environment, otherwise if a model is trained or deployed in a different environment this can\n",
         "# cause errors.  Please take extra care when specifying your dependencies in a production environment.\n",
         "azureml_pip_packages.extend(['pyyaml', sklearn_dep, pandas_dep])\n",
-        "myenv = CondaDependencies.create(pip_packages=azureml_pip_packages)\n",
+        "myenv = CondaDependencies.create(python_version=python_version, pip_packages=azureml_pip_packages)\n",
         "\n",
         "with open(\"myenv.yml\",\"w\") as f:\n",
         "    f.write(myenv.serialize_to_string())\n",
@@ -509,7 +513,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "import requests\n",
+        "from raiutils.webservice import post_with_retries\n",
         "\n",
         "# Create data to test service with\n",
         "examples = x_test[:4]\n",
@@ -519,7 +523,7 @@
         "\n",
         "# Send request to service\n",
         "print(\"POST to url\", service.scoring_uri)\n",
-        "resp = requests.post(service.scoring_uri, input_data, headers=headers)\n",
+        "resp = post_with_retries(service.scoring_uri, input_data, headers)\n",
         "\n",
         "# Can covert back to Python objects from json string if desired\n",
         "print(\"prediction:\", resp.text)"

how-to-use-azureml/explain-model/azure-integration/scoring-time/train-explain-model-on-amlcompute-and-deploy.yml

@@ -12,4 +12,7 @@ dependencies:
   - azureml-dataset-runtime
   - azureml-core
   - ipywidgets
-  - raiwidgets~=0.10.0
+  - raiwidgets~=0.17.0
+  - itsdangerous==2.0.1
+  - markupsafe<2.1.0
+  - raiutils

how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-getting-started.ipynb

@@ -63,6 +63,8 @@
       "outputs": [],
       "source": [
         "import os\n",
+        "import requests\n",
+        "import tempfile\n",
         "import azureml.core\n",
         "from azureml.core import Workspace, Experiment, Datastore\n",
         "from azureml.widgets import RunDetails\n",
@@ -158,9 +160,14 @@
       "metadata": {},
       "outputs": [],
       "source": [
+        "# download data file from remote\n",
+        "response = requests.get(\"https://dprepdata.blob.core.windows.net/demo/Titanic.csv\")\n",
+        "titanic_file = os.path.join(tempfile.mkdtemp(), \"Titanic.csv\")\n",
+        "with open(titanic_file, \"w\") as f:\n",
+        "    f.write(response.content.decode(\"utf-8\"))\n",
         "# get_default_datastore() gets the default Azure Blob Store associated with your workspace.\n",
         "# Here we are reusing the def_blob_store object we obtained earlier\n",
-        "def_blob_store.upload_files([\"./20news.pkl\"], target_path=\"20newsgroups\", overwrite=True)\n",
+        "def_blob_store.upload_files([titanic_file], target_path=\"titanic\", overwrite=True)\n",
         "print(\"Upload call completed\")"
       ]
     },
@@ -286,7 +293,7 @@
         "- [**AzureBatchStep**](https://docs.microsoft.com/en-us/python/api/azureml-pipeline-steps/azureml.pipeline.steps.azurebatch_step.azurebatchstep?view=azure-ml-py): Creates a step for submitting jobs to Azure Batch\n",
         "- [**EstimatorStep**](https://docs.microsoft.com/en-us/python/api/azureml-pipeline-steps/azureml.pipeline.steps.estimator_step.estimatorstep?view=azure-ml-py): Adds a step to run Estimator in a Pipeline.\n",
         "- [**MpiStep**](https://docs.microsoft.com/en-us/python/api/azureml-pipeline-steps/azureml.pipeline.steps.mpi_step.mpistep?view=azure-ml-py): Adds a step to run a MPI job in a Pipeline.\n",
-        "- [**AutoMLStep**](https://docs.microsoft.com/en-us/python/api/azureml-train-automl/azureml.train.automl.automlstep?view=azure-ml-py): Creates a AutoML step in a Pipeline.\n",
+        "- [**AutoMLStep**](https://docs.microsoft.com/en-us/python/api/azureml-pipeline-steps/azureml.pipeline.steps.automlstep?view=azure-ml-py): Creates a AutoML step in a Pipeline.\n",
         "\n",
         "The following code will create a PythonScriptStep to be executed in the Azure Machine Learning Compute we created above using train.py, one of the files already made available in the `source_directory`.\n",
         "\n",

how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-publish-and-run-using-rest-endpoint.ipynb

@@ -120,8 +120,10 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "# Uploading data to the datastore\n",
-        "data_path = def_blob_store.upload_files([\"./20news.pkl\"], target_path=\"20newsgroups\", overwrite=True)"
+        "# Specify a public dataset path\n",
+        "data_path = \"https://dprepdata.blob.core.windows.net/demo/Titanic.csv\"\n",
+        "# Or uploading data to the datastore\n",
+        "# data_path = def_blob_store.upload_files([\"./your_data.pkl\"], target_path=\"your_path\", overwrite=True)"
       ]
     },
     {
@@ -400,11 +402,11 @@
       "source": [
         "try:\n",
         "    response.raise_for_status()\n",
-        "except Exception:    \n",
+        "except Exception as ex:    \n",
         "    raise Exception('Received bad response from the endpoint: {}\\n'\n",
         "                    'Response Code: {}\\n'\n",
         "                    'Headers: {}\\n'\n",
-        "                    'Content: {}'.format(rest_endpoint, response.status_code, response.headers, response.content))\n",
+        "                    'Content: {}'.format(rest_endpoint1, response.status_code, response.headers, response.content)) from ex\n",
         "\n",
         "run_id = response.json().get('Id')\n",
         "print('Submitted pipeline run: ', run_id)"

how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-use-databricks-as-compute-target.ipynb

@@ -875,7 +875,12 @@
         "\n",
         "def populate_environ():\n",
         "    parser = argparse.ArgumentParser(description='Process arguments passed to script')\n",
+        "\n",
+        "    # The AZUREML_SCRIPT_DIRECTORY_NAME argument will be filled in if the DatabricksStep\n",
+        "    # was run using a local source_directory and python_script_name\n",
         "    parser.add_argument('--AZUREML_SCRIPT_DIRECTORY_NAME')\n",
+        "\n",
+        "    # Remaining arguments are filled in for all databricks jobs and can be used to build the run context\n",
         "    parser.add_argument('--AZUREML_RUN_TOKEN')\n",
         "    parser.add_argument('--AZUREML_RUN_TOKEN_EXPIRY')\n",
         "    parser.add_argument('--AZUREML_RUN_ID')\n",
@@ -884,9 +889,10 @@
         "    parser.add_argument('--AZUREML_ARM_WORKSPACE_NAME')\n",
         "    parser.add_argument('--AZUREML_ARM_PROJECT_NAME')\n",
         "    parser.add_argument('--AZUREML_SERVICE_ENDPOINT')\n",
+        "    parser.add_argument('--AZUREML_WORKSPACE_ID')\n",
+        "    parser.add_argument('--AZUREML_EXPERIMENT_ID')\n",
         "\n",
-        "    args = parser.parse_args()\n",
-        "    os.environ['AZUREML_SCRIPT_DIRECTORY_NAME'] = args.AZUREML_SCRIPT_DIRECTORY_NAME\n",
+        "    (args, extra_args) = parser.parse_known_args()\n",
         "    os.environ['AZUREML_RUN_TOKEN'] = args.AZUREML_RUN_TOKEN\n",
         "    os.environ['AZUREML_RUN_TOKEN_EXPIRY'] = args.AZUREML_RUN_TOKEN_EXPIRY\n",
         "    os.environ['AZUREML_RUN_ID'] = args.AZUREML_RUN_ID\n",
@@ -895,10 +901,12 @@
         "    os.environ['AZUREML_ARM_WORKSPACE_NAME'] = args.AZUREML_ARM_WORKSPACE_NAME\n",
         "    os.environ['AZUREML_ARM_PROJECT_NAME'] = args.AZUREML_ARM_PROJECT_NAME\n",
         "    os.environ['AZUREML_SERVICE_ENDPOINT'] = args.AZUREML_SERVICE_ENDPOINT\n",
+        "    os.environ['AZUREML_WORKSPACE_ID'] = args.AZUREML_WORKSPACE_ID\n",
+        "    os.environ['AZUREML_EXPERIMENT_ID'] = args.AZUREML_EXPERIMENT_ID\n",
         "\n",
         "populate_environ()\n",
         "run = Run.get_context(allow_offline=False)\n",
-        "print(run._run_dto[\"parent_run_id\"])\n",
+        "print(run.parent.id)\n",
         "```"
       ]
     },
@@ -947,7 +955,7 @@
       "name": "python",
       "nbconvert_exporter": "python",
       "pygments_lexer": "ipython3",
-      "version": "3.6.2"
+      "version": "3.7.9"
     },
     "order_index": 5,
     "star_tag": [

how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-with-data-dependency-steps.ipynb

@@ -213,7 +213,7 @@
         "blob_input_data = DataReference(\n",
         "    datastore=def_blob_store,\n",
         "    data_reference_name=\"test_data\",\n",
-        "    path_on_datastore=\"20newsgroups/20news.pkl\")\n",
+        "    path_on_datastore=\"titanic/Titanic.csv\")\n",
         "print(\"DataReference object created\")"
       ]
     },
@@ -382,7 +382,7 @@
         "from azureml.pipeline.core import PipelineParameter\n",
         "from azureml.data.datapath import DataPath, DataPathComputeBinding\n",
         "\n",
-        "datapath = DataPath(datastore=def_blob_store, path_on_datastore='20newsgroups/20news.pkl')\n",
+        "datapath = DataPath(datastore=def_blob_store, path_on_datastore='titanic/Titanic.csv')\n",
         "datapath_param = PipelineParameter(name=\"compare_data\", default_value=datapath)\n",
         "data_parameter1 = (datapath_param, DataPathComputeBinding(mode='mount'))"
       ]

how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-with-notebook-runner-step.ipynb

@@ -42,9 +42,7 @@
         "Advantages of running your notebook as a step in pipeline:\n",
         "1. Run your notebook like a python script without converting into .py files, leveraging complete end to end experience of Azure Machine Learning Pipelines.\n",
         "2. Use pipeline intermediate data to and from the notebook along with other steps in pipeline.\n",
-        "3. Parameterize your notebook with [Pipeline Parameters](./aml-pipelines-publish-and-run-using-rest-endpoint.ipynb).\n",
-        "\n",
-        "Try some more [quick start notebooks](https://github.com/microsoft/recommenders/tree/master/notebooks/00_quick_start) with `NotebookRunnerStep`."
+        "3. Parameterize your notebook with [Pipeline Parameters](./aml-pipelines-publish-and-run-using-rest-endpoint.ipynb).\n"
       ]
     },
     {
@@ -61,6 +59,8 @@
       "outputs": [],
       "source": [
         "import os\n",
+        "import requests\n",
+        "import tempfile\n",
         "\n",
         "import azureml.core\n",
         "\n",
@@ -114,7 +114,12 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "Datastore.get(ws, \"workspaceblobstore\").upload_files([\"./20news.pkl\"], target_path=\"20newsgroups\", overwrite=True)\n",
+        "# download data file from remote\n",
+        "response = requests.get(\"https://dprepdata.blob.core.windows.net/demo/Titanic.csv\")\n",
+        "titanic_file = os.path.join(tempfile.mkdtemp(), \"Titanic.csv\")\n",
+        "with open(titanic_file, \"w\") as f:\n",
+        "    f.write(response.content.decode(\"utf-8\"))\n",
+        "Datastore.get(ws, \"workspaceblobstore\").upload_files([titanic_file], target_path=\"titanic\", overwrite=True)\n",
         "print(\"Upload call completed\")"
       ]
     },
@@ -227,7 +232,7 @@
         "input_data = DataReference(\n",
         "    datastore=Datastore.get(ws, \"workspaceblobstore\"),\n",
         "    data_reference_name=\"blob_test_data\",\n",
-        "    path_on_datastore=\"20newsgroups/20news.pkl\")\n",
+        "    path_on_datastore=\"titanic/Titanic.csv\")\n",
         "\n",
         "output_data = PipelineData(name=\"processed_data\",\n",
         "                           datastore=Datastore.get(ws, \"workspaceblobstore\"))"

how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/publish_run_extract/extract.py

@@ -20,7 +20,7 @@ if not (args.output_extract is None):
     os.makedirs(args.output_extract, exist_ok=True)
     print("%s created" % args.output_extract)
 
-with open(os.path.join(args.input_extract, '20news.pkl'), 'rb') as f:
+with open(os.path.join(args.input_extract, 'Titanic.csv'), 'rb') as f:
     content = f.read()
-    with open(os.path.join(args.output_extract, '20news.pkl'), 'wb') as fw:
+    with open(os.path.join(args.output_extract, 'Titanic.csv'), 'wb') as fw:
         fw.write(content)

how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/publish_run_train/train.py

@@ -21,7 +21,7 @@ if not (args.output_train is None):
     os.makedirs(args.output_train, exist_ok=True)
     print("%s created" % args.output_train)
 
-with open(os.path.join(args.input_data, '20news.pkl'), 'rb') as f:
+with open(os.path.join(args.input_data), 'rb') as f:
     content = f.read()
-    with open(os.path.join(args.output_train, '20news.pkl'), 'wb') as fw:
+    with open(os.path.join(args.output_train, 'Titanic.csv'), 'wb') as fw:
         fw.write(content)

how-to-use-azureml/machine-learning-pipelines/nyc-taxi-data-regression-model-building/scripts/prepdata/cleanse.py

@@ -5,17 +5,6 @@ import argparse
 import os
 from azureml.core import Run
 
-
-def get_dict(dict_str):
-    pairs = dict_str.strip("{}").split("\;")
-    new_dict = {}
-    for pair in pairs:
-        key, value = pair.strip().split(":")
-        new_dict[key.strip().strip("'")] = value.strip().strip("'")
-
-    return new_dict
-
-
 print("Cleans the input data")
 
 # Get the input green_taxi_data. To learn more about how to access dataset in your script, please
@@ -23,7 +12,6 @@ print("Cleans the input data")
 run = Run.get_context()
 raw_data = run.input_datasets["raw_data"]
 
-
 parser = argparse.ArgumentParser("cleanse")
 parser.add_argument("--output_cleanse", type=str, help="cleaned taxi data directory")
 parser.add_argument("--useful_columns", type=str, help="useful columns to keep")
@@ -31,15 +19,15 @@ parser.add_argument("--columns", type=str, help="rename column pattern")
 
 args = parser.parse_args()
 
-print("Argument 1(columns to keep): %s" % str(args.useful_columns.strip("[]").split("\;")))
-print("Argument 2(columns renaming mapping): %s" % str(args.columns.strip("{}").split("\;")))
+print("Argument 1(columns to keep): %s" % str(args.useful_columns.strip("[]").split(r'\;')))
+print("Argument 2(columns renaming mapping): %s" % str(args.columns.strip("{}").split(r'\;')))
 print("Argument 3(output cleansed taxi data path): %s" % args.output_cleanse)
 
 # These functions ensure that null data is removed from the dataset,
 # which will help increase machine learning model accuracy.
 
-useful_columns = [s.strip().strip("'") for s in args.useful_columns.strip("[]").split("\;")]
-columns = get_dict(args.columns)
+useful_columns = eval(args.useful_columns.replace(';', ','))
+columns = eval(args.columns.replace(';', ','))
 
 new_df = (raw_data.to_pandas_dataframe()
           .dropna(how='all')

how-to-use-azureml/machine-learning-pipelines/nyc-taxi-data-regression-model-building/scripts/prepdata/filter.py

@@ -29,14 +29,14 @@ print("Argument (output filtered taxi data path): %s" % args.output_filter)
 combined_df = combined_df.astype({"pickup_longitude": 'float64', "pickup_latitude": 'float64',
                                   "dropoff_longitude": 'float64', "dropoff_latitude": 'float64'})
 
-latlong_filtered_df = combined_df[(combined_df.pickup_longitude <= -73.72) &
-                                  (combined_df.pickup_longitude >= -74.09) &
-                                  (combined_df.pickup_latitude <= 40.88) &
-                                  (combined_df.pickup_latitude >= 40.53) &
-                                  (combined_df.dropoff_longitude <= -73.72) &
-                                  (combined_df.dropoff_longitude >= -74.72) &
-                                  (combined_df.dropoff_latitude <= 40.88) &
-                                  (combined_df.dropoff_latitude >= 40.53)]
+latlong_filtered_df = combined_df[(combined_df.pickup_longitude <= -73.72)
+                                  & (combined_df.pickup_longitude >= -74.09)
+                                  & (combined_df.pickup_latitude <= 40.88)
+                                  & (combined_df.pickup_latitude >= 40.53)
+                                  & (combined_df.dropoff_longitude <= -73.72)
+                                  & (combined_df.dropoff_longitude >= -74.72)
+                                  & (combined_df.dropoff_latitude <= 40.88)
+                                  & (combined_df.dropoff_latitude >= 40.53)]
 
 latlong_filtered_df.reset_index(inplace=True, drop=True)
 

how-to-use-azureml/machine-learning-pipelines/nyc-taxi-data-regression-model-building/scripts/trainmodel/train_test_split.py

@@ -1,6 +1,6 @@
 import argparse
 import os
-import azureml.core
+# import azureml.core
 from azureml.core import Run
 from sklearn.model_selection import train_test_split
 
@@ -32,7 +32,7 @@ output_split_train, output_split_test = train_test_split(transformed_df, test_si
 output_split_train.reset_index(inplace=True, drop=True)
 output_split_test.reset_index(inplace=True, drop=True)
 
-if not (args.output_split_train is None and
-        args.output_split_test is None):
+if not (args.output_split_train
+        is None and args.output_split_test is None):
     write_output(output_split_train, args.output_split_train)
     write_output(output_split_test, args.output_split_test)

how-to-use-azureml/ml-frameworks/chainer/train-hyperparameter-tune-deploy-with-chainer/train-hyperparameter-tune-deploy-with-chainer.ipynb

@@ -254,6 +254,7 @@
         "- conda-forge\n",
         "dependencies:\n",
         "- python=3.6.2\n",
+        "- pip=21.3.1\n",
         "- pip:\n",
         "  - azureml-defaults\n",
         "  - azureml-opendatasets\n",
@@ -587,7 +588,7 @@
         "\n",
         "aciconfig = AciWebservice.deploy_configuration(cpu_cores=1,\n",
         "                                               auth_enabled=True, # this flag generates API keys to secure access\n",
-        "                                               memory_gb=1,\n",
+        "                                               memory_gb=2,\n",
         "                                               tags={'name': 'mnist', 'framework': 'Chainer'},\n",
         "                                               description='Chainer DNN with MNIST')\n",
         "\n",

how-to-use-azureml/ml-frameworks/fastai/fastai-with-custom-docker/fastai-with-custom-docker.ipynb

@@ -163,7 +163,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "fastai_env.docker.base_image = \"fastdotai/fastai:latest\"\n",
+        "fastai_env.docker.base_image = \"fastdotai/fastai:2021-02-11\"\n",
         "fastai_env.python.user_managed_dependencies = True"
       ]
     },
@@ -199,7 +199,7 @@
         "Specify docker steps as a string:\n",
         "```python \n",
         "dockerfile = r\"\"\" \\\n",
-        "FROM mcr.microsoft.com/azureml/base:intelmpi2018.3-ubuntu16.04\n",
+        "FROM mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu20.04\n",
         "RUN echo \"Hello from custom container!\" \\\n",
         "\"\"\"\n",
         "```\n",

how-to-use-azureml/ml-frameworks/keras/train-hyperparameter-tune-deploy-with-keras/train-hyperparameter-tune-deploy-with-keras.ipynb

@@ -431,6 +431,7 @@
         "- conda-forge\n",
         "dependencies:\n",
         "- python=3.6.2\n",
+        "- pip=21.3.1\n",
         "- pip:\n",
         "  - h5py<=2.10.0\n",
         "  - azureml-defaults\n",

how-to-use-azureml/ml-frameworks/pytorch/train-hyperparameter-tune-deploy-with-pytorch/train-hyperparameter-tune-deploy-with-pytorch.ipynb

@@ -262,6 +262,7 @@
         "- conda-forge\n",
         "dependencies:\n",
         "- python=3.6.2\n",
+        "- pip=21.3.1\n",
         "- pip:\n",
         "  - azureml-defaults\n",
         "  - torch==1.6.0\n",

how-to-use-azureml/ml-frameworks/pytorch/train-hyperparameter-tune-deploy-with-pytorch/train-hyperparameter-tune-deploy-with-pytorch.yml

@@ -6,5 +6,5 @@ dependencies:
   - pillow==5.4.1
   - matplotlib
   - numpy==1.19.3
-  - https://download.pytorch.org/whl/cpu/torch-1.6.0%2Bcpu-cp36-cp36m-win_amd64.whl
-  - https://download.pytorch.org/whl/cpu/torchvision-0.7.0%2Bcpu-cp36-cp36m-win_amd64.whl
+  - https://download.pytorch.org/whl/cpu/torch-1.6.0%2Bcpu-cp38-cp38-win_amd64.whl
+  - https://download.pytorch.org/whl/cpu/torchvision-0.7.0%2Bcpu-cp38-cp38-win_amd64.whl

how-to-use-azureml/ml-frameworks/using-mlflow/train-and-deploy-pytorch/scripts/train.py

@@ -103,15 +103,14 @@ device = torch.device("cuda" if use_cuda else "cpu")
 
 kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
 # Use Azure Open Datasets for MNIST dataset
+datasets.MNIST.mirrors = [
+    "https://azureopendatastorage.azurefd.net/mnist/"
+]
 datasets.MNIST.resources = [
-    ("https://azureopendatastorage.azurefd.net/mnist/train-images-idx3-ubyte.gz",
-     "f68b3c2dcbeaaa9fbdd348bbdeb94873"),
-    ("https://azureopendatastorage.azurefd.net/mnist/train-labels-idx1-ubyte.gz",
-     "d53e105ee54ea40749a09fcbcd1e9432"),
-    ("https://azureopendatastorage.azurefd.net/mnist/t10k-images-idx3-ubyte.gz",
-     "9fb629c4189551a2d022fa330f9573f3"),
-    ("https://azureopendatastorage.azurefd.net/mnist/t10k-labels-idx1-ubyte.gz",
-     "ec29112dd5afa0611ce80d1b7f02629c")
+    ("train-images-idx3-ubyte.gz", "f68b3c2dcbeaaa9fbdd348bbdeb94873"),
+    ("train-labels-idx1-ubyte.gz", "d53e105ee54ea40749a09fcbcd1e9432"),
+    ("t10k-images-idx3-ubyte.gz", "9fb629c4189551a2d022fa330f9573f3"),
+    ("t10k-labels-idx1-ubyte.gz", "ec29112dd5afa0611ce80d1b7f02629c")
 ]
 train_loader = torch.utils.data.DataLoader(
     datasets.MNIST('../data', train=True, download=True,

how-to-use-azureml/reinforcement-learning/README.md

@@ -1,4 +1,7 @@
 
+# Important Note
+Azure Machine Learning reinforcement learning via the `azureml.contrib.train.rl` package that is used on this page will no longer be supported after June 2022. We recommend customers use Ray-on-AML library to facilitate execution of reinforcement learning experiments on Azure Machine Learning. The sample notebooks referenced in [this section](#contents) are updated accordingly to use Ray on AML library.
+
 # Azure Machine Learning - Reinforcement Learning (Public Preview)
 
 <!-- 

how-to-use-azureml/reinforcement-learning/atari-on-distributed-compute/docker/Dockerfile-cpu

@@ -0,0 +1,16 @@
+FROM mcr.microsoft.com/azureml/openmpi3.1.2-ubuntu18.04
+
+RUN pip install ray-on-aml==0.1.6
+RUN pip install gym[atari]==0.19.0
+RUN pip install gym[accept-rom-license]==0.19.0
+RUN pip install ale-py==0.7.0
+RUN pip install azureml-core
+RUN pip install ray==0.8.7
+RUN pip install ray[rllib,tune,serve]==0.8.7
+RUN pip install tensorflow==1.14.0
+
+USER root
+
+RUN apt-get update
+RUN apt-get install -y jq
+RUN apt-get install -y rsync

how-to-use-azureml/reinforcement-learning/atari-on-distributed-compute/docker/Dockerfile-gpu

@@ -0,0 +1,72 @@
+FROM mcr.microsoft.com/azureml/openmpi4.1.0-cuda11.0.3-cudnn8-ubuntu18.04:20211111.v1
+
+# CUDA repository key rotation: https://forums.developer.nvidia.com/t/notice-cuda-linux-repository-key-rotation/212771
+RUN apt-key del 7fa2af80
+ENV distro ubuntu1804
+ENV arch x86_64
+RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/$distro/$arch/3bf863cc.pub
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    python-opengl \
+    rsync \
+    xvfb && \
+    apt-get clean -y && \
+    rm -rf /var/lib/apt/lists/* && \
+    rm -rf /usr/share/man/*
+
+ENV AZUREML_CONDA_ENVIRONMENT_PATH /azureml-envs/tensorflow-2.4
+
+# Create conda environment
+RUN conda create -p $AZUREML_CONDA_ENVIRONMENT_PATH \
+    python=3.7 pip=20.2.4
+
+# Prepend path to AzureML conda environment
+ENV PATH $AZUREML_CONDA_ENVIRONMENT_PATH/bin:$PATH
+
+RUN pip --version
+RUN python --version
+
+# Install ray-on-aml
+RUN pip install 'ray-on-aml==0.1.6'
+
+RUN pip install ray==0.8.7
+RUN pip install gym[atari]==0.19.0
+RUN pip install gym[accept-rom-license]==0.19.0
+
+# Install pip dependencies
+RUN HOROVOD_WITH_TENSORFLOW=1 \
+    pip install 'matplotlib>=3.3,<3.4' \
+                'psutil>=5.8,<5.9' \
+                'tqdm>=4.59,<4.60' \
+                'pandas>=1.1,<1.2' \
+                'scipy>=1.5,<1.6' \
+                'numpy>=1.10,<1.20' \
+                'ipykernel~=6.0' \
+                'azureml-core==1.36.0.post2' \
+                'azureml-defaults==1.36.0' \
+                'azureml-mlflow==1.36.0' \
+                'azureml-telemetry==1.36.0' \
+                'tensorboard==2.4.0' \
+                'tensorflow-gpu==2.4.1' \
+                'tensorflow-datasets==4.3.0' \
+                'onnxruntime-gpu>=1.7,<1.8' \
+                'horovod[tensorflow-gpu]==0.21.3'
+
+RUN pip install --no-cache-dir \
+    azureml-defaults \
+    azureml-dataset-runtime[fuse,pandas] \
+    azureml-contrib-reinforcementlearning \
+    gputil \
+    cloudpickle==1.3.0 \
+    tabulate \
+    dm_tree \
+    lz4 \
+    psutil \
+    setproctitle
+
+# This is required for ray 0.8.7
+RUN pip install -U aiohttp==3.7.4
+
+# This is needed for mpi to locate libpython
+ENV LD_LIBRARY_PATH $AZUREML_CONDA_ENVIRONMENT_PATH/lib:$LD_LIBRARY_PATH
+

how-to-use-azureml/reinforcement-learning/atari-on-distributed-compute/files/pong_rllib.py

@@ -1,40 +1,35 @@
-import ray
+from ray_on_aml.core import Ray_On_AML
+
 import ray.tune as tune
 from ray.rllib import train
 
-import os
-import sys
-
-from azureml.core import Run
 from utils import callbacks
 
-DEFAULT_RAY_ADDRESS = 'localhost:6379'
-
 if __name__ == "__main__":
 
-    # Parse arguments
-    train_parser = train.create_parser()
+    ray_on_aml = Ray_On_AML()
+    ray = ray_on_aml.getRay()
+    if ray:  # in the headnode
+        # Parse arguments
+        train_parser = train.create_parser()
 
-    args = train_parser.parse_args()
-    print("Algorithm config:", args.config)
+        args = train_parser.parse_args()
+        print("Algorithm config:", args.config)
 
-    if args.ray_address is None:
-        args.ray_address = DEFAULT_RAY_ADDRESS
-
-    ray.init(address=args.ray_address)
-
-    tune.run(
-        run_or_experiment=args.run,
-        config={
-            "env": args.env,
-            "num_gpus": args.config["num_gpus"],
-            "num_workers": args.config["num_workers"],
-            "callbacks": {"on_train_result": callbacks.on_train_result},
-            "sample_batch_size": 50,
-            "train_batch_size": 1000,
-            "num_sgd_iter": 2,
-            "num_data_loader_buffers": 2,
-            "model": {"dim": 42},
-        },
-        stop=args.stop,
-        local_dir='./logs')
+        tune.run(
+            run_or_experiment=args.run,
+            config={
+                "env": args.env,
+                "num_gpus": args.config["num_gpus"],
+                "num_workers": args.config["num_workers"],
+                "callbacks": {"on_train_result": callbacks.on_train_result},
+                "sample_batch_size": 50,
+                "train_batch_size": 1000,
+                "num_sgd_iter": 2,
+                "num_data_loader_buffers": 2,
+                "model": {"dim": 42},
+            },
+            stop=args.stop,
+            local_dir='./logs')
+    else:
+        print("in worker node")

how-to-use-azureml/reinforcement-learning/atari-on-distributed-compute/files/utils/callbacks.py

@@ -8,7 +8,7 @@ from azureml.core import Run
 def on_train_result(info):
     '''Callback on train result to record metrics returned by trainer.
     '''
-    run = Run.get_context().parent
+    run = Run.get_context()
     run.log(
         name='episode_reward_mean',
         value=info["result"]["episode_reward_mean"])

how-to-use-azureml/reinforcement-learning/atari-on-distributed-compute/pong_rllib.ipynb

@@ -84,7 +84,11 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646081765827
+        }
+      },
       "outputs": [],
       "source": [
         "%matplotlib inline\n",
@@ -93,7 +97,7 @@
         "import azureml.core\n",
         "\n",
         "# Check core SDK version number\n",
-        "print(\"Azure Machine Learning SDK Version: \", azureml.core.VERSION)"
+        "print(\"Azure Machine Learning SDK version: \", azureml.core.VERSION)"
       ]
     },
     {
@@ -107,7 +111,11 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646081772340
+        }
+      },
       "outputs": [],
       "source": [
         "from azureml.core import Workspace\n",
@@ -127,7 +135,11 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646081775643
+        }
+      },
       "outputs": [],
       "source": [
         "from azureml.core.experiment import Experiment\n",
@@ -137,180 +149,13 @@
         "exp = Experiment(workspace=ws, name=experiment_name)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Create Virtual Network and Network Security Group\n",
-        "\n",
-        "**If you are using separate compute targets for the Ray head and worker, as we do in this notebook**, a virtual network must be created in the resource group.  If you have already created a virtual network in the resource group, you can skip this step.\n",
-        "\n",
-        "> Note that your user role must have permissions to create and manage virtual networks to run the cells below. Talk to your IT admin if you do not have these permissions.\n",
-        "\n",
-        "#### Create Virtual Network\n",
-        "To create the virtual network you first must install the [Azure Networking Python API](https://docs.microsoft.com/python/api/overview/azure/network?view=azure-python).\n",
-        "\n",
-        "`pip install --upgrade azure-mgmt-network`\n",
-        "\n",
-        "Note: In this section we are using [DefaultAzureCredential](https://docs.microsoft.com/python/api/azure-identity/azure.identity.defaultazurecredential?view=azure-python)\n",
-        "class for authentication which, by default, examines several options in turn, and stops on the first option that provides\n",
-        "a token. You will need to log in using Azure CLI, if none of the other options are available (please find more details [here](https://docs.microsoft.com/python/api/azure-identity/azure.identity.defaultazurecredential?view=azure-python))."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "# If you need to install the Azure Networking SDK, uncomment the following line.\n",
-        "#!pip install --upgrade azure-mgmt-network"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "from azure.mgmt.network import NetworkManagementClient\n",
-        "from azure.identity import DefaultAzureCredential\n",
-        "\n",
-        "# Virtual network name\n",
-        "vnet_name =\"rl_pong_vnet\"\n",
-        "\n",
-        "# Default subnet\n",
-        "subnet_name =\"default\"\n",
-        "\n",
-        "# The Azure subscription you are using\n",
-        "subscription_id=ws.subscription_id\n",
-        "\n",
-        "# The resource group for the reinforcement learning cluster\n",
-        "resource_group=ws.resource_group\n",
-        "\n",
-        "# Azure region of the resource group\n",
-        "location=ws.location\n",
-        "\n",
-        "network_client = NetworkManagementClient(credential=DefaultAzureCredential(), subscription_id=subscription_id)\n",
-        "\n",
-        "async_vnet_creation = network_client.virtual_networks.begin_create_or_update(\n",
-        "    resource_group,\n",
-        "    vnet_name,\n",
-        "    {\n",
-        "        'location': location,\n",
-        "        'address_space': {\n",
-        "            'address_prefixes': ['10.0.0.0/16']\n",
-        "        }\n",
-        "    }\n",
-        ")\n",
-        "\n",
-        "async_vnet_creation.wait()\n",
-        "print(\"Virtual network created successfully: \", async_vnet_creation.result())"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Set up Network Security Group on Virtual Network\n",
-        "\n",
-        "Depending on your Azure setup, you may need to open certain ports to make it possible for Azure to manage the compute targets that you create.  The ports that need to be opened are described [here](https://docs.microsoft.com/azure/machine-learning/how-to-enable-virtual-network).\n",
-        "\n",
-        "A common situation is that ports `29876-29877` are closed.  The following code will add a security rule to open these ports.    Or you can do this manually in the [Azure portal](https://portal.azure.com).\n",
-        "\n",
-        "You may need to modify the code below to match your scenario."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "import azure.mgmt.network.models\n",
-        "\n",
-        "security_group_name = vnet_name + '-' + \"nsg\"\n",
-        "security_rule_name = \"AllowAML\"\n",
-        "\n",
-        "# Create a network security group\n",
-        "nsg_params = azure.mgmt.network.models.NetworkSecurityGroup(\n",
-        "    location=location,\n",
-        "    security_rules=[\n",
-        "        azure.mgmt.network.models.SecurityRule(\n",
-        "            name=security_rule_name,\n",
-        "            access=azure.mgmt.network.models.SecurityRuleAccess.allow,\n",
-        "            description='Reinforcement Learning in Azure Machine Learning rule',\n",
-        "            destination_address_prefix='*',\n",
-        "            destination_port_range='29876-29877',\n",
-        "            direction=azure.mgmt.network.models.SecurityRuleDirection.inbound,\n",
-        "            priority=400,\n",
-        "            protocol=azure.mgmt.network.models.SecurityRuleProtocol.tcp,\n",
-        "            source_address_prefix='BatchNodeManagement',\n",
-        "            source_port_range='*'\n",
-        "        ),\n",
-        "    ],\n",
-        ")\n",
-        "\n",
-        "async_nsg_creation = network_client.network_security_groups.begin_create_or_update(\n",
-        "    resource_group,\n",
-        "    security_group_name,\n",
-        "    nsg_params,\n",
-        ")\n",
-        "\n",
-        "async_nsg_creation.wait() \n",
-        "print(\"Network security group created successfully:\", async_nsg_creation.result())\n",
-        "\n",
-        "network_security_group = network_client.network_security_groups.get(\n",
-        "    resource_group,\n",
-        "    security_group_name,\n",
-        ")\n",
-        "\n",
-        "# Define a subnet to be created with network security group\n",
-        "subnet = azure.mgmt.network.models.Subnet(\n",
-        "            id='default',\n",
-        "            address_prefix='10.0.0.0/24',\n",
-        "            network_security_group=network_security_group\n",
-        "            )\n",
-        "    \n",
-        "# Create subnet on virtual network\n",
-        "async_subnet_creation = network_client.subnets.begin_create_or_update(\n",
-        "    resource_group_name=resource_group,\n",
-        "    virtual_network_name=vnet_name,\n",
-        "    subnet_name=subnet_name,\n",
-        "    subnet_parameters=subnet\n",
-        ")\n",
-        "\n",
-        "async_subnet_creation.wait()\n",
-        "print(\"Subnet created successfully:\", async_subnet_creation.result())"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Review the virtual network security rules\n",
-        "Ensure that the virtual network is configured correctly with required ports open. It is possible that you have configured rules with broader range of ports that allows ports 29876-29877 to be opened. Kindly review your network security group rules.  "
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "from files.networkutils import *\n",
-        "from azure.identity import DefaultAzureCredential\n",
-        "\n",
-        "check_vnet_security_rules(DefaultAzureCredential(), ws.subscription_id, ws.resource_group, vnet_name, True)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
         "### Create compute targets\n",
         "\n",
-        "In this example, we show how to set up separate compute targets for the Ray head and Ray worker nodes.\n",
+        "In this example, we show how to set up separate compute targets for the Ray nodes.\n",
         "\n",
         "> Note that if you have an AzureML Data Scientist role, you will not have permission to create compute resources. Talk to your workspace or IT admin to create the compute targets described in this section, if they do not already exist.\n",
         "\n",
@@ -322,149 +167,126 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646086081229
+        }
+      },
       "outputs": [],
       "source": [
         "from azureml.core.compute import AmlCompute, ComputeTarget\n",
         "\n",
-        "# Choose a name for the Ray head cluster\n",
-        "head_compute_name = 'head-gpu'\n",
-        "head_compute_min_nodes = 0\n",
-        "head_compute_max_nodes = 2\n",
+        "# Choose a name for the Ray cluster\n",
+        "compute_name = 'compute-gpu'\n",
+        "compute_min_nodes = 0\n",
+        "compute_max_nodes = 2\n",
         "\n",
         "# This example uses GPU VM. For using CPU VM, set SKU to STANDARD_D2_V2\n",
-        "head_vm_size = 'STANDARD_NC6'\n",
+        "vm_size = 'STANDARD_NC6'\n",
         "\n",
-        "if head_compute_name in ws.compute_targets:\n",
-        "    head_compute_target = ws.compute_targets[head_compute_name]\n",
-        "    if head_compute_target and type(head_compute_target) is AmlCompute:\n",
-        "        if head_compute_target.provisioning_state == 'Succeeded':\n",
-        "            print('found head compute target. just use it', head_compute_name)\n",
+        "if compute_name in ws.compute_targets:\n",
+        "    compute_target = ws.compute_targets[compute_name]\n",
+        "    if compute_target and type(compute_target) is AmlCompute:\n",
+        "        if compute_target.provisioning_state == 'Succeeded':\n",
+        "            print('found compute target. just use it', compute_name)\n",
         "        else: \n",
         "            raise Exception(\n",
-        "                'found head compute target but it is in state', head_compute_target.provisioning_state)\n",
+        "                'found compute target but it is in state', compute_target.provisioning_state)\n",
         "else:\n",
-        "    print('creating a new head compute target...')\n",
+        "    print('creating a new compute target...')\n",
         "    provisioning_config = AmlCompute.provisioning_configuration(\n",
-        "        vm_size=head_vm_size,\n",
-        "        min_nodes=head_compute_min_nodes, \n",
-        "        max_nodes=head_compute_max_nodes,\n",
-        "        vnet_resourcegroup_name=ws.resource_group,\n",
-        "        vnet_name=vnet_name,\n",
-        "        subnet_name='default')\n",
+        "        vm_size=vm_size,\n",
+        "        min_nodes=compute_min_nodes, \n",
+        "        max_nodes=compute_max_nodes,\n",
+        "    )\n",
         "\n",
         "    # Create the cluster\n",
-        "    head_compute_target = ComputeTarget.create(ws, head_compute_name, provisioning_config)\n",
+        "    compute_target = ComputeTarget.create(ws, compute_name, provisioning_config)\n",
         "    \n",
         "    # Can poll for a minimum number of nodes and for a specific timeout. \n",
         "    # If no min node count is provided it will use the scale settings for the cluster\n",
-        "    head_compute_target.wait_for_completion(show_output=True, min_node_count=None, timeout_in_minutes=20)\n",
+        "    compute_target.wait_for_completion(show_output=True, min_node_count=None, timeout_in_minutes=20)\n",
         "    \n",
         "    # For a more detailed view of current AmlCompute status, use get_status()\n",
-        "    print(head_compute_target.get_status().serialize())"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Create worker compute target\n",
-        "\n",
-        "Now we create a compute target with CPUs for the additional Ray worker nodes. CPUs in these worker nodes are used by Ray worker processes. Each Ray worker node, depending on the CPUs on the node, may have multiple Ray worker processes. There can be multiple worker tasks on each worker process (core)."
+        "    print(compute_target.get_status().serialize())"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646093795069
+        },
+        "jupyter": {
+          "outputs_hidden": false,
+          "source_hidden": false
+        },
+        "nteract": {
+          "transient": {
+            "deleting": false
+          }
+        }
+      },
       "outputs": [],
       "source": [
-        "# Choose a name for your Ray worker compute target\n",
-        "worker_compute_name = 'worker-cpu'\n",
-        "worker_compute_min_nodes = 0 \n",
-        "worker_compute_max_nodes = 4\n",
+        "from azureml.core import Environment\n",
+        "import os\n",
         "\n",
-        "# This example uses CPU VM. For using GPU VM, set SKU to STANDARD_NC6\n",
-        "worker_vm_size = 'STANDARD_D2_V2'\n",
+        "ray_environment_name = 'pong-cpu'\n",
+        "ray_environment_dockerfile_path = os.path.join(os.getcwd(), 'docker', 'Dockerfile-cpu')\n",
         "\n",
-        "# Create the compute target if it hasn't been created already\n",
-        "if worker_compute_name in ws.compute_targets:\n",
-        "    worker_compute_target = ws.compute_targets[worker_compute_name]\n",
-        "    if worker_compute_target and type(worker_compute_target) is AmlCompute:\n",
-        "        if worker_compute_target.provisioning_state == 'Succeeded':\n",
-        "            print('found worker compute target. just use it', worker_compute_name)\n",
-        "        else: \n",
-        "            raise Exception(\n",
-        "                'found worker compute target but it is in state', head_compute_target.provisioning_state)\n",
-        "else:\n",
-        "    print('creating a new worker compute target...')\n",
-        "    provisioning_config = AmlCompute.provisioning_configuration(\n",
-        "        vm_size=worker_vm_size,\n",
-        "        min_nodes=worker_compute_min_nodes,\n",
-        "        max_nodes=worker_compute_max_nodes,\n",
-        "        vnet_resourcegroup_name=ws.resource_group,\n",
-        "        vnet_name=vnet_name,\n",
-        "        subnet_name='default')\n",
+        "# Build CPU image\n",
+        "ray_cpu_env = Environment. \\\n",
+        "    from_dockerfile(name=ray_environment_name, dockerfile=ray_environment_dockerfile_path). \\\n",
+        "    register(workspace=ws)\n",
+        "ray_cpu_build_details = ray_cpu_env.build(workspace=ws)\n",
         "\n",
-        "    # Create the compute target\n",
-        "    worker_compute_target = ComputeTarget.create(ws, worker_compute_name, provisioning_config)\n",
-        "    \n",
-        "    # Can poll for a minimum number of nodes and for a specific timeout. \n",
-        "    # If no min node count is provided it will use the scale settings for the cluster\n",
-        "    worker_compute_target.wait_for_completion(show_output=True, min_node_count=None, timeout_in_minutes=20)\n",
-        "    \n",
-        "    # For a more detailed view of current AmlCompute status, use get_status()\n",
-        "    print(worker_compute_target.get_status().serialize())"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Train Pong Agent\n",
-        "To facilitate reinforcement learning, Azure Machine Learning Python SDK provides a high level abstraction, the _ReinforcementLearningEstimator_ class, which allows users to easily construct  reinforcement learning run configurations for the underlying reinforcement learning framework.  Reinforcement Learning in Azure Machine Learning supports the open source [Ray framework](https://ray.io/) and its highly customizable [RLLib](https://ray.readthedocs.io/en/latest/rllib.html#rllib-scalable-reinforcement-learning). In this section we show how to use _ReinforcementLearningEstimator_ and Ray/RLLib framework to train a Pong playing agent.\n",
-        "\n",
-        "\n",
-        "### Define worker configuration\n",
-        "Define a `WorkerConfiguration` using your worker compute target. We specify the number of nodes in the worker compute target to be used for training and additional PIP packages to install on those nodes as a part of setup.\n",
-        "In this case, we define the PIP packages as dependencies for both head and worker nodes. With this setup, the game simulations will run directly on the worker compute nodes."
+        "ray_cpu_build_details.wait_for_completion(show_output=True)"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646160884910
+        },
+        "jupyter": {
+          "outputs_hidden": true,
+          "source_hidden": false
+        },
+        "nteract": {
+          "transient": {
+            "deleting": false
+          }
+        }
+      },
       "outputs": [],
       "source": [
-        "from azureml.contrib.train.rl import WorkerConfiguration\n",
+        "from azureml.core import Environment\n",
         "\n",
-        "# Specify the Ray worker configuration\n",
-        "worker_conf = WorkerConfiguration(\n",
-        "    \n",
-        "    # Azure Machine Learning compute target to run Ray workers\n",
-        "    compute_target=worker_compute_target, \n",
-        "    \n",
-        "    # Number of worker nodes\n",
-        "    node_count=4,\n",
-        "    \n",
-        "    # GPU\n",
-        "    use_gpu=False, \n",
-        "    \n",
-        "    # Shared memory size\n",
-        "    # Uncomment line below to set shm_size for workers (requires Azure Machine Learning SDK 1.33 or greater)\n",
-        "    # shm_size=1024*1024*1024, \n",
-        "    \n",
-        "    # PIP packages to use\n",
-        ")"
+        "ray_environment_name = 'pong-gpu'\n",
+        "ray_environment_dockerfile_path = os.path.join(os.getcwd(), 'docker', 'Dockerfile-gpu')\n",
+        "\n",
+        "# Build GPU image\n",
+        "ray_gpu_env = Environment. \\\n",
+        "    from_dockerfile(name=ray_environment_name, dockerfile=ray_environment_dockerfile_path). \\\n",
+        "    register(workspace=ws)\n",
+        "ray_gpu_build_details = ray_gpu_env.build(workspace=ws)\n",
+        "\n",
+        "ray_gpu_build_details.wait_for_completion(show_output=True)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "### Create reinforcement learning estimator\n",
+        "### Create reinforcement learning training run\n",
         "\n",
-        "The `ReinforcementLearningEstimator` is used to submit a job to Azure Machine Learning to start the Ray experiment run. We define the training script parameters here that will be passed to the estimator. \n",
+        "The code below submits the training run using a `ScriptRunConfig`. By providing the\n",
+        "command to run the training, and a `RunConfig` object configured with your\n",
+        "compute target, number of nodes, and environment image to use.\n",
         "\n",
         "We specify `episode_reward_mean` to 18 as we want to stop the training as soon as the trained agent reaches an average win margin of at least 18 point over opponent over all episodes in the training epoch.\n",
         "Number of Ray worker processes are defined by parameter `num_workers`. We set it to 13 as we have 13 CPUs available in our compute targets. Multiple Ray worker processes parallelizes agent training and helps in achieving our goal faster. \n",
@@ -479,70 +301,44 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646162435310
+        }
+      },
       "outputs": [],
       "source": [
-        "from azureml.contrib.train.rl import ReinforcementLearningEstimator, Ray\n",
+        "from azureml.core import RunConfiguration, ScriptRunConfig, Experiment\n",
+        "from azureml.core.runconfig import DockerConfiguration, RunConfiguration\n",
+        "\n",
+        "experiment_name = 'rllib-pong-multi-node'\n",
+        "\n",
+        "experiment = Experiment(workspace=ws, name=experiment_name)\n",
+        "ray_environment = Environment.get(workspace=ws, name=ray_environment_name)\n",
+        "\n",
+        "aml_run_config_ml = RunConfiguration(communicator='OpenMpi')\n",
+        "aml_run_config_ml.target = compute_target\n",
+        "aml_run_config_ml.docker = DockerConfiguration(use_docker=True)\n",
+        "aml_run_config_ml.node_count = 2\n",
+        "aml_run_config_ml.environment = ray_environment\n",
         "\n",
         "training_algorithm = \"IMPALA\"\n",
         "rl_environment = \"PongNoFrameskip-v4\"\n",
+        "script_name='pong_rllib.py'\n",
         "\n",
-        "# Training script parameters\n",
-        "script_params = {\n",
-        "    \n",
-        "    # Training algorithm, IMPALA in this case\n",
-        "    \"--run\": training_algorithm,\n",
-        "    \n",
-        "    # Environment, Pong in this case\n",
-        "    \"--env\": rl_environment,\n",
-        "    \n",
-        "    # Add additional single quotes at the both ends of string values as we have spaces in the \n",
-        "    # string parameters, outermost quotes are not passed to scripts as they are not actually part of string\n",
-        "    # Number of GPUs\n",
-        "    # Number of ray workers\n",
-        "    \"--config\": '\\'{\"num_gpus\": 1, \"num_workers\": 13}\\'',\n",
-        "    \n",
-        "    # Target episode reward mean to stop the training\n",
-        "    # Total training time in seconds\n",
-        "    \"--stop\": '\\'{\"episode_reward_mean\": 18, \"time_total_s\": 3600}\\'',\n",
-        "}\n",
+        "command=[\n",
+        "    'python', script_name,\n",
+        "    '--run', training_algorithm,\n",
+        "    '--env', rl_environment,\n",
+        "    '--config', '\\'{\"num_gpus\": 1, \"num_workers\": 11}\\'',\n",
+        "    '--stop', '\\'{\"episode_reward_mean\": 18, \"time_total_s\": 3600}\\''\n",
+        "]\n",
         "\n",
-        "#  Reinforcement learning estimator\n",
-        "rl_estimator = ReinforcementLearningEstimator(\n",
-        "    \n",
-        "    # Location of source files\n",
-        "    source_directory='files',\n",
-        "    \n",
-        "    # Python script file\n",
-        "    entry_script=\"pong_rllib.py\",\n",
-        "    \n",
-        "    # Parameters to pass to the script file\n",
-        "    # Defined above.\n",
-        "    script_params=script_params,\n",
-        "    \n",
-        "    # The Azure Machine Learning compute target set up for Ray head nodes\n",
-        "    compute_target=head_compute_target,\n",
-        "    \n",
-        "    # GPU usage\n",
-        "    use_gpu=True,\n",
-        "    \n",
-        "    # Reinforcement learning framework. Currently must be Ray.\n",
-        "    rl_framework=Ray('0.8.3'),\n",
-        "    \n",
-        "    # Ray worker configuration defined above.\n",
-        "    worker_configuration=worker_conf,\n",
-        "    \n",
-        "    # How long to wait for whole cluster to start\n",
-        "    cluster_coordination_timeout_seconds=3600,\n",
-        "    \n",
-        "    # Maximum time for the whole Ray job to run\n",
-        "    # This will cut off the run after an hour\n",
-        "    max_run_duration_seconds=3600,\n",
-        "    \n",
-        "    # Allow the docker container Ray runs in to make full use\n",
-        "    # of the shared memory available from the host OS.\n",
-        "    shm_size=24*1024*1024*1024\n",
-        ")"
+        "config = ScriptRunConfig(source_directory='./files',\n",
+        "                    command=command,\n",
+        "                    run_config = aml_run_config_ml\n",
+        "                   )\n",
+        "training_run = experiment.submit(config)"
       ]
     },
     {
@@ -571,23 +367,6 @@
         "```"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Submit the estimator to start a run\n",
-        "Now we use the rl_estimator configured above to submit a run."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "run = exp.submit(config=rl_estimator)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {},
@@ -605,7 +384,7 @@
       "source": [
         "from azureml.widgets import RunDetails\n",
         "\n",
-        "RunDetails(run).show()"
+        "RunDetails(training_run).show()"
       ]
     },
     {
@@ -614,7 +393,7 @@
       "source": [
         "### Stop the run\n",
         "\n",
-        "To stop the run, call `run.cancel()`."
+        "To stop the run, call `training_run.cancel()`."
       ]
     },
     {
@@ -624,7 +403,7 @@
       "outputs": [],
       "source": [
         "# Uncomment line below to cancel the run\n",
-        "# run.cancel()"
+        "# training_run.cancel()"
       ]
     },
     {
@@ -643,7 +422,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "run.wait_for_completion()"
+        "training_run.wait_for_completion()"
       ]
     },
     {
@@ -663,8 +442,8 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "# Get the reward metrics from worker run\n",
-        "episode_reward_mean = run.get_metrics(name='episode_reward_mean')"
+        "# Get the reward metrics from training_run\n",
+        "episode_reward_mean = training_run.get_metrics(name='episode_reward_mean')"
       ]
     },
     {
@@ -733,6 +512,16 @@
         "name": "vineetg"
       }
     ],
+    "categories": [
+      "how-to-use-azureml",
+      "reinforcement-learning"
+    ],
+    "interpreter": {
+      "hash": "13382f70c1d0595120591d2e358c8d446daf961bf951d1fba9a32631e205d5ab"
+    },
+    "kernel_info": {
+      "name": "python3-azureml"
+    },
     "kernelspec": {
       "display_name": "Python 3.6",
       "language": "python",
@@ -748,10 +537,13 @@
       "name": "python",
       "nbconvert_exporter": "python",
       "pygments_lexer": "ipython3",
-      "version": "3.6.9"
+      "version": "3.8.0"
     },
-    "notice": "Copyright (c) Microsoft Corporation. All rights reserved.\u00e2\u20ac\u00afLicensed under the MIT License.\u00e2\u20ac\u00af "
+    "notice": "Copyright (c) Microsoft Corporation. All rights reserved.\u00c3\u00a2\u00e2\u201a\u00ac\u00c2\u00afLicensed under the MIT License.\u00c3\u00a2\u00e2\u201a\u00ac\u00c2\u00af ",
+    "nteract": {
+      "version": "nteract-front-end@1.0.0"
+    }
   },
   "nbformat": 4,
-  "nbformat_minor": 4
+  "nbformat_minor": 0
 }

how-to-use-azureml/reinforcement-learning/cartpole-on-compute-instance/cartpole_ci.ipynb

@@ -82,7 +82,11 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646344676671
+        }
+      },
       "outputs": [],
       "source": [
         "import azureml.core\n",
@@ -100,7 +104,11 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646344680982
+        }
+      },
       "outputs": [],
       "source": [
         "from azureml.core import Workspace\n",
@@ -123,7 +131,11 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646344684217
+        }
+      },
       "outputs": [],
       "source": [
         "import os.path\n",
@@ -146,7 +158,11 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646344690768
+        }
+      },
       "outputs": [],
       "source": [
         "from azureml.core.compute import ComputeInstance\n",
@@ -194,13 +210,52 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646344835579
+        }
+      },
       "outputs": [],
       "source": [
         "from azureml.core.experiment import Experiment\n",
         "\n",
         "experiment_name = 'CartPole-v0-CI'\n",
-        "exp = Experiment(workspace=ws, name=experiment_name)"
+        "experiment = Experiment(workspace=ws, name=experiment_name)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "gather": {
+          "logged": 1646346293902
+        },
+        "jupyter": {
+          "outputs_hidden": false,
+          "source_hidden": false
+        },
+        "nteract": {
+          "transient": {
+            "deleting": false
+          }
+        }
+      },
+      "outputs": [],
+      "source": [
+        "from azureml.core import Environment\n",
+        "import os\n",
+        "import time\n",
+        "\n",
+        "ray_environment_name = 'cartpole-ray-ci'\n",
+        "ray_environment_dockerfile_path = os.path.join(os.getcwd(), 'files', 'docker', 'Dockerfile')\n",
+        "\n",
+        "# Build environment image\n",
+        "ray_environment = Environment. \\\n",
+        "    from_dockerfile(name=ray_environment_name, dockerfile=ray_environment_dockerfile_path). \\\n",
+        "    register(workspace=ws)\n",
+        "ray_env_build_details = ray_environment.build(workspace=ws)\n",
+        "\n",
+        "ray_env_build_details.wait_for_completion(show_output=True)"
       ]
     },
     {
@@ -208,80 +263,69 @@
       "metadata": {},
       "source": [
         "## Train Cartpole Agent\n",
-        "To facilitate reinforcement learning, Azure Machine Learning Python SDK provides a high level abstraction, the _ReinforcementLearningEstimator_ class, which allows users to easily construct reinforcement learning run configurations for the underlying reinforcement learning framework. Reinforcement Learning in Azure Machine Learning supports the open source [Ray framework](https://ray.io/) and its highly customizable [RLlib](https://ray.readthedocs.io/en/latest/rllib.html#rllib-scalable-reinforcement-learning). In this section we show how to use _ReinforcementLearningEstimator_ and Ray/RLlib framework to train a cartpole playing agent. "
+        "In this section, we show how to use Azure Machine Learning jobs and Ray/RLlib framework to train a cartpole playing agent. "
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "### Create reinforcement learning estimator\n",
+        "### Create reinforcement learning training run\n",
         "\n",
-        "The code below creates an instance of *ReinforcementLearningEstimator*, `training_estimator`, which then will be used to submit a job to Azure Machine Learning to start the Ray experiment run.\n",
-        "\n",
-        "Note that this example is purposely simplified to the minimum. Here is a short description of the parameters we are passing into the constructor:\n",
-        "\n",
-        "- `source_directory`, local directory containing your training script(s) and helper modules,\n",
-        "- `entry_script`, path to your entry script relative to the source directory,\n",
-        "- `script_params`, constant parameters to be passed to each run of training script,\n",
-        "- `compute_target`, reference to the compute target in which the trainer and worker(s) jobs will be executed,\n",
-        "- `rl_framework`, the reinforcement learning framework to be used (currently must be Ray).\n",
-        "\n",
-        "We use the `script_params` parameter to pass in general and algorithm-specific parameters to the training script.\n"
+        "The code below submits the training run using a `ScriptRunConfig`. By providing the\n",
+        "command to run the training, and a `RunConfig` object configured with your\n",
+        "compute target, number of nodes, and environment image to use."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646347120585
+        },
+        "jupyter": {
+          "outputs_hidden": false,
+          "source_hidden": false
+        },
+        "nteract": {
+          "transient": {
+            "deleting": false
+          }
+        }
+      },
       "outputs": [],
       "source": [
-        "from azureml.contrib.train.rl import ReinforcementLearningEstimator, Ray\n",
+        "from azureml.core import Environment\n",
+        "from azureml.core import RunConfiguration, ScriptRunConfig, Experiment\n",
+        "from azureml.core.runconfig import DockerConfiguration, RunConfiguration\n",
         "\n",
-        "training_algorithm = \"PPO\"\n",
-        "rl_environment = \"CartPole-v0\"\n",
+        "training_algorithm = 'PPO'\n",
+        "rl_environment = 'CartPole-v0'\n",
         "\n",
-        "script_params = {\n",
+        "script_name = 'cartpole_training.py'\n",
+        "script_arguments = [\n",
+        "    '--run', training_algorithm,\n",
+        "    '--env', rl_environment,\n",
+        "    '--config', '{\"num_gpus\": 0, \"num_workers\": 1}',\n",
+        "    '--stop', '{\"episode_reward_mean\": 200, \"time_total_s\": 300}',\n",
+        "    '--checkpoint-freq', '2',\n",
+        "    '--checkpoint-at-end',\n",
+        "    '--local-dir', './logs'\n",
+        "]\n",
         "\n",
-        "    # Training algorithm\n",
-        "    \"--run\": training_algorithm,\n",
-        "    \n",
-        "    # Training environment\n",
-        "    \"--env\": rl_environment,\n",
-        "    \n",
-        "    # Algorithm-specific parameters\n",
-        "    \"--config\": '\\'{\"num_gpus\": 0, \"num_workers\": 1}\\'',\n",
-        "    \n",
-        "    # Stop conditions\n",
-        "    \"--stop\": '\\'{\"episode_reward_mean\": 200, \"time_total_s\": 300}\\'',\n",
-        "    \n",
-        "    # Frequency of taking checkpoints\n",
-        "    \"--checkpoint-freq\": 2,\n",
-        "    \n",
-        "    # If a checkpoint should be taken at the end - optional argument with no value\n",
-        "    \"--checkpoint-at-end\": \"\",\n",
-        "    \n",
-        "    # Log directory\n",
-        "    \"--local-dir\": './logs'\n",
-        "}\n",
+        "aml_run_config_ml = RunConfiguration(communicator='OpenMpi')\n",
+        "aml_run_config_ml.target = compute_target\n",
+        "aml_run_config_ml.docker = DockerConfiguration(use_docker=True)\n",
+        "aml_run_config_ml.node_count = 1\n",
+        "aml_run_config_ml.environment = ray_environment\n",
         "\n",
-        "training_estimator = ReinforcementLearningEstimator(\n",
-        "\n",
-        "    # Location of source files\n",
-        "    source_directory='files',\n",
-        "    \n",
-        "    # Python script file\n",
-        "    entry_script='cartpole_training.py',\n",
-        "    \n",
-        "    # A dictionary of arguments to pass to the training script specified in ``entry_script``\n",
-        "    script_params=script_params,\n",
-        "    \n",
-        "    # The Azure Machine Learning compute target set up for Ray head nodes\n",
-        "    compute_target=compute_target,\n",
-        "    \n",
-        "    # Reinforcement learning framework. Currently must be Ray.\n",
-        "    rl_framework=Ray()\n",
-        ")"
+        "training_config = ScriptRunConfig(source_directory='./files',\n",
+        "                    script=script_name,\n",
+        "                    arguments=script_arguments,\n",
+        "                    run_config = aml_run_config_ml\n",
+        "                   )\n",
+        "training_run = experiment.submit(training_config)"
       ]
     },
     {
@@ -304,6 +348,7 @@
         "See [RLlib Training APIs](https://ray.readthedocs.io/en/latest/rllib-training.html#rllib-training-apis) for more details, and also [Training (tune.run, tune.Experiment)](https://ray.readthedocs.io/en/latest/tune/api_docs/execution.html#training-tune-run-tune-experiment) for the complete list of parameters.\n",
         "\n",
         "```python\n",
+        "import os\n",
         "import ray\n",
         "import ray.tune as tune\n",
         "\n",
@@ -311,8 +356,9 @@
         "\n",
         "    # parse arguments ...\n",
         "    \n",
-        "    # Intitialize ray\n",
-        "    ay.init(address=args.ray_address)\n",
+        "    # Start ray head (single node)\n",
+        "    os.system('ray start --head')\n",
+        "    ray.init(address='auto')\n",
         "\n",
         "    # Run training task using tune.run\n",
         "    tune.run(\n",
@@ -326,23 +372,6 @@
         "```"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Submit the estimator to start experiment\n",
-        "Now we use the *training_estimator* to submit a run."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "training_run = exp.submit(training_estimator)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {},
@@ -350,15 +379,17 @@
         "### Monitor experiment\n",
         "Azure Machine Learning provides a Jupyter widget to show the status of an experiment run. You could use this widget to monitor the status of the runs.\n",
         "\n",
-        "Note that _ReinforcementLearningEstimator_ creates at least two runs: (a) A parent run, i.e. the run returned above, and (b) a collection of child runs. The number of the child runs depends on the configuration of the reinforcement learning estimator. In our simple scenario, configured above, only one child run will be created.\n",
-        "\n",
-        "The widget will show a list of the child runs as well. You can click on the link under **Status** to see the details of a child run. It will also show the metrics being logged."
+        "You can click on the link under **Status** to see the details of a child run. It will also show the metrics being logged."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646347127671
+        }
+      },
       "outputs": [],
       "source": [
         "from azureml.widgets import RunDetails\n",
@@ -398,50 +429,23 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646347318682
+        }
+      },
       "outputs": [],
       "source": [
         "training_run.wait_for_completion()"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Get a handle to the child run\n",
-        "You can obtain a handle to the child run as follows. In our scenario, there is only one child run, we have it called `child_run_0`."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "import time\n",
-        "\n",
-        "child_run_0 = None\n",
-        "timeout = 30\n",
-        "while timeout > 0 and not child_run_0:\n",
-        "    child_runs = list(training_run.get_children())\n",
-        "    print('Number of child runs:', len(child_runs))\n",
-        "    if len(child_runs) > 0:\n",
-        "        child_run_0 = child_runs[0]\n",
-        "        break\n",
-        "    time.sleep(2) # Wait for 2 seconds\n",
-        "    timeout -= 2\n",
-        "\n",
-        "print('Child run info:')\n",
-        "print(child_run_0)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
         "## Evaluate Trained Agent and See Results\n",
         "\n",
-        "We can evaluate a previously trained policy using the `rollout.py` helper script provided by RLlib (see [Evaluating Trained Policies](https://ray.readthedocs.io/en/latest/rllib-training.html#evaluating-trained-policies) for more details). Here we use an adaptation of this script to reconstruct a policy from a checkpoint taken and saved during training. We took these checkpoints by setting `checkpoint-freq` and `checkpoint-at-end` parameters above.\n",
+        "We can evaluate a previously trained policy using the `cartpole_rollout.py` helper script provided by RLlib (see [Evaluating Trained Policies](https://ray.readthedocs.io/en/latest/rllib-training.html#evaluating-trained-policies) for more details). Here we use an adaptation of this script to reconstruct a policy from a checkpoint taken and saved during training. We took these checkpoints by setting `checkpoint-freq` and `checkpoint-at-end` parameters above.\n",
         "\n",
         "In this section we show how to get access to these checkpoints data, and then how to use them to evaluate the trained policy."
       ]
@@ -458,7 +462,11 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646347328505
+        }
+      },
       "outputs": [],
       "source": [
         "from os import path\n",
@@ -471,7 +479,7 @@
         "    dir_util.remove_tree(training_artifacts_path)\n",
         "\n",
         "# Download run artifacts to local compute\n",
-        "child_run_0.download_files(training_artifacts_path)"
+        "training_run.download_files(training_artifacts_path)"
       ]
     },
     {
@@ -484,7 +492,11 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646347334571
+        }
+      },
       "outputs": [],
       "source": [
         "# A helper function to find checkpoint files in a directory\n",
@@ -501,7 +513,11 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646347337724
+        }
+      },
       "outputs": [],
       "source": [
         "# Find checkpoints and last checkpoint number\n",
@@ -529,14 +545,18 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646347346085
+        }
+      },
       "outputs": [],
       "source": [
         "# Upload the checkpoint files and create a DataSet\n",
         "from azureml.core import Dataset\n",
         "\n",
         "datastore = ws.get_default_datastore()\n",
-        "checkpoint_dataref = datastore.upload_files(checkpoint_files, target_path='cartpole_checkpoints_' + run_id, overwrite=True)\n",
+        "checkpoint_dataref = datastore.upload_files(checkpoint_files, target_path='cartpole_checkpoints_' + training_run.id, overwrite=True)\n",
         "checkpoint_ds = Dataset.File.from_files(checkpoint_dataref)"
       ]
     },
@@ -550,7 +570,11 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646347354726
+        }
+      },
       "outputs": [],
       "source": [
         "artifacts_paths = checkpoint_ds.to_path()\n",
@@ -564,82 +588,67 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "### Evaluate a trained policy\n",
-        "We need to configure another reinforcement learning estimator, `rollout_estimator`, and then use it to submit another run. Note that the entry script for this estimator now points to `cartpole-rollout.py` script.\n",
-        "Also note how we pass the checkpoints dataset to this script using `inputs` parameter of the _ReinforcementLearningEstimator_.\n",
+        "## Evaluate Trained Agent and See Results\n",
         "\n",
-        "We are using script parameters to pass in the same algorithm and the same environment used during training. We also specify the checkpoint number of the checkpoint we wish to evaluate, `checkpoint-number`, and number of the steps we shall run the rollout, `steps`.\n",
-        "\n",
-        "The checkpoints dataset will be accessible to the rollout script as a mounted folder. The mounted folder and the checkpoint number, passed in via `checkpoint-number`, will be used to create a path to the checkpoint we are going to evaluate. The created checkpoint path then will be passed into RLlib rollout script for evaluation.\n",
-        "\n",
-        "Now let's configure rollout estimator. Note that we use the last checkpoint for evaluation. The assumption is that the last checkpoint points to our best trained agent. You may change this to any of the checkpoint numbers printed above and observe the effect."
+        "We can evaluate a previously trained policy using the `cartpole_rollout.py` helper script provided by RLlib (see [Evaluating Trained Policies](https://ray.readthedocs.io/en/latest/rllib-training.html#evaluating-trained-policies) for more details). Here we use an adaptation of this script to reconstruct a policy from a checkpoint taken and saved during training. We took these checkpoints by setting `checkpoint-freq` and `checkpoint-at-end` parameters above.\n",
+        "In this section we show how to use these checkpoints to evaluate the trained policy."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646347414835
+        },
+        "jupyter": {
+          "outputs_hidden": false,
+          "source_hidden": false
+        },
+        "nteract": {
+          "transient": {
+            "deleting": false
+          }
+        }
+      },
       "outputs": [],
       "source": [
-        "script_params = {    \n",
-        "    # Checkpoint number of the checkpoint from which to roll out\n",
-        "    \"--checkpoint-number\": last_checkpoint_number,\n",
+        "ray_environment_name = 'cartpole-ray-ci'\n",
         "\n",
-        "    # Training algorithm\n",
-        "    \"--run\": training_algorithm,\n",
-        "    \n",
-        "    # Training environment\n",
-        "    \"--env\": rl_environment,\n",
-        "    \n",
-        "    # Algorithm-specific parameters\n",
-        "    \"--config\": '{}',\n",
-        "    \n",
-        "    # Number of rollout steps \n",
-        "    \"--steps\": 2000,\n",
-        "    \n",
-        "    # If should repress rendering of the environment\n",
-        "    \"--no-render\": \"\"\n",
-        "}\n",
+        "experiment_name = 'CartPole-v0-CI'\n",
+        "training_algorithm = 'PPO'\n",
+        "rl_environment = 'CartPole-v0'\n",
         "\n",
-        "rollout_estimator = ReinforcementLearningEstimator(\n",
-        "    # Location of source files\n",
-        "    source_directory='files',\n",
-        "    \n",
-        "    # Python script file\n",
-        "    entry_script='cartpole_rollout.py',\n",
-        "    \n",
-        "    # A dictionary of arguments to pass to the rollout script specified in ``entry_script``\n",
-        "    script_params = script_params,\n",
-        "    \n",
-        "    # Data inputs\n",
-        "    inputs=[\n",
-        "        checkpoint_ds.as_named_input('artifacts_dataset'),\n",
-        "        checkpoint_ds.as_named_input('artifacts_path').as_mount()],\n",
-        "    \n",
-        "    # The Azure Machine Learning compute target\n",
-        "    compute_target=compute_target,\n",
-        "    \n",
-        "    # Reinforcement learning framework. Currently must be Ray.\n",
-        "    rl_framework=Ray(),\n",
-        "    \n",
-        "    # Additional pip packages to install\n",
-        "    pip_packages = ['azureml-dataset-runtime[fuse,pandas]'])"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "Same as before, we use the *rollout_estimator* to submit a run."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "rollout_run = exp.submit(rollout_estimator)"
+        "experiment = Experiment(workspace=ws, name=experiment_name)\n",
+        "ray_environment = Environment.get(workspace=ws, name=ray_environment_name)\n",
+        "\n",
+        "script_name = 'cartpole_rollout.py'\n",
+        "script_arguments = [\n",
+        "    '--run', training_algorithm,\n",
+        "    '--env', rl_environment,\n",
+        "    '--config', '{}',\n",
+        "    '--steps', '2000',\n",
+        "    '--checkpoint-number', str(last_checkpoint_number),\n",
+        "    '--no-render',\n",
+        "    '--artifacts-dataset', checkpoint_ds.as_named_input('artifacts_dataset'),\n",
+        "    '--artifacts-path', checkpoint_ds.as_named_input('artifacts_path').as_mount()\n",
+        "]\n",
+        "\n",
+        "aml_run_config_ml = RunConfiguration(communicator='OpenMpi')\n",
+        "aml_run_config_ml.target = compute_target\n",
+        "aml_run_config_ml.docker = DockerConfiguration(use_docker=True)\n",
+        "aml_run_config_ml.node_count = 1\n",
+        "aml_run_config_ml.environment = ray_environment\n",
+        "aml_run_config_ml.data\n",
+        "\n",
+        "rollout_config = ScriptRunConfig(\n",
+        "                    source_directory='./files',\n",
+        "                    script=script_name,\n",
+        "                    arguments=script_arguments,\n",
+        "                    run_config = aml_run_config_ml\n",
+        "                   )\n",
+        "                   \n",
+        "rollout_run = experiment.submit(rollout_config)"
       ]
     },
     {
@@ -652,7 +661,11 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646347429626
+        }
+      },
       "outputs": [],
       "source": [
         "RunDetails(rollout_run).show()"
@@ -717,6 +730,16 @@
         "name": "hoazari"
       }
     ],
+    "categories": [
+      "how-to-use-azureml",
+      "reinforcement-learning"
+    ],
+    "interpreter": {
+      "hash": "13382f70c1d0595120591d2e358c8d446daf961bf951d1fba9a32631e205d5ab"
+    },
+    "kernel_info": {
+      "name": "python3-azureml"
+    },
     "kernelspec": {
       "display_name": "Python 3.6",
       "language": "python",
@@ -732,10 +755,20 @@
       "name": "python",
       "nbconvert_exporter": "python",
       "pygments_lexer": "ipython3",
-      "version": "3.6.9"
+      "version": "3.7.9"
     },
-    "notice": "Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License."
+    "microsoft": {
+      "host": {
+        "AzureML": {
+          "notebookHasBeenCompleted": true
+        }
+      }
+    },
+    "notice": "Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License.",
+    "nteract": {
+      "version": "nteract-front-end@1.0.0"
+    }
   },
   "nbformat": 4,
-  "nbformat_minor": 4
+  "nbformat_minor": 0
 }

how-to-use-azureml/reinforcement-learning/cartpole-on-compute-instance/files/cartpole_rollout.py

@@ -1,4 +1,3 @@
-import argparse
 import os
 import sys
 
@@ -11,10 +10,7 @@ from azureml.core import Run
 from utils import callbacks
 
 
-DEFAULT_RAY_ADDRESS = 'localhost:6379'
-
-
-def run_rollout(args, parser, ray_address):
+def run_rollout(args, parser):
 
     config = args.config
     if not args.env:
@@ -22,8 +18,6 @@ def run_rollout(args, parser, ray_address):
             parser.error("the following arguments are required: --env")
         args.env = config.get("env")
 
-    ray.init(address=ray_address)
-
     # Create the Trainer from config.
     cls = get_trainable_cls(args.run)
     agent = cls(env=args.env, config=config)
@@ -76,6 +70,10 @@ def run_rollout(args, parser, ray_address):
 
 if __name__ == "__main__":
 
+    # Start ray head (single node)
+    os.system('ray start --head')
+    ray.init(address='auto')
+
     # Add positional argument - serves as placeholder for checkpoint
     argvc = sys.argv[1:]
     argvc.insert(0, 'checkpoint-placeholder')
@@ -88,8 +86,12 @@ if __name__ == "__main__":
         help='Checkpoint number of the checkpoint from which to roll out')
 
     rollout_parser.add_argument(
-        '--ray-address', required=False, default=DEFAULT_RAY_ADDRESS,
-        help='The address of the Ray cluster to connect to')
+        '--artifacts-dataset', required=True,
+        help='The checkpoints artifacts dataset')
+
+    rollout_parser.add_argument(
+        '--artifacts-path', required=True,
+        help='The checkpoints artifacts path')
 
     args = rollout_parser.parse_args(argvc)
 
@@ -116,4 +118,4 @@ if __name__ == "__main__":
     args.checkpoint = checkpoint
 
     # Start rollout
-    run_rollout(args, rollout_parser, args.ray_address)
+    run_rollout(args, rollout_parser)

how-to-use-azureml/reinforcement-learning/cartpole-on-compute-instance/files/cartpole_training.py

@@ -1,17 +1,10 @@
-import argparse
-import os
-import sys
-
 import ray
 from ray.rllib import train
 from ray import tune
+import os
 
 from utils import callbacks
 
-
-DEFAULT_RAY_ADDRESS = 'localhost:6379'
-
-
 if __name__ == "__main__":
 
     # Parse arguments and add callbacks to config
@@ -24,11 +17,9 @@ if __name__ == "__main__":
     if 'monitor' in args.config and args.config['monitor']:
         print("Video capturing is ON!")
 
-    # Start (connect to) Ray cluster
-    if args.ray_address is None:
-        args.ray_address = DEFAULT_RAY_ADDRESS
-
-    ray.init(address=args.ray_address)
+    # Start ray head (single node)
+    os.system('ray start --head')
+    ray.init(address='auto')
 
     # Run training task using tune.run
     tune.run(

how-to-use-azureml/reinforcement-learning/cartpole-on-compute-instance/files/docker/Dockerfile

@@ -0,0 +1,17 @@
+FROM mcr.microsoft.com/azureml/openmpi3.1.2-ubuntu18.04
+
+RUN pip install ray-on-aml==0.1.6
+RUN pip install gym[atari]==0.19.0
+RUN pip install gym[accept-rom-license]==0.19.0
+RUN pip install ale-py==0.7.0
+RUN pip install azureml-core
+RUN pip install azureml-dataset-runtime
+RUN pip install ray==0.8.7
+RUN pip install ray[rllib,tune,serve]==0.8.7
+RUN pip install tensorflow==1.14.0
+
+USER root
+
+RUN apt-get update
+RUN apt-get install -y jq
+RUN apt-get install -y rsync

how-to-use-azureml/reinforcement-learning/cartpole-on-compute-instance/files/utils/callbacks.py

@@ -8,7 +8,7 @@ from azureml.core import Run
 def on_train_result(info):
     '''Callback on train result to record metrics returned by trainer.
     '''
-    run = Run.get_context().parent
+    run = Run.get_context()
     run.log(
         name='episode_reward_mean',
         value=info["result"]["episode_reward_mean"])

how-to-use-azureml/reinforcement-learning/cartpole-on-single-compute/cartpole_sc.ipynb

@@ -82,7 +82,11 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646347616697
+        }
+      },
       "outputs": [],
       "source": [
         "import azureml.core\n",
@@ -101,7 +105,11 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646429058500
+        }
+      },
       "outputs": [],
       "source": [
         "from azureml.core import Workspace\n",
@@ -126,7 +134,11 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646359152101
+        }
+      },
       "outputs": [],
       "source": [
         "from azureml.core.compute import AmlCompute, ComputeTarget\n",
@@ -167,13 +179,51 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646348040613
+        }
+      },
       "outputs": [],
       "source": [
         "from azureml.core.experiment import Experiment\n",
         "\n",
         "experiment_name = 'CartPole-v0-SC'\n",
-        "exp = Experiment(workspace=ws, name=experiment_name)"
+        "experiment = Experiment(workspace=ws, name=experiment_name)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "gather": {
+          "logged": 1646417962898
+        },
+        "jupyter": {
+          "outputs_hidden": false,
+          "source_hidden": false
+        },
+        "nteract": {
+          "transient": {
+            "deleting": false
+          }
+        }
+      },
+      "outputs": [],
+      "source": [
+        "from azureml.core import Environment\n",
+        "import os\n",
+        "\n",
+        "ray_environment_name = 'cartpole-ray-sc'\n",
+        "ray_environment_dockerfile_path = os.path.join(os.getcwd(), 'files', 'docker', 'Dockerfile')\n",
+        "\n",
+        "# Build environment image\n",
+        "ray_environment = Environment. \\\n",
+        "    from_dockerfile(name=ray_environment_name, dockerfile=ray_environment_dockerfile_path). \\\n",
+        "    register(workspace=ws)\n",
+        "ray_env_build_details = ray_environment.build(workspace=ws)\n",
+        "\n",
+        "ray_env_build_details.wait_for_completion(show_output=True)"
       ]
     },
     {
@@ -181,109 +231,79 @@
       "metadata": {},
       "source": [
         "## Train Cartpole Agent\n",
-        "To facilitate reinforcement learning, Azure Machine Learning Python SDK provides a high level abstraction, the _ReinforcementLearningEstimator_ class, which allows users to easily construct reinforcement learning run configurations for the underlying reinforcement learning framework. Reinforcement Learning in Azure Machine Learning supports the open source [Ray framework](https://ray.io/) and its highly customizable [RLlib](https://ray.readthedocs.io/en/latest/rllib.html#rllib-scalable-reinforcement-learning). In this section we show how to use _ReinforcementLearningEstimator_ and Ray/RLlib framework to train a cartpole playing agent. "
+        "In this section, we show how to use Azure Machine Learning jobs and Ray/RLlib framework to train a cartpole playing agent. "
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "### Create reinforcement learning estimator\n",
+        "### Create reinforcement learning training run\n",
         "\n",
-        "The code below creates an instance of *ReinforcementLearningEstimator*, `training_estimator`, which then will be used to submit a job to Azure Machine Learning to start the Ray experiment run.\n",
-        "\n",
-        "Note that this example is purposely simplified to the minimum. Here is a short description of the parameters we are passing into the constructor:\n",
-        "\n",
-        "- `source_directory`, local directory containing your training script(s) and helper modules,\n",
-        "- `entry_script`, path to your entry script relative to the source directory,\n",
-        "- `script_params`, constant parameters to be passed to each run of training script,\n",
-        "- `compute_target`, reference to the compute target in which the trainer and worker(s) jobs will be executed,\n",
-        "- `rl_framework`, the reinforcement learning framework to be used (currently must be Ray).\n",
-        "\n",
-        "We use the `script_params` parameter to pass in general and algorithm-specific parameters to the training script.\n"
+        "The code below submits the training run using a `ScriptRunConfig`. By providing the\n",
+        "command to run the training, and a `RunConfig` object configured with your\n",
+        "compute target, number of nodes, and environment image to use."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646437786449
+        },
+        "jupyter": {
+          "outputs_hidden": false,
+          "source_hidden": false
+        },
+        "nteract": {
+          "transient": {
+            "deleting": false
+          }
+        }
+      },
       "outputs": [],
       "source": [
-        "from azureml.contrib.train.rl import ReinforcementLearningEstimator, Ray\n",
-        "from azureml.core.environment import Environment\n",
+        "from azureml.core import RunConfiguration, ScriptRunConfig, Experiment\n",
+        "from azureml.core.runconfig import DockerConfiguration, RunConfiguration\n",
         "\n",
         "training_algorithm = \"PPO\"\n",
         "rl_environment = \"CartPole-v0\"\n",
         "video_capture = True\n",
-        "\n",
         "if video_capture:\n",
         "    algorithm_config = '\\'{\"num_gpus\": 0, \"num_workers\": 1, \"monitor\": true}\\''\n",
         "else:\n",
         "    algorithm_config = '\\'{\"num_gpus\": 0, \"num_workers\": 1, \"monitor\": false}\\''\n",
         "\n",
-        "script_params = {\n",
+        "script_name = 'cartpole_training.py'\n",
+        "script_arguments = [\n",
+        "    '--run', training_algorithm,\n",
+        "    '--env', rl_environment,\n",
+        "    '--stop', '\\'{\"episode_reward_mean\": 200, \"time_total_s\": 300}\\'',\n",
+        "    '--config', algorithm_config,\n",
+        "    '--checkpoint-freq', '2',\n",
+        "    '--checkpoint-at-end',\n",
+        "    '--local-dir', './logs'\n",
+        "]\n",
         "\n",
-        "    # Training algorithm\n",
-        "    \"--run\": training_algorithm,\n",
-        "    \n",
-        "    # Training environment\n",
-        "    \"--env\": rl_environment,\n",
-        "    \n",
-        "    # Algorithm-specific parameters\n",
-        "    \"--config\": algorithm_config,\n",
-        "    \n",
-        "    # Stop conditions\n",
-        "    \"--stop\": '\\'{\"episode_reward_mean\": 200, \"time_total_s\": 300}\\'',\n",
-        "    \n",
-        "    # Frequency of taking checkpoints\n",
-        "    \"--checkpoint-freq\": 2,\n",
-        "    \n",
-        "    # If a checkpoint should be taken at the end - optional argument with no value\n",
-        "    \"--checkpoint-at-end\": \"\",\n",
-        "    \n",
-        "    # Log directory\n",
-        "    \"--local-dir\": './logs'\n",
-        "}\n",
+        "ray_environment = Environment.get(ws, name=ray_environment_name)\n",
+        "run_config = RunConfiguration(communicator='OpenMpi')\n",
+        "run_config.target = compute_target\n",
+        "run_config.docker = DockerConfiguration(use_docker=True)\n",
+        "run_config.node_count = 1\n",
+        "run_config.environment = ray_environment\n",
+        "command=[\"python\", script_name, *script_arguments]\n",
         "\n",
-        "xvfb_env = None\n",
         "if video_capture:\n",
-        "    # Ray's video capture support requires to run everything under a headless display driver called (xvfb).\n",
-        "    # There are two parts to this:\n",
-        "    # 1. Use a custom docker file with proper instructions to install xvfb, ffmpeg, python-opengl\n",
-        "    # and other dependencies.\n",
-        "   \n",
-        "    with open(\"files/docker/Dockerfile\", \"r\") as f:\n",
-        "        dockerfile=f.read()\n",
+        "    command = [\"xvfb-run -s '-screen 0 640x480x16 -ac +extension GLX +render' \"] + command\n",
+        "    run_config.environment_variables[\"SDL_VIDEODRIVER\"] = \"dummy\"\n",
         "\n",
-        "    xvfb_env = Environment(name='xvfb-vdisplay')\n",
-        "    xvfb_env.docker.base_image = None\n",
-        "    xvfb_env.docker.base_dockerfile = dockerfile\n",
-        "    \n",
-        "    # 2.  Execute the Python process via the xvfb-run command to set up the headless display driver.\n",
-        "    xvfb_env.python.user_managed_dependencies = True\n",
-        "    xvfb_env.python.interpreter_path = \"xvfb-run -s '-screen 0 640x480x16 -ac +extension GLX +render' python\"\n",
+        "trainint_config = ScriptRunConfig(source_directory='./files',\n",
+        "            command=command,\n",
+        "            run_config = run_config\n",
+        "            )\n",
         "\n",
-        "\n",
-        "training_estimator = ReinforcementLearningEstimator(\n",
-        "\n",
-        "    # Location of source files\n",
-        "    source_directory='files',\n",
-        "    \n",
-        "    # Python script file\n",
-        "    entry_script='cartpole_training.py',\n",
-        "    \n",
-        "    # A dictionary of arguments to pass to the training script specified in ``entry_script``\n",
-        "    script_params=script_params,\n",
-        "    \n",
-        "    # The Azure Machine Learning compute target set up for Ray head nodes\n",
-        "    compute_target=compute_target,\n",
-        "    \n",
-        "    # Reinforcement learning framework. Currently must be Ray.\n",
-        "    rl_framework=Ray(),\n",
-        "    \n",
-        "    # Custom environmnet for Xvfb\n",
-        "    environment=xvfb_env\n",
-        ")"
+        "training_run = experiment.submit(trainint_config)"
       ]
     },
     {
@@ -313,8 +333,9 @@
         "\n",
         "    # parse arguments ...\n",
         "    \n",
-        "    # Intitialize ray\n",
-        "    ray.init(address=args.ray_address)\n",
+        "    # Start ray head (single node)\n",
+        "    os.system('ray start --head')\n",
+        "    ray.init(address='auto')\n",
         "\n",
         "    # Run training task using tune.run\n",
         "    tune.run(\n",
@@ -328,40 +349,23 @@
         "```"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Submit the estimator to start experiment\n",
-        "Now we use the *training_estimator* to submit a run."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "training_run = exp.submit(training_estimator)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
         "### Monitor experiment\n",
         "\n",
-        "Azure Machine Learning provides a Jupyter widget to show the status of an experiment run. You could use this widget to monitor the status of the runs.\n",
-        "\n",
-        "Note that _ReinforcementLearningEstimator_ creates at least two runs: (a) A parent run, i.e. the run returned above, and (b) a collection of child runs. The number of the child runs depends on the configuration of the reinforcement learning estimator. In our simple scenario, configured above, only one child run will be created.\n",
-        "\n",
-        "The widget will show a list of the child runs as well. You can click on the link under **Status** to see the details of a child run. It will also show the metrics being logged."
+        "Azure Machine Learning provides a Jupyter widget to show the status of an experiment run. You could use this widget to monitor the status of the runs."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646437627002
+        }
+      },
       "outputs": [],
       "source": [
         "from azureml.widgets import RunDetails\n",
@@ -384,7 +388,7 @@
       "outputs": [],
       "source": [
         "# Uncomment line below to cancel the run\n",
-        "#training_run.cancel()"
+        "# training_run.cancel()"
       ]
     },
     {
@@ -406,37 +410,6 @@
         "training_run.wait_for_completion()"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Get a handle to the child run\n",
-        "You can obtain a handle to the child run as follows. In our scenario, there is only one child run, we have it called `child_run_0`."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "import time\n",
-        "\n",
-        "child_run_0 = None\n",
-        "timeout = 30\n",
-        "while timeout > 0 and not child_run_0:\n",
-        "    child_runs = list(training_run.get_children())\n",
-        "    print('Number of child runs:', len(child_runs))\n",
-        "    if len(child_runs) > 0:\n",
-        "        child_run_0 = child_runs[0]\n",
-        "        break\n",
-        "    time.sleep(2) # Wait for 2 seconds\n",
-        "    timeout -= 2\n",
-        "\n",
-        "print('Child run info:')\n",
-        "print(child_run_0)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {},
@@ -453,8 +426,8 @@
       "source": [
         "from azureml.core import Run\n",
         "\n",
-        "run_id = child_run_0.id # Or set to run id of a completed run (e.g. 'rl-cartpole-v0_1587572312_06e04ace_head')\n",
-        "child_run_0 = Run(exp, run_id=run_id)"
+        "run_id = training_run.id # Or set to run id of a completed run (e.g. 'rl-cartpole-v0_1587572312_06e04ace_head')\n",
+        "run = Run(experiment, run_id=run_id)"
       ]
     },
     {
@@ -467,7 +440,11 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646437652309
+        }
+      },
       "outputs": [],
       "source": [
         "from os import path\n",
@@ -480,7 +457,7 @@
         "    dir_util.remove_tree(training_artifacts_path)\n",
         "\n",
         "# Download run artifacts to local compute\n",
-        "child_run_0.download_files(training_artifacts_path)"
+        "training_run.download_files(training_artifacts_path)"
       ]
     },
     {
@@ -497,7 +474,11 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646437657045
+        }
+      },
       "outputs": [],
       "source": [
         "import shutil\n",
@@ -516,15 +497,10 @@
         "\n",
         "\n",
         "# A helper function to display a movie\n",
-        "from IPython.core.display import display, HTML\n",
+        "from IPython.core.display import Video\n",
+        "from IPython.display import display\n",
         "def display_movie(movie_file):\n",
-        "    display(\n",
-        "        HTML('\\\n",
-        "            <video alt=\"cannot display video\" autoplay loop> \\\n",
-        "                <source src=\"{}\" type=\"video/mp4\"> \\\n",
-        "            </video>'.format(movie_file)\n",
-        "        )\n",
-        "    )"
+        "    display(Video(movie_file, embed=True, html_attributes='controls'))"
       ]
     },
     {
@@ -537,7 +513,11 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646437690241
+        }
+      },
       "outputs": [],
       "source": [
         "mp4_files = find_movies(training_artifacts_path)\n",
@@ -554,7 +534,11 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646437692954
+        }
+      },
       "outputs": [],
       "source": [
         "first_movie = mp4_files[0] if len(mp4_files) > 0 else None\n",
@@ -573,7 +557,11 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {},
+      "metadata": {
+        "gather": {
+          "logged": 1646437717147
+        }
+      },
       "outputs": [],
       "source": [
         "last_movie = mp4_files[-1] if len(mp4_files) > 0 else None\n",
@@ -597,8 +585,8 @@
       "metadata": {},
       "source": [
         "### Evaluate a trained policy\n",
-        "We need to configure another reinforcement learning estimator, `rollout_estimator`, and then use it to submit another run. Note that the entry script for this estimator now points to `cartpole-rollout.py` script.\n",
-        "Also note how we pass the checkpoints dataset to this script using `inputs` parameter of the _ReinforcementLearningEstimator_.\n",
+        "In this section, we submit another job, to evalute a trained policy. The entrypoint for this job is\n",
+        "`cartpole-rollout.py` script, and we we pass the checkpoints dataset to this script as a dataset refrence.\n",
         "\n",
         "We are using script parameters to pass in the same algorithm and the same environment used during training. We also specify the checkpoint number of the checkpoint we wish to evaluate, `checkpoint-number`, and number of the steps we shall run the rollout, `steps`.\n",
         "\n",
@@ -663,7 +651,8 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "Now let's configure rollout estimator. Note that we use the last checkpoint for evaluation. The assumption is that the last checkpoint points to our best trained agent. You may change this to any of the checkpoint numbers printed above and observe the effect."
+        "You can submit the training run using a `ScriptRunConfig`. By providing the\n",
+        "command to run the training, and a `RunConfig` object configured w"
       ]
     },
     {
@@ -672,94 +661,51 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "script_params = {    \n",
-        "    # Checkpoint number of the checkpoint from which to roll out\n",
-        "    \"--checkpoint-number\": last_checkpoint_number,\n",
+        "ray_environment_name = 'cartpole-ray-sc'\n",
         "\n",
-        "    # Training algorithm\n",
-        "    \"--run\": training_algorithm,\n",
-        "    \n",
-        "    # Training environment\n",
-        "    \"--env\": rl_environment,\n",
-        "    \n",
-        "    # Algorithm-specific parameters\n",
-        "    \"--config\": '{}',\n",
-        "    \n",
-        "    # Number of rollout steps \n",
-        "    \"--steps\": 2000,\n",
-        "    \n",
-        "    # If should repress rendering of the environment\n",
-        "    \"--no-render\": \"\",\n",
-        "    \n",
-        "    # The place where recorded videos will be stored\n",
-        "    \"--video-dir\": \"./logs/video\"\n",
-        "}\n",
+        "experiment_name = 'CartPole-v0-SC'\n",
+        "training_algorithm = 'PPO'\n",
+        "rl_environment = 'CartPole-v0'\n",
         "\n",
+        "experiment = Experiment(workspace=ws, name=experiment_name)\n",
+        "ray_environment = Environment.get(workspace=ws, name=ray_environment_name)\n",
+        "\n",
+        "script_name = 'cartpole_rollout.py'\n",
+        "video_capture = True\n",
         "if video_capture:\n",
-        "    script_params.pop(\"--no-render\")\n",
+        "    script_arguments = ['--video-dir', './logs/video']\n",
         "else:\n",
-        "    script_params.pop(\"--video-dir\")\n",
+        "    script_arguments = ['--no-render']\n",
+        "script_arguments = script_arguments + [\n",
+        "    '--run', training_algorithm,\n",
+        "    '--env', rl_environment,\n",
+        "    '--config', '{}',\n",
+        "    '--steps', '2000',\n",
+        "    '--checkpoint-number', str(last_checkpoint_number),\n",
+        "    '--artifacts-dataset', checkpoint_ds.as_named_input('artifacts_dataset'),\n",
+        "    '--artifacts-path', checkpoint_ds.as_named_input('artifacts_path').as_mount()\n",
+        "]\n",
         "\n",
+        "command = [\"python\", script_name, *script_arguments]\n",
         "\n",
-        "# Ray's video capture support requires to run everything under a headless display driver called (xvfb).\n",
-        "# There are two parts to this:\n",
-        "\n",
-        "# 1. Use a custom docker file with proper instructions to install xvfb, ffmpeg, python-opengl\n",
-        "# and other dependencies.\n",
-        "# Note: Even when the rendering is off pyhton-opengl is needed.\n",
-        "\n",
-        "with open(\"files/docker/Dockerfile\", \"r\") as f:\n",
-        "    dockerfile=f.read()\n",
-        "\n",
-        "xvfb_env = Environment(name='xvfb-vdisplay')\n",
-        "xvfb_env.docker.base_image = None\n",
-        "xvfb_env.docker.base_dockerfile = dockerfile\n",
-        "    \n",
-        "# 2.  Execute the Python process via the xvfb-run command to set up the headless display driver.\n",
-        "xvfb_env.python.user_managed_dependencies = True\n",
         "if video_capture:\n",
-        "    xvfb_env.python.interpreter_path = \"xvfb-run -s '-screen 0 640x480x16 -ac +extension GLX +render' python\"\n",
+        "    command = [\"xvfb-run -s '-screen 0 640x480x16 -ac +extension GLX +render' \"] + command\n",
+        "    run_config.environment_variables[\"SDL_VIDEODRIVER\"] = \"dummy\"\n",
         "\n",
+        "run_config = RunConfiguration(communicator='OpenMpi')\n",
+        "run_config.target = compute_target\n",
+        "run_config.docker = DockerConfiguration(use_docker=True)\n",
+        "run_config.node_count = 1\n",
+        "run_config.environment = ray_environment\n",
         "\n",
-        "rollout_estimator = ReinforcementLearningEstimator(\n",
-        "    # Location of source files\n",
-        "    source_directory='files',\n",
-        "    \n",
-        "    # Python script file\n",
-        "    entry_script='cartpole_rollout.py',\n",
-        "    \n",
-        "    # A dictionary of arguments to pass to the rollout script specified in ``entry_script``\n",
-        "    script_params = script_params,\n",
-        "    \n",
-        "    # Data inputs\n",
-        "    inputs=[\n",
-        "        checkpoint_ds.as_named_input('artifacts_dataset'),\n",
-        "        checkpoint_ds.as_named_input('artifacts_path').as_mount()],\n",
-        "    \n",
-        "    # The Azure Machine Learning compute target set up for Ray head nodes\n",
-        "    compute_target=compute_target,\n",
-        "    \n",
-        "    # Reinforcement learning framework. Currently must be Ray.\n",
-        "    rl_framework=Ray(),\n",
-        "    \n",
-        "    # Custom environmnet for Xvfb\n",
-        "    environment=xvfb_env)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "Same as before, we use the *rollout_estimator* to submit a run."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "rollout_run = exp.submit(rollout_estimator)"
+        "rollout_config = ScriptRunConfig(\n",
+        "                    source_directory='./files',\n",
+        "                    command=command,\n",
+        "                    run_config=run_config\n",
+        "                   )\n",
+        "\n",
+        "rollout_run = experiment.submit(rollout_config)\n",
+        "rollout_run"
       ]
     },
     {
@@ -811,11 +757,6 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "# Get a handle to child run\n",
-        "child_runs = list(rollout_run.get_children())\n",
-        "print('Number of child runs:', len(child_runs))\n",
-        "child_run_0 = child_runs[0]\n",
-        "\n",
         "# Download rollout artifacts\n",
         "rollout_artifacts_path = path.join(\"logs\", \"rollout\")\n",
         "print(\"Rollout artifacts path:\", rollout_artifacts_path)\n",
@@ -824,7 +765,7 @@
         "    dir_util.remove_tree(rollout_artifacts_path)\n",
         "\n",
         "# Download videos to local compute\n",
-        "child_run_0.download_files(\"logs/video\", output_directory = rollout_artifacts_path)"
+        "rollout_run.download_files(\"logs/video\", output_directory = rollout_artifacts_path)"
       ]
     },
     {
@@ -914,6 +855,16 @@
         "name": "dasommer"
       }
     ],
+    "categories": [
+      "how-to-use-azureml",
+      "reinforcement-learning"
+    ],
+    "interpreter": {
+      "hash": "13382f70c1d0595120591d2e358c8d446daf961bf951d1fba9a32631e205d5ab"
+    },
+    "kernel_info": {
+      "name": "python38-azureml"
+    },
     "kernelspec": {
       "display_name": "Python 3.6",
       "language": "python",
@@ -929,10 +880,13 @@
       "name": "python",
       "nbconvert_exporter": "python",
       "pygments_lexer": "ipython3",
-      "version": "3.6.9"
+      "version": "3.7.9"
     },
-    "notice": "Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License."
+    "notice": "Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License.",
+    "nteract": {
+      "version": "nteract-front-end@1.0.0"
+    }
   },
   "nbformat": 4,
-  "nbformat_minor": 4
+  "nbformat_minor": 0
 }

how-to-use-azureml/reinforcement-learning/cartpole-on-single-compute/files/cartpole_rollout.py

@@ -1,4 +1,3 @@
-import argparse
 import os
 import sys
 
@@ -11,10 +10,7 @@ from azureml.core import Run
 from utils import callbacks
 
 
-DEFAULT_RAY_ADDRESS = 'localhost:6379'
-
-
-def run_rollout(args, parser, ray_address):
+def run_rollout(args, parser):
 
     config = args.config
     if not args.env:
@@ -22,8 +18,6 @@ def run_rollout(args, parser, ray_address):
             parser.error("the following arguments are required: --env")
         args.env = config.get("env")
 
-    ray.init(address=ray_address)
-
     # Create the Trainer from config.
     cls = get_trainable_cls(args.run)
     agent = cls(env=args.env, config=config)
@@ -76,6 +70,10 @@ def run_rollout(args, parser, ray_address):
 
 if __name__ == "__main__":
 
+    # Start ray head (single node)
+    os.system('ray start --head')
+    ray.init(address='auto')
+
     # Add positional argument - serves as placeholder for checkpoint
     argvc = sys.argv[1:]
     argvc.insert(0, 'checkpoint-placeholder')
@@ -88,8 +86,12 @@ if __name__ == "__main__":
         help='Checkpoint number of the checkpoint from which to roll out')
 
     rollout_parser.add_argument(
-        '--ray-address', required=False, default=DEFAULT_RAY_ADDRESS,
-        help='The address of the Ray cluster to connect to')
+        '--artifacts-dataset', required=True,
+        help='The checkpoints artifacts dataset')
+
+    rollout_parser.add_argument(
+        '--artifacts-path', required=True,
+        help='The checkpoints artifacts path')
 
     args = rollout_parser.parse_args(argvc)
 
@@ -116,4 +118,4 @@ if __name__ == "__main__":
     args.checkpoint = checkpoint
 
     # Start rollout
-    run_rollout(args, rollout_parser, args.ray_address)
+    run_rollout(args, rollout_parser)

how-to-use-azureml/reinforcement-learning/cartpole-on-single-compute/files/cartpole_training.py

@@ -1,17 +1,10 @@
-import argparse
 import os
-import sys
-
 import ray
 from ray.rllib import train
 from ray import tune
 
 from utils import callbacks
 
-
-DEFAULT_RAY_ADDRESS = 'localhost:6379'
-
-
 if __name__ == "__main__":
 
     # Parse arguments and add callbacks to config
@@ -24,11 +17,9 @@ if __name__ == "__main__":
     if 'monitor' in args.config and args.config['monitor']:
         print("Video capturing is ON!")
 
-    # Start (connect to) Ray cluster
-    if args.ray_address is None:
-        args.ray_address = DEFAULT_RAY_ADDRESS
-
-    ray.init(address=args.ray_address)
+    # Start ray head (single node)
+    os.system('ray start --head')
+    ray.init(address='auto')
 
     # Run training task using tune.run
     tune.run(

how-to-use-azureml/reinforcement-learning/cartpole-on-single-compute/files/docker/Dockerfile

@@ -8,7 +8,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     rm -rf /var/lib/apt/lists/* && \
     rm -rf /usr/share/man/*
 
-RUN conda install -y conda=4.7.12 python=3.7 && conda clean -ay && \
+RUN conda install -y conda=4.12.0 python=3.7 && conda clean -ay
+RUN pip install ray-on-aml==0.1.6 & \
     pip install --no-cache-dir \
     azureml-defaults \
     azureml-dataset-runtime[fuse,pandas] \
@@ -22,10 +23,12 @@ RUN conda install -y conda=4.7.12 python=3.7 && conda clean -ay && \
     tabulate \
     dm_tree \
     lz4 \
-    ray==0.8.3 \
-    ray[rllib,dashboard,tune]==0.8.3 \
     psutil \
     setproctitle \
-    gym[atari] && \
+    pygame \
+    gym[classic_control]==0.19.0 && \
     conda install -y -c conda-forge x264='1!152.20180717' ffmpeg=4.0.2 && \
     conda install -c anaconda opencv
+
+RUN pip install --upgrade ray==0.8.3 \
+    ray[rllib,dashboard,tune]==0.8.3