Merge pull request #302 from rastala/master

version 1.0.23

Dockerfiles/1.0.21/Dockerfile

@@ -0,0 +1,29 @@
+FROM continuumio/miniconda:4.5.11
+
+# install git
+RUN apt-get update && apt-get upgrade -y && apt-get install -y git
+
+# create a new conda environment named azureml
+RUN conda create -n azureml -y -q Python=3.6
+
+# install additional packages used by sample notebooks. this is optional
+RUN ["/bin/bash", "-c", "source activate azureml && conda install -y tqdm cython matplotlib scikit-learn"]
+
+# install azurmel-sdk components
+RUN ["/bin/bash", "-c", "source activate azureml && pip install azureml-sdk[notebooks]==1.0.21"]
+
+# clone Azure ML GitHub sample notebooks
+RUN cd /home && git clone -b "azureml-sdk-1.0.21" --single-branch https://github.com/Azure/MachineLearningNotebooks.git
+
+# generate jupyter configuration file
+RUN ["/bin/bash", "-c", "source activate azureml && mkdir ~/.jupyter && cd ~/.jupyter && jupyter notebook --generate-config"]
+
+# set an emtpy token for Jupyter to remove authentication. 
+# this is NOT recommended for production environment
+RUN echo "c.NotebookApp.token = ''" >> ~/.jupyter/jupyter_notebook_config.py
+
+# open up port 8887 on the container
+EXPOSE 8887
+
+# start Jupyter notebook server on port 8887 when the container starts
+CMD /bin/bash -c "cd /home/MachineLearningNotebooks && source activate azureml && jupyter notebook --port 8887 --no-browser --ip 0.0.0.0 --allow-root"

how-to-use-azureml/automated-machine-learning/README.md

@@ -189,6 +189,11 @@ jupyter notebook
     - Dataset: [Dominick's grocery sales of orange juice](forecasting-b/dominicks_OJ.csv)
     - Example of training an AutoML forecasting model on multiple time-series
 
+- [auto-ml-classification-with-onnx.ipynb](classification-with-onnx/auto-ml-classification-with-onnx.ipynb)
+    - Dataset: scikit learn's [digit dataset](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_digits.html#sklearn.datasets.load_digits)
+    - Simple example of using Auto ML for classification with ONNX models
+    - Uses local compute for training
+
 <a name="documentation"></a>
 See [Configure automated machine learning experiments](https://docs.microsoft.com/azure/machine-learning/service/how-to-configure-auto-train) to learn how more about the the settings and features available for automated machine learning experiments.
 
@@ -233,6 +238,13 @@ If a sample notebook fails with an error that property, method or library does n
 ## Numpy import fails on Windows
 Some Windows environments see an error loading numpy with the latest Python version 3.6.8.  If you see this issue, try with Python version 3.6.7.
 
+## Numpy import fails
+Check the tensorflow version in the automated ml conda environment. Supported versions are < 1.13. Uninstall tensorflow from the environment if version is >= 1.13
+You may check the version of tensorflow and uninstall as follows
+1) start a command shell, activate conda environment where automated ml packages are installed
+2) enter `pip freeze` and look for `tensorflow` , if found, the version listed should be < 1.13 
+3) If the listed version is a not a supported version,  `pip uninstall tensorflow` in the command shell and enter y for confirmation. 
+
 ## Remote run: DsvmCompute.create fails 
 There are several reasons why the DsvmCompute.create can fail.  The reason is usually in the error message but you have to look at the end of the error message for the detailed reason.  Some common reasons are:
 1) `Compute name is invalid, it should start with a letter, be between 2 and 16 character, and only include letters (a-zA-Z), numbers (0-9) and \'-\'.`  Note that underscore is not allowed in the name.

how-to-use-azureml/automated-machine-learning/automl_env.yml

@@ -0,0 +1,22 @@
+name: azure_automl
+dependencies:
+  # The python interpreter version.
+  # Currently Azure ML only supports 3.5.2 and later.
+- python>=3.5.2,<3.6.8
+- nb_conda
+- matplotlib==2.1.0
+- numpy>=1.11.0,<1.15.0
+- cython
+- urllib3<1.24
+- scipy>=1.0.0,<=1.1.0
+- scikit-learn>=0.18.0,<=0.19.1
+- pandas>=0.22.0,<0.23.0
+- tensorflow>=1.12.0
+- py-xgboost<=0.80
+
+- pip:
+  # Required packages for AzureML execution, history, and data preparation.
+  - azureml-sdk[automl,explain]
+  - azureml-widgets
+  - pandas_ml
+  

how-to-use-azureml/automated-machine-learning/automl_env_mac.yml

@@ -0,0 +1,23 @@
+name: azure_automl
+dependencies:
+  # The python interpreter version.
+  # Currently Azure ML only supports 3.5.2 and later.
+- python>=3.5.2,<3.6.8
+- nb_conda
+- matplotlib==2.1.0
+- numpy>=1.15.3
+- cython
+- urllib3<1.24
+- scipy>=1.0.0,<=1.1.0
+- scikit-learn>=0.18.0,<=0.19.1
+- pandas>=0.22.0,<0.23.0
+- tensorflow>=1.12.0
+- py-xgboost<=0.80
+
+- pip:
+  # Required packages for AzureML execution, history, and data preparation.
+  - azureml-sdk[automl,explain]
+  - azureml-widgets
+  - pandas_ml
+  
+

how-to-use-azureml/automated-machine-learning/automl_setup.cmd

@@ -0,0 +1,51 @@
+@echo off
+set conda_env_name=%1
+set automl_env_file=%2
+set options=%3
+set PIP_NO_WARN_SCRIPT_LOCATION=0
+
+IF "%conda_env_name%"=="" SET conda_env_name="azure_automl"
+IF "%automl_env_file%"=="" SET automl_env_file="automl_env.yml"
+
+IF NOT EXIST %automl_env_file% GOTO YmlMissing
+
+call conda activate %conda_env_name% 2>nul:
+
+if not errorlevel 1 (
+  echo Upgrading azureml-sdk[automl,notebooks,explain] in existing conda environment %conda_env_name%
+  call pip install --upgrade azureml-sdk[automl,notebooks,explain]
+  if errorlevel 1 goto ErrorExit
+) else (
+  call conda env create -f %automl_env_file% -n %conda_env_name%
+)
+
+call conda activate %conda_env_name% 2>nul:
+if errorlevel 1 goto ErrorExit
+
+call python -m ipykernel install --user --name %conda_env_name% --display-name "Python (%conda_env_name%)"
+
+REM azureml.widgets is now installed as part of the pip install under the conda env.
+REM Removing the old user install so that the notebooks will use the latest widget.
+call jupyter nbextension uninstall --user --py azureml.widgets
+
+echo.
+echo.
+echo ***************************************
+echo * AutoML setup completed successfully *
+echo ***************************************
+IF NOT "%options%"=="nolaunch" (
+  echo.
+  echo Starting jupyter notebook - please run the configuration notebook 
+  echo.
+  jupyter notebook --log-level=50 --notebook-dir='..\..'
+)
+
+goto End
+
+:YmlMissing
+echo File %automl_env_file% not found.
+
+:ErrorExit
+echo Install failed
+
+:End

how-to-use-azureml/automated-machine-learning/automl_setup_linux.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+CONDA_ENV_NAME=$1
+AUTOML_ENV_FILE=$2
+OPTIONS=$3
+PIP_NO_WARN_SCRIPT_LOCATION=0
+
+if [ "$CONDA_ENV_NAME" == "" ]
+then
+  CONDA_ENV_NAME="azure_automl"
+fi
+
+if [ "$AUTOML_ENV_FILE" == "" ]
+then
+  AUTOML_ENV_FILE="automl_env.yml"
+fi
+
+if [ ! -f $AUTOML_ENV_FILE ]; then
+    echo "File $AUTOML_ENV_FILE not found"
+    exit 1
+fi
+
+if source activate $CONDA_ENV_NAME 2> /dev/null
+then
+   echo "Upgrading azureml-sdk[automl,notebooks,explain] in existing conda environment" $CONDA_ENV_NAME
+   pip install --upgrade azureml-sdk[automl,notebooks,explain] &&
+   jupyter nbextension uninstall --user --py azureml.widgets
+else
+   conda env create -f $AUTOML_ENV_FILE -n $CONDA_ENV_NAME &&
+   source activate $CONDA_ENV_NAME &&
+   python -m ipykernel install --user --name $CONDA_ENV_NAME --display-name "Python ($CONDA_ENV_NAME)" &&
+   jupyter nbextension uninstall --user --py azureml.widgets &&
+   echo "" &&
+   echo "" &&
+   echo "***************************************" &&
+   echo "* AutoML setup completed successfully *" &&
+   echo "***************************************" &&
+   if [ "$OPTIONS" != "nolaunch" ]
+   then
+      echo "" &&
+      echo "Starting jupyter notebook - please run the configuration notebook" &&
+      echo "" &&
+      jupyter notebook --log-level=50 --notebook-dir '../..'
+   fi
+fi
+
+if [ $? -gt 0 ]
+then
+   echo "Installation failed"
+fi
+
+

how-to-use-azureml/automated-machine-learning/automl_setup_mac.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+CONDA_ENV_NAME=$1
+AUTOML_ENV_FILE=$2
+OPTIONS=$3
+PIP_NO_WARN_SCRIPT_LOCATION=0
+
+if [ "$CONDA_ENV_NAME" == "" ]
+then
+  CONDA_ENV_NAME="azure_automl"
+fi
+
+if [ "$AUTOML_ENV_FILE" == "" ]
+then
+  AUTOML_ENV_FILE="automl_env_mac.yml"
+fi
+
+if [ ! -f $AUTOML_ENV_FILE ]; then
+    echo "File $AUTOML_ENV_FILE not found"
+    exit 1
+fi
+
+if source activate $CONDA_ENV_NAME 2> /dev/null
+then
+   echo "Upgrading azureml-sdk[automl,notebooks,explain] in existing conda environment" $CONDA_ENV_NAME
+   pip install --upgrade azureml-sdk[automl,notebooks,explain] &&
+   jupyter nbextension uninstall --user --py azureml.widgets
+else
+   conda env create -f $AUTOML_ENV_FILE -n $CONDA_ENV_NAME &&
+   source activate $CONDA_ENV_NAME &&
+   conda install lightgbm -c conda-forge -y &&
+   python -m ipykernel install --user --name $CONDA_ENV_NAME --display-name "Python ($CONDA_ENV_NAME)" &&
+   jupyter nbextension uninstall --user --py azureml.widgets &&
+   pip install numpy==1.15.3 &&
+   echo "" &&
+   echo "" &&
+   echo "***************************************" &&
+   echo "* AutoML setup completed successfully *" &&
+   echo "***************************************" &&
+   if [ "$OPTIONS" != "nolaunch" ]
+   then
+      echo "" &&
+      echo "Starting jupyter notebook - please run the configuration notebook" &&
+      echo "" &&
+      jupyter notebook --log-level=50 --notebook-dir '../..'
+   fi
+fi
+
+if [ $? -gt 0 ]
+then
+   echo "Installation failed"
+fi
+
+
+

how-to-use-azureml/automated-machine-learning/classification-with-deployment/auto-ml-classification-with-deployment.ipynb

@@ -139,7 +139,6 @@
         "                             primary_metric = 'AUC_weighted',\n",
         "                             iteration_timeout_minutes = 20,\n",
         "                             iterations = 10,\n",
-        "                             n_cross_validations = 2,\n",
         "                             verbosity = logging.INFO,\n",
         "                             X = X_train, \n",
         "                             y = y_train,\n",
@@ -263,7 +262,7 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "To ensure the fit results are consistent with the training results, the SDK dependency versions need to be the same as the environment that trains the model. Details about retrieving the versions can be found in notebook [12.auto-ml-retrieve-the-training-sdk-versions](12.auto-ml-retrieve-the-training-sdk-versions.ipynb)."
+        "To ensure the fit results are consistent with the training results, the SDK dependency versions need to be the same as the environment that trains the model. The following cells create a file, myenv.yml, which specifies the dependencies from the run."
       ]
     },
     {

how-to-use-azureml/automated-machine-learning/classification-with-onnx/auto-ml-classification-with-onnx.ipynb

@@ -0,0 +1,284 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
+        "\n",
+        "Licensed under the MIT License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# Automated Machine Learning\n",
+        "_**Classification with Local Compute**_\n",
+        "\n",
+        "## Contents\n",
+        "1. [Introduction](#Introduction)\n",
+        "1. [Setup](#Setup)\n",
+        "1. [Data](#Data)\n",
+        "1. [Train](#Train)\n",
+        "1. [Results](#Results)\n",
+        "1. [Test](#Test)\n",
+        "\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Introduction\n",
+        "\n",
+        "In this example we use the scikit-learn's [digit dataset](http://scikit-learn.org/stable/datasets/index.html#optical-recognition-of-handwritten-digits-dataset) to showcase how you can use AutoML for a simple classification problem.\n",
+        "\n",
+        "Make sure you have executed the [configuration](../../../configuration.ipynb) before running this notebook.\n",
+        "\n",
+        "Please find the ONNX related documentations [here](https://github.com/onnx/onnx).\n",
+        "\n",
+        "In this notebook you will learn how to:\n",
+        "1. Create an `Experiment` in an existing `Workspace`.\n",
+        "2. Configure AutoML using `AutoMLConfig`.\n",
+        "3. Train the model using local compute with ONNX compatible config on.\n",
+        "4. Explore the results and save the ONNX model."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Setup\n",
+        "\n",
+        "As part of the setup you have already created an Azure ML `Workspace` object. For AutoML you will need to create an `Experiment` object, which is a named object in a `Workspace` used to run experiments."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import logging\n",
+        "\n",
+        "from matplotlib import pyplot as plt\n",
+        "import numpy as np\n",
+        "import pandas as pd\n",
+        "from sklearn import datasets\n",
+        "\n",
+        "import azureml.core\n",
+        "from azureml.core.experiment import Experiment\n",
+        "from azureml.core.workspace import Workspace\n",
+        "from azureml.train.automl import AutoMLConfig"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "ws = Workspace.from_config()\n",
+        "\n",
+        "# Choose a name for the experiment and specify the project folder.\n",
+        "experiment_name = 'automl-classification-onnx'\n",
+        "project_folder = './sample_projects/automl-classification-onnx'\n",
+        "\n",
+        "experiment = Experiment(ws, experiment_name)\n",
+        "\n",
+        "output = {}\n",
+        "output['SDK version'] = azureml.core.VERSION\n",
+        "output['Subscription ID'] = ws.subscription_id\n",
+        "output['Workspace Name'] = ws.name\n",
+        "output['Resource Group'] = ws.resource_group\n",
+        "output['Location'] = ws.location\n",
+        "output['Project Directory'] = project_folder\n",
+        "output['Experiment Name'] = experiment.name\n",
+        "pd.set_option('display.max_colwidth', -1)\n",
+        "outputDf = pd.DataFrame(data = output, index = [''])\n",
+        "outputDf.T"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Data\n",
+        "\n",
+        "This uses scikit-learn's [load_digits](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_digits.html) method."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "digits = datasets.load_digits()\n",
+        "\n",
+        "# Exclude the first 100 rows from training so that they can be used for test.\n",
+        "X_train = digits.data[100:,:]\n",
+        "y_train = digits.target[100:]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Train with enable ONNX compatible models config on\n",
+        "\n",
+        "Instantiate an `AutoMLConfig` object to specify the settings and data used to run the experiment.\n",
+        "\n",
+        "Set the parameter enable_onnx_compatible_models=True, if you also want to generate the ONNX compatible models. Please note, the forecasting task and TensorFlow models are not ONNX compatible yet.\n",
+        "\n",
+        "|Property|Description|\n",
+        "|-|-|\n",
+        "|**task**|classification or regression|\n",
+        "|**primary_metric**|This is the metric that you want to optimize. Classification supports the following primary metrics: <br><i>accuracy</i><br><i>AUC_weighted</i><br><i>average_precision_score_weighted</i><br><i>norm_macro_recall</i><br><i>precision_score_weighted</i>|\n",
+        "|**iteration_timeout_minutes**|Time limit in minutes for each iteration.|\n",
+        "|**iterations**|Number of iterations. In each iteration AutoML trains a specific pipeline with the data.|\n",
+        "|**X**|(sparse) array-like, shape = [n_samples, n_features]|\n",
+        "|**y**|(sparse) array-like, shape = [n_samples, ], Multi-class targets.|\n",
+        "|**enable_onnx_compatible_models**|Enable the ONNX compatible models in the experiment.|\n",
+        "|**path**|Relative path to the project folder. AutoML stores configuration files for the experiment under this folder. You can specify a new empty folder.|"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "automl_config = AutoMLConfig(task = 'classification',\n",
+        "                             debug_log = 'automl_errors.log',\n",
+        "                             primary_metric = 'AUC_weighted',\n",
+        "                             iteration_timeout_minutes = 60,\n",
+        "                             iterations = 10,\n",
+        "                             verbosity = logging.INFO,\n",
+        "                             X = X_train, \n",
+        "                             y = y_train,\n",
+        "                             enable_onnx_compatible_models=True,\n",
+        "                             path = project_folder)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Call the `submit` method on the experiment object and pass the run configuration. Execution of local runs is synchronous. Depending on the data and the number of iterations this can run for a while.\n",
+        "In this example, we specify `show_output = True` to print currently running iterations to the console."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "local_run = experiment.submit(automl_config, show_output = True)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "local_run"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Results"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Widget for Monitoring Runs\n",
+        "\n",
+        "The widget will first report a \"loading\" status while running the first iteration. After completing the first iteration, an auto-updating graph and table will be shown. The widget will refresh once per minute, so you should see the graph update as child runs complete.\n",
+        "\n",
+        "**Note:** The widget displays a link at the bottom. Use this link to open a web interface to explore the individual run details."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.widgets import RunDetails\n",
+        "RunDetails(local_run).show() "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Retrieve the Best ONNX Model\n",
+        "\n",
+        "Below we select the best pipeline from our iterations. The `get_output` method returns the best run and the fitted model. The Model includes the pipeline and any pre-processing.  Overloads on `get_output` allow you to retrieve the best run and fitted model for *any* logged metric or for a particular *iteration*.\n",
+        "\n",
+        "Set the parameter return_onnx_model=True to retrieve the best ONNX model, instead of the Python model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "best_run, onnx_mdl = local_run.get_output(return_onnx_model=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Save the best ONNX model"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.train.automl._vendor.automl.client.core.common.onnx_convert import OnnxConverter\n",
+        "onnx_fl_path = \"./best_model.onnx\"\n",
+        "OnnxConverter.save_onnx_model(onnx_mdl, onnx_fl_path)"
+      ]
+    }
+  ],
+  "metadata": {
+    "authors": [
+      {
+        "name": "savitam"
+      }
+    ],
+    "kernelspec": {
+      "display_name": "Python 3.6",
+      "language": "python",
+      "name": "python36"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.6.6"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 2
+}

how-to-use-azureml/automated-machine-learning/classification-with-whitelisting/auto-ml-classification-with-whitelisting.ipynb

@@ -71,11 +71,17 @@
         "import azureml.core\n",
         "from azureml.core.experiment import Experiment\n",
         "from azureml.core.workspace import Workspace\n",
-        "try:\n",
+        "import sys\n",
+        "whitelist_models=[\"LightGBM\"]\n",
+        "if \"3.7\" != sys.version[0:3]:\n",
+        "    try:\n",
         "        import tensorflow as tf1\n",
-        "except ImportError:\n",
+        "    except ImportError:\n",
         "        from pip._internal import main\n",
         "        main(['install', 'tensorflow>=1.10.0,<=1.12.0'])\n",
+        "        logging.getLogger().setLevel(logging.ERROR)\n",
+        "    whitelist_models=[\"TensorFlowLinearClassifier\", \"TensorFlowDNN\"]\n",
+        "\n",
         "from azureml.train.automl import AutoMLConfig"
       ]
     },
@@ -160,12 +166,11 @@
         "                             primary_metric = 'AUC_weighted',\n",
         "                             iteration_timeout_minutes = 60,\n",
         "                             iterations = 10,\n",
-        "                             n_cross_validations = 3,\n",
         "                             verbosity = logging.INFO,\n",
         "                             X = X_train, \n",
         "                             y = y_train,\n",
         "                             enable_tf=True,\n",
-        "                             whitelist_models=[\"TensorFlowLinearClassifier\", \"TensorFlowDNN\"],\n",
+        "                             whitelist_models=whitelist_models,\n",
         "                             path = project_folder)"
       ]
     },

how-to-use-azureml/automated-machine-learning/classification/auto-ml-classification.ipynb

@@ -135,7 +135,6 @@
         "|**primary_metric**|This is the metric that you want to optimize. Classification supports the following primary metrics: <br><i>accuracy</i><br><i>AUC_weighted</i><br><i>average_precision_score_weighted</i><br><i>norm_macro_recall</i><br><i>precision_score_weighted</i>|\n",
         "|**iteration_timeout_minutes**|Time limit in minutes for each iteration.|\n",
         "|**iterations**|Number of iterations. In each iteration AutoML trains a specific pipeline with the data.|\n",
-        "|**n_cross_validations**|Number of cross validation splits.|\n",
         "|**X**|(sparse) array-like, shape = [n_samples, n_features]|\n",
         "|**y**|(sparse) array-like, shape = [n_samples, ], Multi-class targets.|\n",
         "|**path**|Relative path to the project folder. AutoML stores configuration files for the experiment under this folder. You can specify a new empty folder.|"
@@ -152,7 +151,6 @@
         "                             primary_metric = 'AUC_weighted',\n",
         "                             iteration_timeout_minutes = 60,\n",
         "                             iterations = 25,\n",
-        "                             n_cross_validations = 3,\n",
         "                             verbosity = logging.INFO,\n",
         "                             X = X_train, \n",
         "                             y = y_train,\n",

how-to-use-azureml/automated-machine-learning/dataprep/auto-ml-dataprep.ipynb

@@ -163,8 +163,7 @@
         "    \"iterations\" : 2,\n",
         "    \"primary_metric\" : 'AUC_weighted',\n",
         "    \"preprocess\" : False,\n",
-        "    \"verbosity\" : logging.INFO,\n",
-        "    \"n_cross_validations\": 3\n",
+        "    \"verbosity\" : logging.INFO\n",
         "}"
       ]
     },

how-to-use-azureml/automated-machine-learning/missing-data-blacklist-early-termination/auto-ml-missing-data-blacklist-early-termination.ipynb

@@ -154,7 +154,6 @@
         "|**primary_metric**|This is the metric that you want to optimize. Classification supports the following primary metrics: <br><i>accuracy</i><br><i>AUC_weighted</i><br><i>average_precision_score_weighted</i><br><i>norm_macro_recall</i><br><i>precision_score_weighted</i>|\n",
         "|**iteration_timeout_minutes**|Time limit in minutes for each iteration.|\n",
         "|**iterations**|Number of iterations. In each iteration AutoML trains a specific pipeline with the data.|\n",
-        "|**n_cross_validations**|Number of cross validation splits.|\n",
         "|**preprocess**|Setting this to *True* enables AutoML to perform preprocessing on the input to handle *missing data*, and to perform some common *feature extraction*.|\n",
         "|**experiment_exit_score**|*double* value indicating the target for *primary_metric*. <br>Once the target is surpassed the run terminates.|\n",
         "|**blacklist_models**|*List* of *strings* indicating machine learning algorithms for AutoML to avoid in this run.<br><br> Allowed values for **Classification**<br><i>LogisticRegression</i><br><i>SGD</i><br><i>MultinomialNaiveBayes</i><br><i>BernoulliNaiveBayes</i><br><i>SVM</i><br><i>LinearSVM</i><br><i>KNN</i><br><i>DecisionTree</i><br><i>RandomForest</i><br><i>ExtremeRandomTrees</i><br><i>LightGBM</i><br><i>GradientBoosting</i><br><i>TensorFlowDNN</i><br><i>TensorFlowLinearClassifier</i><br><br>Allowed values for **Regression**<br><i>ElasticNet</i><br><i>GradientBoosting</i><br><i>DecisionTree</i><br><i>KNN</i><br><i>LassoLars</i><br><i>SGD</i><br><i>RandomForest</i><br><i>ExtremeRandomTrees</i><br><i>LightGBM</i><br><i>TensorFlowLinearRegressor</i><br><i>TensorFlowDNN</i>|\n",
@@ -174,7 +173,6 @@
         "                             primary_metric = 'AUC_weighted',\n",
         "                             iteration_timeout_minutes = 60,\n",
         "                             iterations = 20,\n",
-        "                             n_cross_validations = 5,\n",
         "                             preprocess = True,\n",
         "                             experiment_exit_score = 0.9984,\n",
         "                             blacklist_models = ['KNN','LinearSVM'],\n",

how-to-use-azureml/azure-databricks/automl/automl-databricks-local-with-deployment.ipynb

@@ -207,6 +207,7 @@
         "import os\n",
         "import random\n",
         "import time\n",
+        "import json\n",
         "\n",
         "from matplotlib import pyplot as plt\n",
         "from matplotlib.pyplot import imshow\n",
@@ -427,7 +428,7 @@
         "                             debug_log = 'automl_errors.log',\n",
         "                             primary_metric = 'AUC_weighted',\n",
         "                             iteration_timeout_minutes = 10,\n",
-        "                             iterations = 30,\n",
+        "                             iterations = 5,\n",
         "                             preprocess = True,\n",
         "                             n_cross_validations = 10,\n",
         "                             max_concurrent_iterations = 2, #change it based on number of worker nodes\n",
@@ -591,22 +592,21 @@
         "%%writefile score.py\n",
         "import pickle\n",
         "import json\n",
-        "import numpy\n",
+        "import numpy as np\n",
         "import azureml.train.automl\n",
         "from sklearn.externals import joblib\n",
         "from azureml.core.model import Model\n",
-        "\n",
+        "import pandas as pd\n",
         "\n",
         "def init():\n",
         "    global model\n",
-        "    model_path = Model.get_model_path(model_name = '<<modelid>>') # this name is model.id of model that we want to deploy\n",
+        "    model_path = Model.get_model_path(model_name = '<<model_id>>') # this name is model.id of model that we want to deploy\n",
         "    # deserialize the model file back into a sklearn model\n",
         "    model = joblib.load(model_path)\n",
         "\n",
-        "def run(rawdata):\n",
+        "def run(raw_data):\n",
         "    try:\n",
-        "        data = json.loads(rawdata)['data']\n",
-        "        data = numpy.array(data)\n",
+        "        data = (pd.DataFrame(np.array(json.loads(raw_data)['data']), columns=[str(i) for i in range(0,64)]))\n",
         "        result = model.predict(data)\n",
         "    except Exception as e:\n",
         "        result = str(e)\n",
@@ -614,6 +614,22 @@
         "    return json.dumps({\"result\":result.tolist()})"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "#Replace <<model_id>>\n",
+        "content = \"\"\n",
+        "with open(\"score.py\", \"r\") as fo:\n",
+        "    content = fo.read()\n",
+        "\n",
+        "new_content = content.replace(\"<<model_id>>\", local_run.model_id)\n",
+        "with open(\"score.py\", \"w\") as fw:\n",
+        "    fw.write(new_content)"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {},
@@ -672,16 +688,19 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "\n",
         "# this will take 10-15 minutes to finish\n",
         "\n",
-        "service_name = \"<<servicename>>\"\n",
+        "import uuid\n",
+        "from azureml.core.image import ContainerImage\n",
+        "\n",
+        "guid = str(uuid.uuid4()).split(\"-\")[0]\n",
+        "service_name = \"myservice-{}\".format(guid)\n",
+        "print(\"Creating service with name: {}\".format(service_name))\n",
         "runtime = \"spark-py\" \n",
         "driver_file = \"score.py\"\n",
         "my_conda_file = \"mydeployenv.yml\"\n",
         "\n",
         "# image creation\n",
-        "from azureml.core.image import ContainerImage\n",
         "myimage_config = ContainerImage.image_configuration(execution_script = driver_file, \n",
         "                                    runtime = runtime, \n",
         "                                    conda_file = 'mydeployenv.yml')\n",
@@ -744,18 +763,39 @@
       "metadata": {},
       "outputs": [],
       "source": [
+        "import json\n",
         "# Randomly select digits and test.\n",
         "for index in np.random.choice(len(y_test), 2, replace = False):\n",
         "    print(index)\n",
-        "    predicted = fitted_model.predict(X_test[index:index + 1])[0]\n",
+        "    test_sample = json.dumps({'data':X_test[index:index + 1].values.tolist()})\n",
+        "    predicted = myservice.run(input_data = test_sample)\n",
         "    label = y_test.values[index]\n",
-        "    title = \"Label value = %d  Predicted value = %d \" % (label, predicted)\n",
+        "    predictedDict = json.loads(predicted)\n",
+        "    title = \"Label value = %d  Predicted value = %s \" % ( label,predictedDict['result'][0])    \n",
         "    fig = plt.figure(3, figsize = (5,5))\n",
         "    ax1 = fig.add_axes((0,0,.8,.8))\n",
         "    ax1.set_title(title)\n",
         "    plt.imshow(images[index], cmap = plt.cm.gray_r, interpolation = 'nearest')\n",
         "    display(fig)"
       ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "### Delete the service"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "myservice.delete()"
+      ]
     }
   ],
   "metadata": {

how-to-use-azureml/deployment/production-deploy-to-aks/production-deploy-to-aks.ipynb

@@ -167,6 +167,31 @@
         "image.wait_for_creation(show_output = True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Use a custom Docker image\n",
+        "\n",
+        "You can also specify a custom Docker image to be used as base image if you don't want to use the default base image provided by Azure ML. Please make sure the custom Docker image has Ubuntu >= 16.04, Conda >= 4.5.\\* and Python(3.5.\\* or 3.6.\\*).\n",
+        "\n",
+        "Only Supported for `ContainerImage`(from azureml.core.image) with `python` runtime.\n",
+        "```python\n",
+        "# use an image available in public Container Registry without authentication\n",
+        "image_config.base_image = \"mcr.microsoft.com/azureml/o16n-sample-user-base/ubuntu-miniconda\"\n",
+        "\n",
+        "# or, use an image available in a private Container Registry\n",
+        "image_config.base_image = \"myregistry.azurecr.io/mycustomimage:1.0\"\n",
+        "image_config.base_image_registry.address = \"myregistry.azurecr.io\"\n",
+        "image_config.base_image_registry.username = \"username\"\n",
+        "image_config.base_image_registry.password = \"password\"\n",
+        "\n",
+        "# or, use an image built during training.\n",
+        "image_config.base_image = run.properties[\"AzureML.DerivedImageName\"]\n",
+        "```\n",
+        "You can get the address of training image from the properties of a Run object. Only new runs submitted with azureml-sdk>=1.0.22 to AMLCompute targets will have the 'AzureML.DerivedImageName' property. Instructions on how to get a Run can be found in [manage-runs](../../training/manage-runs/manage-runs.ipynb). \n"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {},
@@ -317,7 +342,7 @@
   "metadata": {
     "authors": [
       {
-        "name": "raymondl"
+        "name": "aashishb"
       }
     ],
     "kernelspec": {

how-to-use-azureml/deployment/register-model-create-image-deploy-service/register-model-create-image-deploy-service.ipynb

@@ -261,6 +261,31 @@
         "image.wait_for_creation(show_output = True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Use a custom Docker image\n",
+        "\n",
+        "You can also specify a custom Docker image to be used as base image if you don't want to use the default base image provided by Azure ML. Please make sure the custom Docker image has Ubuntu >= 16.04, Conda >= 4.5.\\* and Python(3.5.\\* or 3.6.\\*).\n",
+        "\n",
+        "Only Supported for `ContainerImage`(from azureml.core.image) with `python` runtime.\n",
+        "```python\n",
+        "# use an image available in public Container Registry without authentication\n",
+        "image_config.base_image = \"mcr.microsoft.com/azureml/o16n-sample-user-base/ubuntu-miniconda\"\n",
+        "\n",
+        "# or, use an image available in a private Container Registry\n",
+        "image_config.base_image = \"myregistry.azurecr.io/mycustomimage:1.0\"\n",
+        "image_config.base_image_registry.address = \"myregistry.azurecr.io\"\n",
+        "image_config.base_image_registry.username = \"username\"\n",
+        "image_config.base_image_registry.password = \"password\"\n",
+        "\n",
+        "# or, use an image built during training.\n",
+        "image_config.base_image = run.properties[\"AzureML.DerivedImageName\"]\n",
+        "```\n",
+        "You can get the address of training image from the properties of a Run object. Only new runs submitted with azureml-sdk>=1.0.22 to AMLCompute targets will have the 'AzureML.DerivedImageName' property. Instructions on how to get a Run can be found in [manage-runs](../../training/manage-runs/manage-runs.ipynb). \n"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {},
@@ -395,7 +420,7 @@
   "metadata": {
     "authors": [
       {
-        "name": "raymondl"
+        "name": "aashishb"
       }
     ],
     "kernelspec": {

how-to-use-azureml/machine-learning-pipelines/README.md

@@ -38,18 +38,19 @@ In this directory, there are two types of notebooks:
 
 * The first type of notebooks will introduce you to core Azure Machine Learning Pipelines features. These notebooks below belong in this category, and are designed to go in sequence; they're all located in the "intro-to-pipelines" folder:
 
-1. [aml-pipelines-getting-started.ipynb](https://aka.ms/pl-get-started)
-2. [aml-pipelines-with-data-dependency-steps.ipynb](https://aka.ms/pl-data-dep)
-3. [aml-pipelines-publish-and-run-using-rest-endpoint.ipynb](https://aka.ms/pl-pub-rep)
-4. [aml-pipelines-data-transfer.ipynb](https://aka.ms/pl-data-trans)
-5. [aml-pipelines-use-databricks-as-compute-target.ipynb](https://aka.ms/pl-databricks)
-6. [aml-pipelines-use-adla-as-compute-target.ipynb](https://aka.ms/pl-adla)
-7. [aml-pipelines-parameter-tuning-with-hyperdrive.ipynb](https://aka.ms/pl-hyperdrive)
-8. [aml-pipelines-how-to-use-azurebatch-to-run-a-windows-executable.ipynb](https://aka.ms/pl-azbatch)
-9. [aml-pipelines-setup-schedule-for-a-published-pipeline.ipynb](https://aka.ms/pl-schedule)
-10. [aml-pipelines-with-automated-machine-learning-step.ipynb](https://aka.ms/pl-automl)
+1. [aml-pipelines-getting-started.ipynb](https://aka.ms/pl-get-started): Start with this notebook to understand the concepts of using Azure Machine Learning Pipelines. This notebook will show you how to runs steps in parallel and in sequence.
+2. [aml-pipelines-with-data-dependency-steps.ipynb](https://aka.ms/pl-data-dep): This notebooks shows how to connect steps in your pipeline using data. Data produced by one step is used by subsequent steps to force an explicit dependency between steps. 
+3. [aml-pipelines-publish-and-run-using-rest-endpoint.ipynb](https://aka.ms/pl-pub-rep): Once you are satisfied with your iterative runs in, you could publish your pipeline to get a REST endpoint which could be invoked from non-Pythons clients as well. 
+4. [aml-pipelines-data-transfer.ipynb](https://aka.ms/pl-data-trans): This notebook shows how you transfer data between supported datastores.
+5. [aml-pipelines-use-databricks-as-compute-target.ipynb](https://aka.ms/pl-databricks): This notebooks shows how you can use Pipelines to send your compute payload to Azure Databricks.
+6. [aml-pipelines-use-adla-as-compute-target.ipynb](https://aka.ms/pl-adla): This notebook shows how you can use Azure Data Lake Analytics (ADLA) as a compute target.
+7. [aml-pipelines-how-to-use-estimatorstep.ipynb](https://aka.ms/pl-estimator): This notebook shows how to use the EstimatorStep.
+7. [aml-pipelines-parameter-tuning-with-hyperdrive.ipynb](https://aka.ms/pl-hyperdrive): HyperDriveStep in Pipelines shows how you can do hyper parameter tuning using Pipelines.
+8. [aml-pipelines-how-to-use-azurebatch-to-run-a-windows-executable.ipynb](https://aka.ms/pl-azbatch): AzureBatchStep can be used to run your custom code in AzureBatch cluster.
+9. [aml-pipelines-setup-schedule-for-a-published-pipeline.ipynb](https://aka.ms/pl-schedule): Once you publish a Pipeline, you can schedule it to trigger based on an interval or on data change in a defined datastore. 
+10. [aml-pipelines-with-automated-machine-learning-step.ipynb](https://aka.ms/pl-automl): AutoMLStep in Pipelines shows how you can do automated machine learning using Pipelines.
 
 * The second type of notebooks illustrate more sophisticated scenarios, and are independent of each other. These notebooks include:
 
-1. [pipeline-batch-scoring.ipynb](https://aka.ms/pl-batch-score)
+1. [pipeline-batch-scoring.ipynb](https://aka.ms/pl-batch-score): This notebook demonstrates how to run a batch scoring job using Azure Machine Learning pipelines.
 2. [pipeline-style-transfer.ipynb](https://aka.ms/pl-style-trans)

how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-data-transfer.ipynb

@@ -141,7 +141,7 @@
         "    print(\"registered blob datastore with name: %s\" % blob_datastore_name)\n",
         "\n",
         "# CLI:\n",
-        "# az ml datastore register-blob -n <datastore-name> -a <account-name> -c <container-name> -k <account-key> [-t <sas-token>]"
+        "# az ml datastore attach-blob -n <datastore-name> -a <account-name> -c <container-name> -k <account-key> [-t <sas-token>]"
       ]
     },
     {

how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-how-to-use-estimatorstep.ipynb

@@ -0,0 +1,281 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
+        "\n",
+        "Licensed under the MIT License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# How to use EstimatorStep in AML Pipeline\n",
+        "\n",
+        "This notebook shows how to use the EstimatorStep with Azure Machine Learning Pipelines. Estimator is a convenient object in Azure Machine Learning that wraps run configuration information to help simplify the tasks of specifying how a script is executed.\n",
+        "\n",
+        "\n",
+        "## Prerequisite:\n",
+        "* Understand the [architecture and terms](https://docs.microsoft.com/azure/machine-learning/service/concept-azure-machine-learning-architecture) introduced by Azure Machine Learning\n",
+        "* Go through the [configuration notebook](../../../configuration.ipynb) to:\n",
+        "    * install the AML SDK\n",
+        "    * create a workspace and its configuration file (`config.json`)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Let's get started. First let's import some Python libraries."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import azureml.core\n",
+        "# check core SDK version number\n",
+        "print(\"Azure ML SDK Version: \", azureml.core.VERSION)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Initialize workspace\n",
+        "Initialize a [Workspace](https://docs.microsoft.com/azure/machine-learning/service/concept-azure-machine-learning-architecture#workspace) object from the existing workspace you created in the Prerequisites step. `Workspace.from_config()` creates a workspace object from the details stored in `config.json`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core import Workspace\n",
+        "ws = Workspace.from_config()\n",
+        "print('Workspace name: ' + ws.name, \n",
+        "      'Azure region: ' + ws.location, \n",
+        "      'Subscription id: ' + ws.subscription_id, \n",
+        "      'Resource group: ' + ws.resource_group, sep = '\\n')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Create or Attach existing AmlCompute\n",
+        "You will need to create a [compute target](https://docs.microsoft.com/azure/machine-learning/service/concept-azure-machine-learning-architecture#compute-target) for training your model. In this tutorial, you create `AmlCompute` as your training compute resource."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "If we could not find the cluster with the given name, then we will create a new cluster here. We will create an `AmlCompute` cluster of `STANDARD_NC6` GPU VMs. This process is broken down into 3 steps:\n",
+        "1. create the configuration (this step is local and only takes a second)\n",
+        "2. create the cluster (this step will take about **20 seconds**)\n",
+        "3. provision the VMs to bring the cluster to the initial size (of 1 in this case). This step will take about **3-5 minutes** and is providing only sparse output in the process. Please make sure to wait until the call returns before moving to the next cell"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.compute import ComputeTarget, AmlCompute\n",
+        "from azureml.core.compute_target import ComputeTargetException\n",
+        "\n",
+        "# choose a name for your cluster\n",
+        "cluster_name = \"cpucluster\"\n",
+        "\n",
+        "try:\n",
+        "    cpu_cluster = ComputeTarget(workspace=ws, name=cluster_name)\n",
+        "    print('Found existing compute target')\n",
+        "except ComputeTargetException:\n",
+        "    print('Creating a new compute target...')\n",
+        "    compute_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_NC6', max_nodes=4)\n",
+        "\n",
+        "    # create the cluster\n",
+        "    cpu_cluster = ComputeTarget.create(ws, cluster_name, compute_config)\n",
+        "\n",
+        "    # can poll for a minimum number of nodes and for a specific timeout. \n",
+        "    # if no min node count is provided it uses the scale settings for the cluster\n",
+        "    cpu_cluster.wait_for_completion(show_output=True, min_node_count=None, timeout_in_minutes=20)\n",
+        "\n",
+        "# use get_status() to get a detailed status for the current cluster. \n",
+        "print(cpu_cluster.get_status().serialize())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Now that you have created the compute target, let's see what the workspace's `compute_targets` property returns. You should now see one entry named 'cpucluster' of type `AmlCompute`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Use a simple script\n",
+        "We have already created a simple \"hello world\" script. This is the script that we will submit through the estimator pattern. It prints a hello-world message, and if Azure ML SDK is installed, it will also logs an array of values ([Fibonacci numbers](https://en.wikipedia.org/wiki/Fibonacci_number))."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Build an Estimator object\n",
+        "Estimator by default will attempt to use Docker-based execution. You can also enable Docker and let estimator pick the default CPU image supplied by Azure ML for execution. You can target an AmlCompute cluster (or any other supported compute target types). You can also customize the conda environment by adding conda and/or pip packages.\n",
+        "\n",
+        "> Note: The arguments to the entry script used in the Estimator object should be specified as *list* using\n",
+        "    'estimator_entry_script_arguments' parameter when instantiating EstimatorStep. Estimator object's parameter\n",
+        "    'script_params' accepts a dictionary. However 'estimator_entry_script_arguments' parameter expects arguments as\n",
+        "    a list.\n",
+        "\n",
+        "> Estimator object initialization involves specifying a list of DataReference objects in its 'inputs' parameter.\n",
+        "    In Pipelines, a step can take another step's output or DataReferences as input. So when creating an EstimatorStep,\n",
+        "    the parameters 'inputs' and 'outputs' need to be set explicitly and that will override 'inputs' parameter\n",
+        "    specified in the Estimator object."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core import Datastore\n",
+        "from azureml.data.data_reference import DataReference\n",
+        "from azureml.pipeline.core import PipelineData\n",
+        "\n",
+        "def_blob_store = Datastore(ws, \"workspaceblobstore\")\n",
+        "\n",
+        "input_data = DataReference(\n",
+        "    datastore=def_blob_store,\n",
+        "    data_reference_name=\"input_data\",\n",
+        "    path_on_datastore=\"20newsgroups/20news.pkl\")\n",
+        "\n",
+        "output = PipelineData(\"output\", datastore=def_blob_store)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.train.estimator import Estimator\n",
+        "\n",
+        "est = Estimator(source_directory='.', \n",
+        "                compute_target=cpu_cluster, \n",
+        "                entry_script='dummy_train.py', \n",
+        "                conda_packages=['scikit-learn'])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Create an EstimatorStep\n",
+        "[EstimatorStep](https://docs.microsoft.com/en-us/python/api/azureml-pipeline-steps/azureml.pipeline.steps.estimator_step.estimatorstep?view=azure-ml-py) adds a step to run Estimator in a Pipeline.\n",
+        "\n",
+        "- **name:** Name of the step\n",
+        "- **estimator:** Estimator object\n",
+        "- **estimator_entry_script_arguments:** \n",
+        "- **runconfig_pipeline_params:** Override runconfig properties at runtime using key-value pairs each with name of the runconfig property and PipelineParameter for that property\n",
+        "- **inputs:** Inputs\n",
+        "- **outputs:** Output is list of PipelineData\n",
+        "- **compute_target:** Compute target to use \n",
+        "- **allow_reuse:** Whether the step should reuse previous results when run with the same settings/inputs. If this is false, a new run will always be generated for this step during pipeline execution.\n",
+        "- **version:** Optional version tag to denote a change in functionality for the step"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.pipeline.steps import EstimatorStep\n",
+        "\n",
+        "est_step = EstimatorStep(name=\"Estimator_Train\", \n",
+        "                         estimator=est, \n",
+        "                         estimator_entry_script_arguments=[\"--datadir\", input_data, \"--output\", output],\n",
+        "                         runconfig_pipeline_params=None, \n",
+        "                         inputs=[input_data], \n",
+        "                         outputs=[output], \n",
+        "                         compute_target=cpu_cluster)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Build and Submit the Experiment"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.pipeline.core import Pipeline\n",
+        "from azureml.core import Experiment\n",
+        "pipeline = Pipeline(workspace=ws, steps=[est_step])\n",
+        "pipeline_run = Experiment(ws, 'Estimator_sample').submit(pipeline)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## View Run Details"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.widgets import RunDetails\n",
+        "RunDetails(pipeline_run).show()"
+      ]
+    }
+  ],
+  "metadata": {
+    "authors": [
+      {
+        "name": "sanpil"
+      }
+    ],
+    "kernelspec": {
+      "display_name": "Python 3.6",
+      "language": "python",
+      "name": "python36"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.6.7"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 2
+}

how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-use-databricks-as-compute-target.ipynb

@@ -168,7 +168,7 @@
       "metadata": {},
       "source": [
         "## Data Connections with Inputs and Outputs\n",
-        "The DatabricksStep supports Azure Bloband ADLS for inputs and outputs. You also will need to define a [Secrets](https://docs.azuredatabricks.net/user-guide/secrets/index.html) scope to enable authentication to external data sources such as Blob and ADLS from Databricks.\n",
+        "The DatabricksStep supports Azure Blob and ADLS for inputs and outputs. You also will need to define a [Secrets](https://docs.azuredatabricks.net/user-guide/secrets/index.html) scope to enable authentication to external data sources such as Blob and ADLS from Databricks.\n",
         "\n",
         "- Databricks documentation on [Azure Blob](https://docs.azuredatabricks.net/spark/latest/data-sources/azure/azure-storage.html)\n",
         "- Databricks documentation on [ADLS](https://docs.databricks.com/spark/latest/data-sources/azure/azure-datalake.html)\n",

how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-with-automated-machine-learning-step.ipynb

@@ -0,0 +1,542 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Copyright (c) Microsoft Corporation. All rights reserved.  \n",
+        "Licensed under the MIT License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# Azure Machine Learning Pipeline with AutoMLStep\n",
+        "This notebook demonstrates the use of AutoMLStep in Azure Machine Learning Pipeline."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Introduction\n",
+        "In this example we use the scikit-learn's [digit dataset](http://scikit-learn.org/stable/datasets/index.html#optical-recognition-of-handwritten-digits-dataset) to showcase how you can use AutoML for a simple classification problem.\n",
+        "\n",
+        "Make sure you have executed the [configuration](../../../configuration.ipynb) before running this notebook.\n",
+        "\n",
+        "In this notebook you would see\n",
+        "1. Create an `Experiment` in an existing `Workspace`.\n",
+        "2. Create or Attach existing AmlCompute to a workspace.\n",
+        "3. Configure AutoML using `AutoMLConfig`.\n",
+        "4. Use AutoMLStep\n",
+        "5. Train the model using AmlCompute\n",
+        "6. Explore the results.\n",
+        "7. Test the best fitted model.\n",
+        "\n",
+        "In addition this notebook showcases the following features\n",
+        "- **Parallel** executions for iterations\n",
+        "- **Asynchronous** tracking of progress\n",
+        "- Retrieving models for any iteration or logged metric\n",
+        "- Specifying AutoML settings as `**kwargs`"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Azure Machine Learning and Pipeline SDK-specific imports"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import logging\n",
+        "import os\n",
+        "import csv\n",
+        "\n",
+        "from matplotlib import pyplot as plt\n",
+        "import numpy as np\n",
+        "import pandas as pd\n",
+        "from sklearn import datasets\n",
+        "\n",
+        "import azureml.core\n",
+        "from azureml.core.experiment import Experiment\n",
+        "from azureml.core.workspace import Workspace\n",
+        "from azureml.train.automl import AutoMLConfig\n",
+        "from azureml.core.compute import AmlCompute\n",
+        "from azureml.core.compute import ComputeTarget\n",
+        "from azureml.core.runconfig import RunConfiguration\n",
+        "from azureml.core.conda_dependencies import CondaDependencies\n",
+        "\n",
+        "from azureml.train.automl import AutoMLStep\n",
+        "\n",
+        "# Check core SDK version number\n",
+        "print(\"SDK version:\", azureml.core.VERSION)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Initialize Workspace\n",
+        "Initialize a workspace object from persisted configuration. Make sure the config file is present at .\\config.json"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "ws = Workspace.from_config()\n",
+        "print(ws.name, ws.resource_group, ws.location, ws.subscription_id, sep = '\\n')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Create an Azure ML experiment\n",
+        "Let's create an experiment named \"automl-classification\" and a folder to hold the training scripts. The script runs will be recorded under the experiment in Azure.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Choose a name for the run history container in the workspace.\n",
+        "experiment_name = 'automlstep-classification'\n",
+        "project_folder = './project'\n",
+        "\n",
+        "experiment = Experiment(ws, experiment_name)\n",
+        "experiment"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Create or Attach existing AmlCompute\n",
+        "You will need to create a [compute target](https://docs.microsoft.com/azure/machine-learning/service/concept-azure-machine-learning-architecture#compute-target) for your AutoML run. In this tutorial, you create `AmlCompute` as your training compute resource.\n",
+        "\n",
+        "**Creation of AmlCompute takes approximately 5 minutes.** If the AmlCompute with that name is already in your workspace this code will skip the creation process."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Choose a name for your cluster.\n",
+        "amlcompute_cluster_name = \"cpucluster\"\n",
+        "\n",
+        "found = False\n",
+        "# Check if this compute target already exists in the workspace.\n",
+        "cts = ws.compute_targets\n",
+        "if amlcompute_cluster_name in cts and cts[amlcompute_cluster_name].type == 'AmlCompute':\n",
+        "    found = True\n",
+        "    print('Found existing compute target.')\n",
+        "    compute_target = cts[amlcompute_cluster_name]\n",
+        "    \n",
+        "if not found:\n",
+        "    print('Creating a new compute target...')\n",
+        "    provisioning_config = AmlCompute.provisioning_configuration(vm_size = \"STANDARD_D2_V2\", # for GPU, use \"STANDARD_NC6\"\n",
+        "                                                                #vm_priority = 'lowpriority', # optional\n",
+        "                                                                max_nodes = 4)\n",
+        "\n",
+        "    # Create the cluster.\n",
+        "    compute_target = ComputeTarget.create(ws, amlcompute_cluster_name, provisioning_config)\n",
+        "    \n",
+        "    # Can poll for a minimum number of nodes and for a specific timeout.\n",
+        "    # If no min_node_count is provided, it will use the scale settings for the cluster.\n",
+        "    compute_target.wait_for_completion(show_output = True, min_node_count = 1, timeout_in_minutes = 10)\n",
+        "    \n",
+        "     # For a more detailed view of current AmlCompute status, use get_status()."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Prepare and Point to Data\n",
+        "For remote executions, you need to make the data accessible from the remote compute.\n",
+        "This can be done by uploading the data to DataStore.\n",
+        "In this example, we upload scikit-learn's [load_digits](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_digits.html) data."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "data_train = datasets.load_digits()\n",
+        "\n",
+        "if not os.path.isdir('data'):\n",
+        "    os.mkdir('data')\n",
+        "    \n",
+        "if not os.path.exists(project_folder):\n",
+        "    os.makedirs(project_folder)\n",
+        "    \n",
+        "pd.DataFrame(data_train.data).to_csv(\"data/X_train.tsv\", index=False, header=False, quoting=csv.QUOTE_ALL, sep=\"\\t\")\n",
+        "pd.DataFrame(data_train.target).to_csv(\"data/y_train.tsv\", index=False, header=False, sep=\"\\t\")\n",
+        "\n",
+        "ds = ws.get_default_datastore()\n",
+        "ds.upload(src_dir='./data', target_path='bai_data', overwrite=True, show_progress=True)\n",
+        "\n",
+        "from azureml.data.data_reference import DataReference      \n",
+        "input_data = DataReference(datastore=ds, \n",
+        "                           data_reference_name=\"input_data_reference\",\n",
+        "                           path_on_datastore='bai_data',\n",
+        "                           mode='download',\n",
+        "                           path_on_compute='/tmp/azureml_runs',\n",
+        "                           overwrite=False)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# create a new RunConfig object\n",
+        "conda_run_config = RunConfiguration(framework=\"python\")\n",
+        "\n",
+        "# Set compute target to AmlCompute\n",
+        "#conda_run_config.target = compute_target\n",
+        "\n",
+        "conda_run_config.environment.docker.enabled = True\n",
+        "conda_run_config.environment.docker.base_image = azureml.core.runconfig.DEFAULT_CPU_IMAGE\n",
+        "\n",
+        "cd = CondaDependencies.create(pip_packages=['azureml-sdk[automl]'], conda_packages=['numpy'], pin_sdk_version=False)\n",
+        "conda_run_config.environment.python.conda_dependencies = cd\n",
+        "\n",
+        "print('run config is ready')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "%%writefile $project_folder/get_data.py\n",
+        "\n",
+        "import pandas as pd\n",
+        "\n",
+        "def get_data():\n",
+        "    X_train = pd.read_csv(\"/tmp/azureml_runs/bai_data/X_train.tsv\", delimiter=\"\\t\", header=None, quotechar='\"')\n",
+        "    y_train = pd.read_csv(\"/tmp/azureml_runs/bai_data/y_train.tsv\", delimiter=\"\\t\", header=None, quotechar='\"')\n",
+        "\n",
+        "    return { \"X\" : X_train.values, \"y\" : y_train[0].values }\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Set up AutoMLConfig for Training\n",
+        "\n",
+        "You can specify `automl_settings` as `**kwargs` as well. Also note that you can use a `get_data()` function for local excutions too.\n",
+        "\n",
+        "**Note:** When using AmlCompute, you can't pass Numpy arrays directly to the fit method.\n",
+        "\n",
+        "|Property|Description|\n",
+        "|-|-|\n",
+        "|**primary_metric**|This is the metric that you want to optimize. Classification supports the following primary metrics: <br><i>accuracy</i><br><i>AUC_weighted</i><br><i>average_precision_score_weighted</i><br><i>norm_macro_recall</i><br><i>precision_score_weighted</i>|\n",
+        "|**iteration_timeout_minutes**|Time limit in minutes for each iteration.|\n",
+        "|**iterations**|Number of iterations. In each iteration AutoML trains a specific pipeline with the data.|\n",
+        "|**n_cross_validations**|Number of cross validation splits.|\n",
+        "|**max_concurrent_iterations**|Maximum number of iterations that would be executed in parallel. This should be less than the number of cores on the DSVM.|"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "automl_settings = {\n",
+        "    \"iteration_timeout_minutes\": 5,\n",
+        "    \"iterations\": 20,\n",
+        "    \"n_cross_validations\": 5,\n",
+        "    \"primary_metric\": 'AUC_weighted',\n",
+        "    \"preprocess\": False,\n",
+        "    \"max_concurrent_iterations\": 3,\n",
+        "    \"verbosity\": logging.INFO\n",
+        "}\n",
+        "automl_config = AutoMLConfig(task = 'classification',\n",
+        "                             debug_log = 'automl_errors.log',\n",
+        "                             path = project_folder,\n",
+        "                             compute_target=compute_target,\n",
+        "                             run_configuration=conda_run_config,\n",
+        "                             data_script = project_folder + \"/get_data.py\",\n",
+        "                             **automl_settings\n",
+        "                            )"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Call the `submit` method on the experiment object and pass the run configuration. For remote runs the execution is asynchronous, so you will see the iterations get populated as they complete. You can interact with the widgets and models even when the experiment is running to retrieve the best model up to that point. Once you are satisfied with the model, you can cancel a particular iteration or the whole run.\n",
+        "In this example, we specify `show_output = False` to suppress console output while the run is in progress."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Define AutoMLStep"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "automl_step = AutoMLStep(\n",
+        "    name='automl_module',\n",
+        "    experiment=experiment,\n",
+        "    automl_config=automl_config,\n",
+        "    inputs=[input_data],\n",
+        "    allow_reuse=True)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.pipeline.core import Pipeline\n",
+        "pipeline = Pipeline(\n",
+        "    description=\"pipeline_with_automlstep\",\n",
+        "    workspace=ws,    \n",
+        "    steps=[automl_step])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "pipeline_run = experiment.submit(pipeline)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.widgets import RunDetails\n",
+        "RunDetails(pipeline_run).show()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "pipeline_run.wait_for_completion()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Examine Results\n",
+        "\n",
+        "#### Loading executed runs\n",
+        "In case you need to load a previously executed run, enable the cell below and replace the `run_id` value."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.train.automl.run import AutoMLRun\n",
+        "\n",
+        "# only one step exists in this pipeline\n",
+        "run_id = None\n",
+        "step_runs = pipeline_run.get_children()\n",
+        "for run in step_runs:\n",
+        "    run_id=run._run_id\n",
+        "    \n",
+        "automl_run = AutoMLRun(experiment = experiment, run_id=run_id)\n",
+        "automl_run"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Retrieve All Child Runs\n",
+        "You can also use SDK methods to fetch all the child runs and see individual metrics that we log."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "children = list(automl_run.get_children())\n",
+        "metricslist = {}\n",
+        "for run in children:\n",
+        "    properties = run.get_properties()\n",
+        "    metrics = {k: v for k, v in run.get_metrics().items() if isinstance(v, float)}\n",
+        "    metricslist[int(properties['iteration'])] = metrics\n",
+        "\n",
+        "rundata = pd.DataFrame(metricslist).sort_index(1)\n",
+        "rundata"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Retrieve the Best Model\n",
+        "\n",
+        "Below we select the best pipeline from our iterations. The `get_output` method returns the best run and the fitted model. The Model includes the pipeline and any pre-processing.  Overloads on `get_output` allow you to retrieve the best run and fitted model for *any* logged metric or for a particular *iteration*."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "best_run, fitted_model = automl_run.get_output()\n",
+        "print(best_run)\n",
+        "print(fitted_model)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Best Model Based on Any Other Metric\n",
+        "Show the run and the model which has the smallest `log_loss` value:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "lookup_metric = \"log_loss\"\n",
+        "best_run, fitted_model = automl_run.get_output(metric = lookup_metric)\n",
+        "print(best_run)\n",
+        "print(fitted_model)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Model from a Specific Iteration\n",
+        "Show the run and the model from the third iteration:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "iteration = 3\n",
+        "third_run, third_model = automl_run.get_output(iteration=iteration)\n",
+        "print(third_run)\n",
+        "print(third_model)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Test the Model\n",
+        "\n",
+        "### Load Test Data"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "digits = datasets.load_digits()\n",
+        "X_test = digits.data[:10, :]\n",
+        "y_test = digits.target[:10]\n",
+        "images = digits.images[:10]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Testing Our Best Fitted Model"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Randomly select digits and test.\n",
+        "for index in np.random.choice(len(y_test), 3, replace = False):\n",
+        "    print(index)\n",
+        "    predicted = fitted_model.predict(X_test[index:index + 1])[0]\n",
+        "    label = y_test[index]\n",
+        "    title = \"Label value = %d  Predicted value = %d \" % (label, predicted)\n",
+        "    fig = plt.figure(1, figsize=(3,3))\n",
+        "    ax1 = fig.add_axes((0,0,.8,.8))\n",
+        "    ax1.set_title(title)\n",
+        "    plt.imshow(images[index], cmap = plt.cm.gray_r, interpolation = 'nearest')\n",
+        "    plt.show()"
+      ]
+    }
+  ],
+  "metadata": {
+    "authors": [
+      {
+        "name": "sanpil"
+      }
+    ],
+    "kernelspec": {
+      "display_name": "Python 3.6",
+      "language": "python",
+      "name": "python36"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.6.7"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 2
+}

how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/dummy_train.py

@@ -0,0 +1,30 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+import argparse
+import os
+
+print("*********************************************************")
+print("Hello Azure ML!")
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--datadir', type=str, help="data directory")
+parser.add_argument('--output', type=str, help="output")
+args = parser.parse_args()
+
+print("Argument 1: %s" % args.datadir)
+print("Argument 2: %s" % args.output)
+
+if not (args.output is None):
+    os.makedirs(args.output, exist_ok=True)
+    print("%s created" % args.output)
+
+try:
+    from azureml.core import Run
+    run = Run.get_context()
+    print("Log Fibonacci numbers.")
+    run.log_list('Fibonacci numbers', [0, 1, 1, 2, 3, 5, 8, 13, 21, 34])
+    run.complete()
+except:
+    print("Warning: you need to install Azure ML SDK in order to log metrics.")
+
+print("*********************************************************")

how-to-use-azureml/training-with-deep-learning/distributed-tensorflow-with-horovod/distributed-tensorflow-with-horovod.ipynb

@@ -285,7 +285,9 @@
       "metadata": {},
       "source": [
         "### Create a TensorFlow estimator\n",
-        "The AML SDK's TensorFlow estimator enables you to easily submit TensorFlow training jobs for both single-node and distributed runs. For more information on the TensorFlow estimator, refer [here](https://docs.microsoft.com/azure/machine-learning/service/how-to-train-tensorflow)."
+        "The AML SDK's TensorFlow estimator enables you to easily submit TensorFlow training jobs for both single-node and distributed runs. For more information on the TensorFlow estimator, refer [here](https://docs.microsoft.com/azure/machine-learning/service/how-to-train-tensorflow).\n",
+        "\n",
+        "The TensorFlow estimator also takes a `framework_version` parameter -- if no version is provided, the estimator will default to the latest version supported by AzureML. Use `TensorFlow.get_supported_versions()` to get a list of all versions supported by your current SDK version or see the [SDK documentation](https://docs.microsoft.com/en-us/python/api/azureml-train-core/azureml.train.dnn?view=azure-ml-py) for the versions supported in the most current release."
       ]
     },
     {
@@ -307,7 +309,8 @@
         "                      node_count=2,\n",
         "                      process_count_per_node=1,\n",
         "                      distributed_backend='mpi',\n",
-        "                      use_gpu=True)"
+        "                      use_gpu=True, \n",
+        "                      framework_version='1.12')"
       ]
     },
     {

how-to-use-azureml/training-with-deep-learning/how-to-use-estimator/how-to-use-estimator.ipynb

@@ -291,7 +291,7 @@
       "outputs": [],
       "source": [
         "# use a custom Docker image\n",
-        "from azureml.core.runconfig import ContainerRegistry\n",
+        "from azureml.core.container_registry import ContainerRegistry\n",
         "\n",
         "# this is an image available in Docker Hub\n",
         "image_name = 'continuumio/miniconda3'\n",
@@ -309,7 +309,8 @@
         "est = Estimator(source_directory='.', compute_target='local', \n",
         "                entry_script='dummy_train.py',\n",
         "                custom_docker_image=image_name,\n",
-        "                image_registry_details=image_registry_details,\n",
+        "                # uncomment below line to use your private ACR\n",
+        "                #image_registry_details=image_registry_details,\n",
         "                user_managed=user_managed_dependencies\n",
         "                )\n",
         "\n",
@@ -336,7 +337,7 @@
   "metadata": {
     "authors": [
       {
-        "name": "minxia"
+        "name": "maxluk"
       }
     ],
     "kernelspec": {
@@ -356,7 +357,7 @@
       "pygments_lexer": "ipython3",
       "version": "3.6.8"
     },
-    "msauthor": "haining"
+    "msauthor": "minxia"
   },
   "nbformat": 4,
   "nbformat_minor": 2

how-to-use-azureml/training-with-deep-learning/train-hyperparameter-tune-deploy-with-keras/train-hyperparameter-tune-deploy-with-keras.ipynb

@@ -396,7 +396,7 @@
         "est = TensorFlow(source_directory=script_folder,\n",
         "                 script_params=script_params,\n",
         "                 compute_target=compute_target, \n",
-        "                 conda_packages=['keras', 'matplotlib'],\n",
+        "                 pip_packages=['keras', 'matplotlib'],\n",
         "                 entry_script='keras_mnist.py', \n",
         "                 use_gpu=True)"
       ]
@@ -792,7 +792,7 @@
       "outputs": [],
       "source": [
         "best_run = hdr.get_best_run_by_primary_metric()\n",
-        "print(best_run.get_details()['runDefinition']['Arguments'])"
+        "print(best_run.get_details()['runDefinition']['arguments'])"
       ]
     },
     {
@@ -1144,7 +1144,7 @@
   "metadata": {
     "authors": [
       {
-        "name": "haining"
+        "name": "maxluk"
       }
     ],
     "kernelspec": {
@@ -1164,7 +1164,7 @@
       "pygments_lexer": "ipython3",
       "version": "3.6.7"
     },
-    "msauthor": "haining"
+    "msauthor": "maxluk"
   },
   "nbformat": 4,
   "nbformat_minor": 2

how-to-use-azureml/training-with-deep-learning/train-hyperparameter-tune-deploy-with-tensorflow/train-hyperparameter-tune-deploy-with-tensorflow.ipynb

@@ -396,7 +396,10 @@
       "source": [
         "## Create TensorFlow estimator\n",
         "Next, we construct an `azureml.train.dnn.TensorFlow` estimator object, use the Batch AI cluster as compute target, and pass the mount-point of the datastore to the training code as a parameter.\n",
-        "The TensorFlow estimator is providing a simple way of launching a TensorFlow training job on a compute target. It will automatically provide a docker image that has TensorFlow installed -- if additional pip or conda packages are required, their names can be passed in via the `pip_packages` and `conda_packages` arguments and they will be included in the resulting docker."
+        "\n",
+        "The TensorFlow estimator is providing a simple way of launching a TensorFlow training job on a compute target. It will automatically provide a docker image that has TensorFlow installed -- if additional pip or conda packages are required, their names can be passed in via the `pip_packages` and `conda_packages` arguments and they will be included in the resulting docker.\n",
+        "\n",
+        "The TensorFlow estimator also takes a `framework_version` parameter -- if no version is provided, the estimator will default to the latest version supported by AzureML. Use `TensorFlow.get_supported_versions()` to get a list of all versions supported by your current SDK version or see the [SDK documentation](https://docs.microsoft.com/en-us/python/api/azureml-train-core/azureml.train.dnn?view=azure-ml-py) for the versions supported in the most current release."
       ]
     },
     {
@@ -419,7 +422,8 @@
         "                 script_params=script_params,\n",
         "                 compute_target=compute_target,\n",
         "                 entry_script='tf_mnist.py', \n",
-        "                 use_gpu=True)"
+        "                 use_gpu=True, \n",
+        "                 framework_version='1.12')"
       ]
     },
     {
@@ -1158,7 +1162,7 @@
       "name": "python",
       "nbconvert_exporter": "python",
       "pygments_lexer": "ipython3",
-      "version": "3.6.8"
+      "version": "3.6.6"
     },
     "msauthor": "minxia"
   },

how-to-use-azureml/training/logging-api/logging-api.ipynb

@@ -54,7 +54,7 @@
         "\n",
         "The experiment's Run History report page automatically creates a report that can be customized to show the KPI's, charts, and column sets that are interesting to you. \n",
         "\n",
-    "| ![Run Details](./img/run_details.PNG) | ![Run History](./img/run_history.PNG) |\n",
+        "| ![Run Details](./img/run_details.PNG) | ![Run History](./img/run_history.png) |\n",
         "|:--:|:--:|\n",
         "| *Run Details* | *Run History* |\n",
         "\n",
@@ -93,7 +93,7 @@
         "\n",
         "# Check core SDK version number\n",
         "\n",
-    "print(\"This notebook was created using SDK version AZUREML-SDK-VERSION, you are currently running version\", azureml.core.VERSION)"
+        "print(\"This notebook was created using SDK version 1.0.23, you are currently running version\", azureml.core.VERSION)"
       ]
     },
     {

how-to-use-azureml/training/train-on-local/train-on-local.ipynb

@@ -488,7 +488,7 @@
   "metadata": {
     "authors": [
       {
-        "name": "haining"
+        "name": "roastala"
       }
     ],
     "kernelspec": {

how-to-use-azureml/training/train-on-remote-vm/train-on-remote-vm.ipynb

@@ -615,7 +615,7 @@
   "metadata": {
     "authors": [
       {
-        "name": "haining"
+        "name": "roastala"
       }
     ],
     "kernelspec": {

how-to-use-azureml/training/train-within-notebook/train-within-notebook.ipynb

@@ -673,7 +673,7 @@
   "metadata": {
     "authors": [
       {
-        "name": "haining"
+        "name": "roastala"
       }
     ],
     "kernelspec": {

tutorials/img-classification-part1-training.ipynb

@@ -661,7 +661,7 @@
   "metadata": {
     "authors": [
       {
-        "name": "haining"
+        "name": "roastala"
       }
     ],
     "kernelspec": {
@@ -681,7 +681,7 @@
       "pygments_lexer": "ipython3",
       "version": "3.6.8"
     },
-    "msauthor": "haining"
+    "msauthor": "roastala"
   },
   "nbformat": 4,
   "nbformat_minor": 2

tutorials/img-classification-part2-deploy.ipynb

@@ -592,7 +592,7 @@
   "metadata": {
     "authors": [
       {
-        "name": "haining"
+        "name": "roastala"
       }
     ],
     "kernelspec": {

tutorials/regression-part1-data-prep.ipynb

@@ -60,7 +60,7 @@
         "Use the following to install necessary packages if you don't already have them.\n",
         "\n",
         "```shell\n",
-        "pip install azureml-dataprep\n",
+        "pip install \"azureml-dataprep>=1.1.0,<1.2.0\"\n",
         "```\n",
         "\n",
         "Import the SDK."
@@ -557,8 +557,7 @@
         "import os\n",
         "file_path = os.path.join(os.getcwd(), \"dflows.dprep\")\n",
         "\n",
-        "package = dprep.Package([final_df])\n",
-        "package.save(file_path)"
+        "final_df.save(file_path)"
       ]
     },
     {

tutorials/regression-part2-automated-ml.ipynb

@@ -137,8 +137,7 @@
         "\n",
         "file_path = os.path.join(os.getcwd(), \"dflows.dprep\")\n",
         "\n",
-        "package_saved = dprep.Package.open(file_path)\n",
-        "dflow_prepared = package_saved.dataflows[0]\n",
+        "dflow_prepared = dprep.Dataflow.open(file_path)\n",
         "dflow_prepared.get_profile()"
       ]
     },