Merge pull request #247 from rastala/master

version 1.0.18

Dockerfiles/1.0.17/Dockerfile

@@ -0,0 +1,29 @@
+FROM continuumio/miniconda:4.5.11
+
+# install git
+RUN apt-get update && apt-get upgrade -y && apt-get install -y git
+
+# create a new conda environment named azureml
+RUN conda create -n azureml -y -q Python=3.6
+
+# install additional packages used by sample notebooks. this is optional
+RUN ["/bin/bash", "-c", "source activate azureml && conda install -y tqdm cython matplotlib scikit-learn"]
+
+# install azurmel-sdk components
+RUN ["/bin/bash", "-c", "source activate azureml && pip install azureml-sdk[notebooks]==1.0.17"]
+
+# clone Azure ML GitHub sample notebooks
+RUN cd /home && git clone -b "azureml-sdk-1.0.17" --single-branch https://github.com/Azure/MachineLearningNotebooks.git
+
+# generate jupyter configuration file
+RUN ["/bin/bash", "-c", "source activate azureml && mkdir ~/.jupyter && cd ~/.jupyter && jupyter notebook --generate-config"]
+
+# set an emtpy token for Jupyter to remove authentication. 
+# this is NOT recommended for production environment
+RUN echo "c.NotebookApp.token = ''" >> ~/.jupyter/jupyter_notebook_config.py
+
+# open up port 8887 on the container
+EXPOSE 8887
+
+# start Jupyter notebook server on port 8887 when the container starts
+CMD /bin/bash -c "cd /home/MachineLearningNotebooks && source activate azureml && jupyter notebook --port 8887 --no-browser --ip 0.0.0.0 --allow-root"

configuration.ipynb

@@ -96,7 +96,7 @@
       "source": [
         "import azureml.core\n",
         "\n",
-        "print(\"This notebook was created using version 1.0.17 of the Azure ML SDK\")\n",
+        "print(\"This notebook was created using version 1.0.18 of the Azure ML SDK\")\n",
         "print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
       ]
     },

contrib/RAPIDS/azure-ml-with-nvidia-rapids.ipynb

@@ -20,7 +20,7 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "The [RAPIDS](https://www.developer.nvidia.com/rapids) suite of software libraries from NVIDIA enables the execution of end-to-end data science and analytics pipelines entirely on GPUs. In many machine learning projects, a significant portion of the model training time is spent in setting up the data; this stage of the process is known as Extraction, Transformation and Loading, or ETL. By using the DataFrame API for ETL and GPU-capable ML algorithms in RAPIDS, data preparation and training models can be done in GPU-accelerated end-to-end pipelines without incurring serialization costs between the pipeline stages. This notebook demonstrates how to use NVIDIA RAPIDS to prepare data and train model in Azure.\n",
+        "The [RAPIDS](https://www.developer.nvidia.com/rapids) suite of software libraries from NVIDIA enables the execution of end-to-end data science and analytics pipelines entirely on GPUs. In many machine learning projects, a significant portion of the model training time is spent in setting up the data; this stage of the process is known as Extraction, Transformation and Loading, or ETL. By using the DataFrame API for ETL\u00c2\u00a0and GPU-capable ML algorithms in RAPIDS, data preparation and training models can be done in GPU-accelerated end-to-end pipelines without incurring serialization costs between the pipeline stages. This notebook demonstrates how to use NVIDIA RAPIDS to prepare data and train model\u00c2\u00a0in Azure.\n",
         " \n",
         "In this notebook, we will do the following:\n",
         " \n",

googleade5d7141b3f2910.html

@@ -1 +0,0 @@
-google-site-verification: googleade5d7141b3f2910.html

how-to-use-azureml/automated-machine-learning/README.md

@@ -119,7 +119,7 @@ bash automl_setup_linux.sh
     - Retrieving models for any iteration or logged metric
     - Specify automl settings as kwargs
 
-- [auto-ml-remote-batchai.ipynb](remote-batchai/auto-ml-remote-batchai.ipynb)
+- [auto-ml-remote-amlcompute.ipynb](remote-batchai/auto-ml-remote-amlcompute.ipynb)
     - Dataset: scikit learn's [digit dataset](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_digits.html#sklearn.datasets.load_digits)
     - Example of using automated ML for classification using remote AmlCompute for training
     - Parallel execution of iterations

how-to-use-azureml/automated-machine-learning/automl_env.yml

@@ -16,6 +16,7 @@ dependencies:
 
 - pip:
   # Required packages for AzureML execution, history, and data preparation.
-  - azureml-sdk[automl,notebooks,explain]
+  - azureml-sdk[automl,explain]
+  - azureml-widgets
   - pandas_ml
   

how-to-use-azureml/automated-machine-learning/automl_env_mac.yml

@@ -16,7 +16,8 @@ dependencies:
 
 - pip:
   # Required packages for AzureML execution, history, and data preparation.
-  - azureml-sdk[automl,notebooks,explain]
+  - azureml-sdk[automl,explain]
+  - azureml-widgets
   - pandas_ml
   
 

how-to-use-azureml/automated-machine-learning/dataprep-remote-execution/auto-ml-dataprep-remote-execution.ipynb

@@ -195,7 +195,7 @@
         "    dsvm_compute = DsvmCompute.create(ws, name = dsvm_name, provisioning_configuration = dsvm_config)\n",
         "    dsvm_compute.wait_for_completion(show_output = True)\n",
         "    print(\"Waiting one minute for ssh to be accessible\")\n",
-        "    time.sleep(60) # Wait for ssh to be accessible"
+        "    time.sleep(90) # Wait for ssh to be accessible"
       ]
     },
     {

how-to-use-azureml/automated-machine-learning/remote-amlcompute/auto-ml-remote-amlcompute.ipynb

@@ -0,0 +1,555 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
+        "\n",
+        "Licensed under the MIT License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# Automated Machine Learning\n",
+        "_**Remote Execution using AmlCompute**_\n",
+        "\n",
+        "## Contents\n",
+        "1. [Introduction](#Introduction)\n",
+        "1. [Setup](#Setup)\n",
+        "1. [Data](#Data)\n",
+        "1. [Train](#Train)\n",
+        "1. [Results](#Results)\n",
+        "1. [Test](#Test)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Introduction\n",
+        "In this example we use the scikit-learn's [digit dataset](http://scikit-learn.org/stable/datasets/index.html#optical-recognition-of-handwritten-digits-dataset) to showcase how you can use AutoML for a simple classification problem.\n",
+        "\n",
+        "Make sure you have executed the [configuration](../../../configuration.ipynb) before running this notebook.\n",
+        "\n",
+        "In this notebook you would see\n",
+        "1. Create an `Experiment` in an existing `Workspace`.\n",
+        "2. Create or Attach existing AmlCompute to a workspace.\n",
+        "3. Configure AutoML using `AutoMLConfig`.\n",
+        "4. Train the model using AmlCompute\n",
+        "5. Explore the results.\n",
+        "6. Test the best fitted model.\n",
+        "\n",
+        "In addition this notebook showcases the following features\n",
+        "- **Parallel** executions for iterations\n",
+        "- **Asynchronous** tracking of progress\n",
+        "- **Cancellation** of individual iterations or the entire run\n",
+        "- Retrieving models for any iteration or logged metric\n",
+        "- Specifying AutoML settings as `**kwargs`"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Setup\n",
+        "\n",
+        "As part of the setup you have already created an Azure ML `Workspace` object. For AutoML you will need to create an `Experiment` object, which is a named object in a `Workspace` used to run experiments."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import logging\n",
+        "import os\n",
+        "import csv\n",
+        "\n",
+        "from matplotlib import pyplot as plt\n",
+        "import numpy as np\n",
+        "import pandas as pd\n",
+        "from sklearn import datasets\n",
+        "\n",
+        "import azureml.core\n",
+        "from azureml.core.experiment import Experiment\n",
+        "from azureml.core.workspace import Workspace\n",
+        "from azureml.train.automl import AutoMLConfig"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "ws = Workspace.from_config()\n",
+        "\n",
+        "# Choose a name for the run history container in the workspace.\n",
+        "experiment_name = 'automl-remote-amlcompute'\n",
+        "project_folder = './project'\n",
+        "\n",
+        "experiment = Experiment(ws, experiment_name)\n",
+        "\n",
+        "output = {}\n",
+        "output['SDK version'] = azureml.core.VERSION\n",
+        "output['Subscription ID'] = ws.subscription_id\n",
+        "output['Workspace Name'] = ws.name\n",
+        "output['Resource Group'] = ws.resource_group\n",
+        "output['Location'] = ws.location\n",
+        "output['Project Directory'] = project_folder\n",
+        "output['Experiment Name'] = experiment.name\n",
+        "pd.set_option('display.max_colwidth', -1)\n",
+        "outputDf = pd.DataFrame(data = output, index = [''])\n",
+        "outputDf.T"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Create or Attach existing AmlCompute\n",
+        "You will need to create a [compute target](https://docs.microsoft.com/azure/machine-learning/service/concept-azure-machine-learning-architecture#compute-target) for your AutoML run. In this tutorial, you create `AmlCompute` as your training compute resource.\n",
+        "\n",
+        "**Creation of AmlCompute takes approximately 5 minutes.** If the AmlCompute with that name is already in your workspace this code will skip the creation process.\n",
+        "\n",
+        "As with other Azure services, there are limits on certain resources (e.g. AmlCompute) associated with the Azure Machine Learning service. Please read [this article](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-manage-quotas) on the default limits and how to request more quota."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.compute import AmlCompute\n",
+        "from azureml.core.compute import ComputeTarget\n",
+        "\n",
+        "# Choose a name for your cluster.\n",
+        "amlcompute_cluster_name = \"automlcl\"\n",
+        "\n",
+        "found = False\n",
+        "# Check if this compute target already exists in the workspace.\n",
+        "cts = ws.compute_targets\n",
+        "if amlcompute_cluster_name in cts and cts[amlcompute_cluster_name].type == 'AmlCompute':\n",
+        "    found = True\n",
+        "    print('Found existing compute target.')\n",
+        "    compute_target = cts[amlcompute_cluster_name]\n",
+        "    \n",
+        "if not found:\n",
+        "    print('Creating a new compute target...')\n",
+        "    provisioning_config = AmlCompute.provisioning_configuration(vm_size = \"STANDARD_D2_V2\", # for GPU, use \"STANDARD_NC6\"\n",
+        "                                                                #vm_priority = 'lowpriority', # optional\n",
+        "                                                                max_nodes = 6)\n",
+        "\n",
+        "    # Create the cluster.\n",
+        "    compute_target = ComputeTarget.create(ws, amlcompute_cluster_name, provisioning_config)\n",
+        "    \n",
+        "    # Can poll for a minimum number of nodes and for a specific timeout.\n",
+        "    # If no min_node_count is provided, it will use the scale settings for the cluster.\n",
+        "    compute_target.wait_for_completion(show_output = True, min_node_count = None, timeout_in_minutes = 20)\n",
+        "    \n",
+        "     # For a more detailed view of current AmlCompute status, use get_status()."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": []
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Data\n",
+        "For remote executions, you need to make the data accessible from the remote compute.\n",
+        "This can be done by uploading the data to DataStore.\n",
+        "In this example, we upload scikit-learn's [load_digits](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_digits.html) data."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "data_train = datasets.load_digits()\n",
+        "\n",
+        "if not os.path.isdir('data'):\n",
+        "    os.mkdir('data')\n",
+        "    \n",
+        "if not os.path.exists(project_folder):\n",
+        "    os.makedirs(project_folder)\n",
+        "    \n",
+        "pd.DataFrame(data_train.data).to_csv(\"data/X_train.tsv\", index=False, header=False, quoting=csv.QUOTE_ALL, sep=\"\\t\")\n",
+        "pd.DataFrame(data_train.target).to_csv(\"data/y_train.tsv\", index=False, header=False, sep=\"\\t\")\n",
+        "\n",
+        "ds = ws.get_default_datastore()\n",
+        "ds.upload(src_dir='./data', target_path='bai_data', overwrite=True, show_progress=True)\n",
+        "\n",
+        "from azureml.core.runconfig import DataReferenceConfiguration\n",
+        "dr = DataReferenceConfiguration(datastore_name=ds.name, \n",
+        "                   path_on_datastore='bai_data', \n",
+        "                   path_on_compute='/tmp/azureml_runs',\n",
+        "                   mode='download', # download files from datastore to compute target\n",
+        "                   overwrite=False)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.runconfig import RunConfiguration\n",
+        "from azureml.core.conda_dependencies import CondaDependencies\n",
+        "\n",
+        "# create a new RunConfig object\n",
+        "conda_run_config = RunConfiguration(framework=\"python\")\n",
+        "\n",
+        "# Set compute target to AmlCompute\n",
+        "conda_run_config.target = compute_target\n",
+        "conda_run_config.environment.docker.enabled = True\n",
+        "conda_run_config.environment.docker.base_image = azureml.core.runconfig.DEFAULT_CPU_IMAGE\n",
+        "\n",
+        "# set the data reference of the run coonfiguration\n",
+        "conda_run_config.data_references = {ds.name: dr}\n",
+        "\n",
+        "cd = CondaDependencies.create(pip_packages=['azureml-sdk[automl]'], conda_packages=['numpy'])\n",
+        "conda_run_config.environment.python.conda_dependencies = cd"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "%%writefile $project_folder/get_data.py\n",
+        "\n",
+        "import pandas as pd\n",
+        "\n",
+        "def get_data():\n",
+        "    X_train = pd.read_csv(\"/tmp/azureml_runs/bai_data/X_train.tsv\", delimiter=\"\\t\", header=None, quotechar='\"')\n",
+        "    y_train = pd.read_csv(\"/tmp/azureml_runs/bai_data/y_train.tsv\", delimiter=\"\\t\", header=None, quotechar='\"')\n",
+        "\n",
+        "    return { \"X\" : X_train.values, \"y\" : y_train[0].values }\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Train\n",
+        "\n",
+        "You can specify `automl_settings` as `**kwargs` as well. Also note that you can use a `get_data()` function for local excutions too.\n",
+        "\n",
+        "**Note:** When using AmlCompute, you can't pass Numpy arrays directly to the fit method.\n",
+        "\n",
+        "|Property|Description|\n",
+        "|-|-|\n",
+        "|**primary_metric**|This is the metric that you want to optimize. Classification supports the following primary metrics: <br><i>accuracy</i><br><i>AUC_weighted</i><br><i>average_precision_score_weighted</i><br><i>norm_macro_recall</i><br><i>precision_score_weighted</i>|\n",
+        "|**iteration_timeout_minutes**|Time limit in minutes for each iteration.|\n",
+        "|**iterations**|Number of iterations. In each iteration AutoML trains a specific pipeline with the data.|\n",
+        "|**n_cross_validations**|Number of cross validation splits.|\n",
+        "|**max_concurrent_iterations**|Maximum number of iterations that would be executed in parallel. This should be less than the number of cores on the DSVM.|"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "automl_settings = {\n",
+        "    \"iteration_timeout_minutes\": 2,\n",
+        "    \"iterations\": 20,\n",
+        "    \"n_cross_validations\": 5,\n",
+        "    \"primary_metric\": 'AUC_weighted',\n",
+        "    \"preprocess\": False,\n",
+        "    \"max_concurrent_iterations\": 5,\n",
+        "    \"verbosity\": logging.INFO\n",
+        "}\n",
+        "\n",
+        "automl_config = AutoMLConfig(task = 'classification',\n",
+        "                             debug_log = 'automl_errors.log',\n",
+        "                             path = project_folder,\n",
+        "                             run_configuration=conda_run_config,\n",
+        "                             data_script = project_folder + \"/get_data.py\",\n",
+        "                             **automl_settings\n",
+        "                            )\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Call the `submit` method on the experiment object and pass the run configuration. For remote runs the execution is asynchronous, so you will see the iterations get populated as they complete. You can interact with the widgets and models even when the experiment is running to retrieve the best model up to that point. Once you are satisfied with the model, you can cancel a particular iteration or the whole run.\n",
+        "In this example, we specify `show_output = False` to suppress console output while the run is in progress."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "remote_run = experiment.submit(automl_config, show_output = False)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "remote_run"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Results\n",
+        "\n",
+        "#### Loading executed runs\n",
+        "In case you need to load a previously executed run, enable the cell below and replace the `run_id` value."
+      ]
+    },
+    {
+      "cell_type": "raw",
+      "metadata": {},
+      "source": [
+        "remote_run = AutoMLRun(experiment = experiment, run_id = 'AutoML_5db13491-c92a-4f1d-b622-8ab8d973a058')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Widget for Monitoring Runs\n",
+        "\n",
+        "The widget will first report a \"loading\" status while running the first iteration. After completing the first iteration, an auto-updating graph and table will be shown. The widget will refresh once per minute, so you should see the graph update as child runs complete.\n",
+        "\n",
+        "You can click on a pipeline to see run properties and output logs.  Logs are also available on the DSVM under `/tmp/azureml_run/{iterationid}/azureml-logs`\n",
+        "\n",
+        "**Note:** The widget displays a link at the bottom. Use this link to open a web interface to explore the individual run details."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "remote_run"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.widgets import RunDetails\n",
+        "RunDetails(remote_run).show() "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Wait until the run finishes.\n",
+        "remote_run.wait_for_completion(show_output = True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "\n",
+        "#### Retrieve All Child Runs\n",
+        "You can also use SDK methods to fetch all the child runs and see individual metrics that we log."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "children = list(remote_run.get_children())\n",
+        "metricslist = {}\n",
+        "for run in children:\n",
+        "    properties = run.get_properties()\n",
+        "    metrics = {k: v for k, v in run.get_metrics().items() if isinstance(v, float)}\n",
+        "    metricslist[int(properties['iteration'])] = metrics\n",
+        "\n",
+        "rundata = pd.DataFrame(metricslist).sort_index(1)\n",
+        "rundata"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Cancelling Runs\n",
+        "\n",
+        "You can cancel ongoing remote runs using the `cancel` and `cancel_iteration` functions."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Cancel the ongoing experiment and stop scheduling new iterations.\n",
+        "# remote_run.cancel()\n",
+        "\n",
+        "# Cancel iteration 1 and move onto iteration 2.\n",
+        "# remote_run.cancel_iteration(1)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Retrieve the Best Model\n",
+        "\n",
+        "Below we select the best pipeline from our iterations. The `get_output` method returns the best run and the fitted model. The Model includes the pipeline and any pre-processing.  Overloads on `get_output` allow you to retrieve the best run and fitted model for *any* logged metric or for a particular *iteration*."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "best_run, fitted_model = remote_run.get_output()\n",
+        "print(best_run)\n",
+        "print(fitted_model)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Best Model Based on Any Other Metric\n",
+        "Show the run and the model which has the smallest `log_loss` value:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "lookup_metric = \"log_loss\"\n",
+        "best_run, fitted_model = remote_run.get_output(metric = lookup_metric)\n",
+        "print(best_run)\n",
+        "print(fitted_model)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Model from a Specific Iteration\n",
+        "Show the run and the model from the third iteration:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "iteration = 3\n",
+        "third_run, third_model = remote_run.get_output(iteration=iteration)\n",
+        "print(third_run)\n",
+        "print(third_model)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Test\n",
+        "\n",
+        "#### Load Test Data"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "digits = datasets.load_digits()\n",
+        "X_test = digits.data[:10, :]\n",
+        "y_test = digits.target[:10]\n",
+        "images = digits.images[:10]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Testing Our Best Fitted Model"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Randomly select digits and test.\n",
+        "for index in np.random.choice(len(y_test), 2, replace = False):\n",
+        "    print(index)\n",
+        "    predicted = fitted_model.predict(X_test[index:index + 1])[0]\n",
+        "    label = y_test[index]\n",
+        "    title = \"Label value = %d  Predicted value = %d \" % (label, predicted)\n",
+        "    fig = plt.figure(1, figsize=(3,3))\n",
+        "    ax1 = fig.add_axes((0,0,.8,.8))\n",
+        "    ax1.set_title(title)\n",
+        "    plt.imshow(images[index], cmap = plt.cm.gray_r, interpolation = 'nearest')\n",
+        "    plt.show()"
+      ]
+    }
+  ],
+  "metadata": {
+    "authors": [
+      {
+        "name": "savitam"
+      }
+    ],
+    "kernelspec": {
+      "display_name": "Python 3.6",
+      "language": "python",
+      "name": "python36"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.6.6"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 2
+}

how-to-use-azureml/automated-machine-learning/remote-execution-with-datastore/auto-ml-remote-execution-with-datastore.ipynb

@@ -128,7 +128,7 @@
         "    dsvm_compute = DsvmCompute.create(ws, name=compute_target_name, provisioning_configuration=dsvm_config)\n",
         "    dsvm_compute.wait_for_completion(show_output=True)\n",
         "    print(\"Waiting one minute for ssh to be accessible\")\n",
-        "    time.sleep(60) # Wait for ssh to be accessible"
+        "    time.sleep(90) # Wait for ssh to be accessible"
       ]
     },
     {

how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-publish-and-run-using-rest-endpoint.ipynb

@@ -77,7 +77,7 @@
       "source": [
         "from azureml.core.compute_target import ComputeTargetException\n",
         "\n",
-        "aml_compute_target = \"aml-compute\"\n",
+        "aml_compute_target = \"cpucluster\"\n",
         "try:\n",
         "    aml_compute = AmlCompute(ws, aml_compute_target)\n",
         "    print(\"found existing compute target.\")\n",
@@ -280,7 +280,8 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "## Publish the pipeline"
+        "## Run published pipeline\n",
+        "### Publish the pipeline"
       ]
     },
     {
@@ -290,7 +291,34 @@
       "outputs": [],
       "source": [
         "published_pipeline1 = pipeline1.publish(name=\"My_New_Pipeline\", description=\"My Published Pipeline Description\")\n",
-        "print(published_pipeline1.id)"
+        "published_pipeline1"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Get published pipeline\n",
+        "\n",
+        "You can get the published pipeline using **pipeline id**.\n",
+        "\n",
+        "To get all the published pipelines for a given workspace(ws): \n",
+        "```css\n",
+        "all_pub_pipelines = PublishedPipeline.get_all(ws)\n",
+        "```"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.pipeline.core import PublishedPipeline\n",
+        "\n",
+        "pipeline_id = published_pipeline1.id # use your published pipeline id\n",
+        "published_pipeline = PublishedPipeline.get(ws, pipeline_id)\n",
+        "published_pipeline"
       ]
     },
     {
@@ -312,9 +340,9 @@
         "auth = InteractiveLoginAuthentication()\n",
         "aad_token = auth.get_authentication_header()\n",
         "\n",
-        "rest_endpoint1 = published_pipeline1.endpoint\n",
+        "rest_endpoint1 = published_pipeline.endpoint\n",
         "\n",
-        "print(rest_endpoint1)\n",
+        "print(\"You can perform HTTP POST on URL {} to trigger this pipeline\".format(rest_endpoint1))\n",
         "\n",
         "# specify the param when running the pipeline\n",
         "response = requests.post(rest_endpoint1, \n",

how-to-use-azureml/machine-learning-pipelines/pipeline-batch-scoring/pipeline-batch-scoring.ipynb

@@ -165,7 +165,7 @@
       "outputs": [],
       "source": [
         "# choose a name for your cluster\n",
-        "aml_compute_name = os.environ.get(\"AML_COMPUTE_NAME\", \"gpu-cluster\")\n",
+        "aml_compute_name = os.environ.get(\"AML_COMPUTE_NAME\", \"gpucluster\")\n",
         "cluster_min_nodes = os.environ.get(\"AML_COMPUTE_MIN_NODES\", 0)\n",
         "cluster_max_nodes = os.environ.get(\"AML_COMPUTE_MAX_NODES\", 1)\n",
         "vm_size = os.environ.get(\"AML_COMPUTE_SKU\", \"STANDARD_NC6\")\n",
@@ -466,7 +466,35 @@
         "published_pipeline = pipeline_run.publish_pipeline(\n",
         "    name=\"Inception_v3_scoring\", description=\"Batch scoring using Inception v3 model\", version=\"1.0\")\n",
         "\n",
-        "published_id = published_pipeline.id"
+        "published_pipeline"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Get published pipeline\n",
+        "\n",
+        "You can get the published pipeline using **pipeline id**.\n",
+        "\n",
+        "To get all the published pipelines for a given workspace(ws): \n",
+        "```css\n",
+        "all_pub_pipelines = PublishedPipeline.get_all(ws)\n",
+        "```"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.pipeline.core import PublishedPipeline\n",
+        "\n",
+        "pipeline_id = published_pipeline.id # use your published pipeline id\n",
+        "published_pipeline = PublishedPipeline.get(ws, pipeline_id)\n",
+        "\n",
+        "published_pipeline"
       ]
     },
     {

how-to-use-azureml/machine-learning-pipelines/pipeline-style-transfer/pipeline-style-transfer.ipynb

@@ -450,7 +450,35 @@
         "published_pipeline = pipeline_run.publish_pipeline(\n",
         "    name=\"batch score style transfer\", description=\"style transfer\", version=\"1.0\")\n",
         "\n",
-        "published_id = published_pipeline.id"
+        "published_pipeline"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Get published pipeline\n",
+        "\n",
+        "You can get the published pipeline using **pipeline id**.\n",
+        "\n",
+        "To get all the published pipelines for a given workspace(ws): \n",
+        "```css\n",
+        "all_pub_pipelines = PublishedPipeline.get_all(ws)\n",
+        "```"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.pipeline.core import PublishedPipeline\n",
+        "\n",
+        "pipeline_id = published_pipeline.id # use your published pipeline id\n",
+        "published_pipeline = PublishedPipeline.get(ws, pipeline_id)\n",
+        "\n",
+        "published_pipeline"
       ]
     },
     {

how-to-use-azureml/training/train-on-remote-vm/train-on-remote-vm.ipynb

@@ -227,7 +227,7 @@
         "                                                        private_key_file='./.ssh/id_rsa')\n",
         "    attached_dsvm_compute = ComputeTarget.attach(workspace=ws,\n",
         "                                                 name=compute_target_name,\n",
-        "                                                 attach_config=attach_config)\n",
+        "                                                 attach_configuration=attach_config)\n",
         "    attached_dsvm_compute.wait_for_completion(show_output=True)"
       ]
     },