fix drift-on-aks failure, will be ready by next release

update samples from Release-163 as a part of 1.0.79 SDK release

Dockerfiles/1.0.23/Dockerfile

@@ -0,0 +1,29 @@
+FROM continuumio/miniconda:4.5.11
+
+# install git
+RUN apt-get update && apt-get upgrade -y && apt-get install -y git
+
+# create a new conda environment named azureml
+RUN conda create -n azureml -y -q Python=3.6
+
+# install additional packages used by sample notebooks. this is optional
+RUN ["/bin/bash", "-c", "source activate azureml && conda install -y tqdm cython matplotlib scikit-learn"]
+
+# install azurmel-sdk components
+RUN ["/bin/bash", "-c", "source activate azureml && pip install azureml-sdk[notebooks]==1.0.23"]
+
+# clone Azure ML GitHub sample notebooks
+RUN cd /home && git clone -b "azureml-sdk-1.0.23" --single-branch https://github.com/Azure/MachineLearningNotebooks.git
+
+# generate jupyter configuration file
+RUN ["/bin/bash", "-c", "source activate azureml && mkdir ~/.jupyter && cd ~/.jupyter && jupyter notebook --generate-config"]
+
+# set an emtpy token for Jupyter to remove authentication. 
+# this is NOT recommended for production environment
+RUN echo "c.NotebookApp.token = ''" >> ~/.jupyter/jupyter_notebook_config.py
+
+# open up port 8887 on the container
+EXPOSE 8887
+
+# start Jupyter notebook server on port 8887 when the container starts
+CMD /bin/bash -c "cd /home/MachineLearningNotebooks && source activate azureml && jupyter notebook --port 8887 --no-browser --ip 0.0.0.0 --allow-root"

Dockerfiles/1.0.30/Dockerfile

@@ -0,0 +1,29 @@
+FROM continuumio/miniconda:4.5.11
+
+# install git
+RUN apt-get update && apt-get upgrade -y && apt-get install -y git
+
+# create a new conda environment named azureml
+RUN conda create -n azureml -y -q Python=3.6
+
+# install additional packages used by sample notebooks. this is optional
+RUN ["/bin/bash", "-c", "source activate azureml && conda install -y tqdm cython matplotlib scikit-learn"]
+
+# install azurmel-sdk components
+RUN ["/bin/bash", "-c", "source activate azureml && pip install azureml-sdk[notebooks]==1.0.30"]
+
+# clone Azure ML GitHub sample notebooks
+RUN cd /home && git clone -b "azureml-sdk-1.0.30" --single-branch https://github.com/Azure/MachineLearningNotebooks.git
+
+# generate jupyter configuration file
+RUN ["/bin/bash", "-c", "source activate azureml && mkdir ~/.jupyter && cd ~/.jupyter && jupyter notebook --generate-config"]
+
+# set an emtpy token for Jupyter to remove authentication. 
+# this is NOT recommended for production environment
+RUN echo "c.NotebookApp.token = ''" >> ~/.jupyter/jupyter_notebook_config.py
+
+# open up port 8887 on the container
+EXPOSE 8887
+
+# start Jupyter notebook server on port 8887 when the container starts
+CMD /bin/bash -c "cd /home/MachineLearningNotebooks && source activate azureml && jupyter notebook --port 8887 --no-browser --ip 0.0.0.0 --allow-root"

Dockerfiles/1.0.33/Dockerfile

@@ -0,0 +1,29 @@
+FROM continuumio/miniconda:4.5.11
+
+# install git
+RUN apt-get update && apt-get upgrade -y && apt-get install -y git
+
+# create a new conda environment named azureml
+RUN conda create -n azureml -y -q Python=3.6
+
+# install additional packages used by sample notebooks. this is optional
+RUN ["/bin/bash", "-c", "source activate azureml && conda install -y tqdm cython matplotlib scikit-learn"]
+
+# install azurmel-sdk components
+RUN ["/bin/bash", "-c", "source activate azureml && pip install azureml-sdk[notebooks]==1.0.33"]
+
+# clone Azure ML GitHub sample notebooks
+RUN cd /home && git clone -b "azureml-sdk-1.0.33" --single-branch https://github.com/Azure/MachineLearningNotebooks.git
+
+# generate jupyter configuration file
+RUN ["/bin/bash", "-c", "source activate azureml && mkdir ~/.jupyter && cd ~/.jupyter && jupyter notebook --generate-config"]
+
+# set an emtpy token for Jupyter to remove authentication. 
+# this is NOT recommended for production environment
+RUN echo "c.NotebookApp.token = ''" >> ~/.jupyter/jupyter_notebook_config.py
+
+# open up port 8887 on the container
+EXPOSE 8887
+
+# start Jupyter notebook server on port 8887 when the container starts
+CMD /bin/bash -c "cd /home/MachineLearningNotebooks && source activate azureml && jupyter notebook --port 8887 --no-browser --ip 0.0.0.0 --allow-root"

Dockerfiles/1.0.41/Dockerfile

@@ -0,0 +1,29 @@
+FROM continuumio/miniconda:4.5.11
+
+# install git
+RUN apt-get update && apt-get upgrade -y && apt-get install -y git
+
+# create a new conda environment named azureml
+RUN conda create -n azureml -y -q Python=3.6
+
+# install additional packages used by sample notebooks. this is optional
+RUN ["/bin/bash", "-c", "source activate azureml && conda install -y tqdm cython matplotlib scikit-learn"]
+
+# install azurmel-sdk components
+RUN ["/bin/bash", "-c", "source activate azureml && pip install azureml-sdk[notebooks]==1.0.41"]
+
+# clone Azure ML GitHub sample notebooks
+RUN cd /home && git clone -b "azureml-sdk-1.0.41" --single-branch https://github.com/Azure/MachineLearningNotebooks.git
+
+# generate jupyter configuration file
+RUN ["/bin/bash", "-c", "source activate azureml && mkdir ~/.jupyter && cd ~/.jupyter && jupyter notebook --generate-config"]
+
+# set an emtpy token for Jupyter to remove authentication. 
+# this is NOT recommended for production environment
+RUN echo "c.NotebookApp.token = ''" >> ~/.jupyter/jupyter_notebook_config.py
+
+# open up port 8887 on the container
+EXPOSE 8887
+
+# start Jupyter notebook server on port 8887 when the container starts
+CMD /bin/bash -c "cd /home/MachineLearningNotebooks && source activate azureml && jupyter notebook --port 8887 --no-browser --ip 0.0.0.0 --allow-root"

Dockerfiles/1.0.43/Dockerfile

@@ -0,0 +1,29 @@
+FROM continuumio/miniconda:4.5.11
+
+# install git
+RUN apt-get update && apt-get upgrade -y && apt-get install -y git
+
+# create a new conda environment named azureml
+RUN conda create -n azureml -y -q Python=3.6
+
+# install additional packages used by sample notebooks. this is optional
+RUN ["/bin/bash", "-c", "source activate azureml && conda install -y tqdm cython matplotlib scikit-learn"]
+
+# install azurmel-sdk components
+RUN ["/bin/bash", "-c", "source activate azureml && pip install azureml-sdk[notebooks]==1.0.43"]
+
+# clone Azure ML GitHub sample notebooks
+RUN cd /home && git clone -b "azureml-sdk-1.0.43" --single-branch https://github.com/Azure/MachineLearningNotebooks.git
+
+# generate jupyter configuration file
+RUN ["/bin/bash", "-c", "source activate azureml && mkdir ~/.jupyter && cd ~/.jupyter && jupyter notebook --generate-config"]
+
+# set an emtpy token for Jupyter to remove authentication. 
+# this is NOT recommended for production environment
+RUN echo "c.NotebookApp.token = ''" >> ~/.jupyter/jupyter_notebook_config.py
+
+# open up port 8887 on the container
+EXPOSE 8887
+
+# start Jupyter notebook server on port 8887 when the container starts
+CMD /bin/bash -c "cd /home/MachineLearningNotebooks && source activate azureml && jupyter notebook --port 8887 --no-browser --ip 0.0.0.0 --allow-root"

Licenses/sdk-license/LICENSE

@@ -1,3 +1,4 @@
+    
 This software is made available to you on the condition that you agree to
 [your agreement][1] governing your use of Azure.
 If you do not have an existing agreement governing your use of Azure, you agree that 

NBSETUP.md

@@ -1,6 +1,4 @@
-# Setting up environment
-
----
+# Set up your notebook environment for Azure Machine Learning
 
 To run the notebooks in this repository use one of following options.
 
@@ -12,9 +10,7 @@ Azure Notebooks is a hosted Jupyter-based notebook service in the Azure cloud. A
 1. Follow the instructions in the [Configuration](configuration.ipynb) notebook to create and connect to a workspace
 1. Open one of the sample notebooks
 
-    **Make sure the Azure Notebook kernel is set to `Python 3.6`** when you open a notebook
-
-    ![set kernel to Python 3.6](images/python36.png)
+    **Make sure the Azure Notebook kernel is set to `Python 3.6`** when you open a notebook by choosing Kernel > Change Kernel > Python 3.6 from the menus.
 
 ## **Option 2: Use your own notebook server**
 
@@ -28,11 +24,8 @@ pip install azureml-sdk
 git clone https://github.com/Azure/MachineLearningNotebooks.git
 
 # below steps are optional
-# install the base SDK and a Jupyter notebook server
-pip install azureml-sdk[notebooks]
-
-# install the data prep component
-pip install azureml-dataprep
+# install the base SDK, Jupyter notebook server and tensorboard
+pip install azureml-sdk[notebooks,tensorboard]
 
 # install model explainability component
 pip install azureml-sdk[explain]
@@ -58,8 +51,7 @@ Please make sure you start with the [Configuration](configuration.ipynb) noteboo
 
 ### Video walkthrough:
 
-[![Get Started video](images/yt_cover.png)](https://youtu.be/VIsXeTuW3FU)
-
+[!VIDEO https://youtu.be/VIsXeTuW3FU]
 
 ## **Option 3: Use Docker**
 
@@ -90,9 +82,6 @@ Now you can point your browser to http://localhost:8887. We recommend that you s
 If you need additional Azure ML SDK components, you can either modify the Docker files before you build the Docker images to add additional steps, or install them through command line in the live container after you build the Docker image. For example:
 
 ```sh
-# install dataprep components
-pip install azureml-dataprep
-
 # install the core SDK and automated ml components
 pip install azureml-sdk[automl]
 

README.md

@@ -2,7 +2,8 @@
 
 This repository contains example notebooks demonstrating the [Azure Machine Learning](https://azure.microsoft.com/en-us/services/machine-learning-service/) Python SDK which allows you to build, train, deploy and manage machine learning solutions using Azure.  The AML SDK allows you the choice of using local or cloud compute resources, while managing and maintaining the complete data science workflow from the cloud.
 
-![Azure ML workflow](https://raw.githubusercontent.com/MicrosoftDocs/azure-docs/master/articles/machine-learning/service/media/overview-what-is-azure-ml/aml.png)
+![Azure ML Workflow](https://raw.githubusercontent.com/MicrosoftDocs/azure-docs/master/articles/machine-learning/service/media/concept-azure-machine-learning-architecture/workflow.png)
+
 
 ## Quick installation
 ```sh
@@ -11,17 +12,17 @@ pip install azureml-sdk
 Read more detailed instructions on [how to set up your environment](./NBSETUP.md) using Azure Notebook service, your own Jupyter notebook server, or Docker.
 
 ## How to navigate and use the example notebooks?
-You should always run the [Configuration](./configuration.ipynb) notebook first when setting up a notebook library on a new machine or in a new environment. It configures your notebook library to connect to an Azure Machine Learning workspace, and sets up your workspace and compute to be used by many of the other examples. 
+If you are using an Azure Machine Learning Notebook VM, you are all set. Otherwise, you should always run the [Configuration](./configuration.ipynb) notebook first when setting up a notebook library on a new machine or in a new environment. It configures your notebook library to connect to an Azure Machine Learning workspace, and sets up your workspace and compute to be used by many of the other examples. 
+This [index](.index.md) should assist in navigating the Azure Machine Learning notebook samples and encourage efficient retrieval of topics and content. 
 
 If you want to...
 
  * ...try out and explore Azure ML, start with image classification tutorials: [Part 1 (Training)](./tutorials/img-classification-part1-training.ipynb) and [Part 2 (Deployment)](./tutorials/img-classification-part2-deploy.ipynb).
- * ...prepare your data and do automated machine learning, start with regression tutorials: [Part 1 (Data Prep)](./tutorials/regression-part1-data-prep.ipynb) and [Part 2 (Automated ML)](./tutorials/regression-part2-automated-ml.ipynb).
  * ...learn about experimentation and tracking run history, first [train within Notebook](./how-to-use-azureml/training/train-within-notebook/train-within-notebook.ipynb), then try [training on remote VM](./how-to-use-azureml/training/train-on-remote-vm/train-on-remote-vm.ipynb) and [using logging APIs](./how-to-use-azureml/training/logging-api/logging-api.ipynb).
  * ...train deep learning models at scale, first learn about [Machine Learning Compute](./how-to-use-azureml/training/train-on-amlcompute/train-on-amlcompute.ipynb), and then try [distributed hyperparameter tuning](./how-to-use-azureml/training-with-deep-learning/train-hyperparameter-tune-deploy-with-pytorch/train-hyperparameter-tune-deploy-with-pytorch.ipynb) and [distributed training](./how-to-use-azureml/training-with-deep-learning/distributed-pytorch-with-horovod/distributed-pytorch-with-horovod.ipynb).
  * ...deploy models as a realtime scoring service, first learn the basics by [training within Notebook and deploying to Azure Container Instance](./how-to-use-azureml/training/train-within-notebook/train-within-notebook.ipynb), then learn how to [register and manage models, and create Docker images](./how-to-use-azureml/deployment/register-model-create-image-deploy-service/register-model-create-image-deploy-service.ipynb), and [production deploy models on Azure Kubernetes Cluster](./how-to-use-azureml/deployment/production-deploy-to-aks/production-deploy-to-aks.ipynb).
- * ...deploy models as a batch scoring service, first [train a model within Notebook](./how-to-use-azureml/training/train-within-notebook/train-within-notebook.ipynb), learn how to [register and manage models](./how-to-use-azureml/deployment/register-model-create-image-deploy-service/register-model-create-image-deploy-service.ipynb), then [create Machine Learning Compute for scoring compute](./how-to-use-azureml/training/train-on-amlcompute/train-on-amlcompute.ipynb), and [use Machine Learning Pipelines to deploy your model](./how-to-use-azureml/machine-learning-pipelines/pipeline-mpi-batch-prediction.ipynb).
- * ...monitor your deployed models, learn about using [App Insights](./how-to-use-azureml/deployment/enable-app-insights-in-production-service/enable-app-insights-in-production-service.ipynb) and [model data collection](./how-to-use-azureml/deployment/enable-data-collection-for-models-in-aks/enable-data-collection-for-models-in-aks.ipynb).
+ * ...deploy models as a batch scoring service, first [train a model within Notebook](./how-to-use-azureml/training/train-within-notebook/train-within-notebook.ipynb), learn how to [register and manage models](./how-to-use-azureml/deployment/register-model-create-image-deploy-service/register-model-create-image-deploy-service.ipynb), then [create Machine Learning Compute for scoring compute](./how-to-use-azureml/training/train-on-amlcompute/train-on-amlcompute.ipynb), and [use Machine Learning Pipelines to deploy your model](https://aka.ms/pl-batch-scoring).
+ * ...monitor your deployed models, learn about using [App Insights](./how-to-use-azureml/deployment/enable-app-insights-in-production-service/enable-app-insights-in-production-service.ipynb).
 
 ## Tutorials
 
@@ -38,6 +39,7 @@ The [How to use Azure ML](./how-to-use-azureml) folder contains specific example
 - [Machine Learning Pipelines](./how-to-use-azureml/machine-learning-pipelines) - Examples showing how to create and use reusable pipelines for training and batch scoring
 - [Deployment](./how-to-use-azureml/deployment) - Examples showing how to deploy and manage machine learning models and solutions
 - [Azure Databricks](./how-to-use-azureml/azure-databricks) - Examples showing how to use Azure ML with Azure Databricks
+- [Monitor Models](./how-to-use-azureml/monitor-models) - Examples showing how to enable model monitoring services such as DataDrift
 
 ---
 ## Documentation
@@ -48,9 +50,27 @@ The [How to use Azure ML](./how-to-use-azureml) folder contains specific example
 
 ---
 
+
+## Community Repository 
+Visit this [community repository](https://github.com/microsoft/MLOps/tree/master/examples) to find useful end-to-end sample notebooks. Also, please follow these [contribution guidelines](https://github.com/microsoft/MLOps/blob/master/contributing.md) when contributing to this repository.   
+
 ## Projects using Azure Machine Learning
 
 Visit following repos to see projects contributed by Azure ML users:
+ - [AMLSamples](https://github.com/Azure/AMLSamples) Number of end-to-end examples, including face recognition, predictive maintenance, customer churn and sentiment analysis.
+ - [Learn about Natural Language Processing best practices using Azure Machine Learning service](https://github.com/microsoft/nlp)
+ - [Pre-Train BERT models using Azure Machine Learning service](https://github.com/Microsoft/AzureML-BERT)
+ - [Fashion MNIST with Azure ML SDK](https://github.com/amynic/azureml-sdk-fashion)
+ - [UMass Amherst Student Samples](https://github.com/katiehouse3/microsoft-azure-ml-notebooks) - A number of end-to-end machine learning notebooks, including machine translation, image classification, and customer churn, created by students in the 696DS course at UMass Amherst.
+ 
+## Data/Telemetry 
+This repository collects usage data and sends it to Mircosoft to help improve our products and services. Read Microsoft's [privacy statement to learn more](https://privacy.microsoft.com/en-US/privacystatement)
 
- - [Fine tune natural language processing models using Azure Machine Learning service](https://github.com/Microsoft/AzureML-BERT)
- - [Fashion MNIST with Azure ML SDK](https://github.com/amynic/azureml-sdk-fashion)
+To opt out of tracking, please go to the raw markdown or .ipynb files and remove the following line of code:
+
+```sh
+    "![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/README.png)"
+```
+This URL will be slightly different depending on the file. 
+
+ ![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/README.png)

configuration.ipynb

@@ -9,6 +9,13 @@
         "Licensed under the MIT License."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/configuration.png)"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {},
@@ -51,7 +58,7 @@
         "\n",
         "### What is an Azure Machine Learning workspace\n",
         "\n",
-        "An Azure ML Workspace is an Azure resource that organizes and coordinates the actions of many other Azure resources to assist in executing and sharing machine learning workflows.  In particular, an Azure ML Workspace coordinates storage, databases, and compute resources providing added functionality for machine learning experimentation, deployment, inferencing, and the monitoring of deployed models."
+        "An Azure ML Workspace is an Azure resource that organizes and coordinates the actions of many other Azure resources to assist in executing and sharing machine learning workflows.  In particular, an Azure ML Workspace coordinates storage, databases, and compute resources providing added functionality for machine learning experimentation, deployment, inference, and the monitoring of deployed models."
       ]
     },
     {
@@ -96,7 +103,7 @@
       "source": [
         "import azureml.core\n",
         "\n",
-        "print(\"This notebook was created using version 1.0.21 of the Azure ML SDK\")\n",
+        "print(\"This notebook was created using version 1.0.79 of the Azure ML SDK\")\n",
         "print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
       ]
     },
@@ -207,7 +214,10 @@
         "* You do not have permission to create a resource group if it's non-existing.\n",
         "* You are not a subscription owner or contributor and no Azure ML workspaces have ever been created in this subscription\n",
         "\n",
-        "If workspace creation fails, please work with your IT admin to provide you with the appropriate permissions or to provision the required resources."
+        "If workspace creation fails, please work with your IT admin to provide you with the appropriate permissions or to provision the required resources.\n",
+        "\n",
+        "**Note**: A Basic workspace is created by default. If you would like to create an Enterprise workspace, please specify sku = 'enterprise'.\n",
+        "Please visit our [pricing page](https://azure.microsoft.com/en-us/pricing/details/machine-learning/) for more details on our Enterprise edition.\n"
       ]
     },
     {
@@ -228,6 +238,7 @@
         "                      resource_group = resource_group, \n",
         "                      location = workspace_region,\n",
         "                      create_resource_group = True,\n",
+        "                      sku = 'basic',\n",
         "                      exist_ok = True)\n",
         "ws.get_details()\n",
         "\n",
@@ -251,7 +262,7 @@
         "```shell\n",
         "az vm list-skus -o tsv\n",
         "```\n",
-        "* min_nodes - this sets the minimum size of the cluster.  If you set the minimum to 0 the cluster will shut down all nodes while note in use.  Setting this number to a value higher than 0 will allow for faster start-up times, but you will also be billed when the cluster is not in use.\n",
+        "* min_nodes - this sets the minimum size of the cluster.  If you set the minimum to 0 the cluster will shut down all nodes while not in use.  Setting this number to a value higher than 0 will allow for faster start-up times, but you will also be billed when the cluster is not in use.\n",
         "* max_nodes - this sets the maximum size of the cluster.  Setting this to a larger number allows for more concurrency and a greater distributed processing of scale-out jobs.\n",
         "\n",
         "\n",
@@ -268,14 +279,14 @@
         "from azureml.core.compute_target import ComputeTargetException\n",
         "\n",
         "# Choose a name for your CPU cluster\n",
-        "cpu_cluster_name = \"cpucluster\"\n",
+        "cpu_cluster_name = \"cpu-cluster\"\n",
         "\n",
         "# Verify that cluster does not exist already\n",
         "try:\n",
         "    cpu_cluster = ComputeTarget(workspace=ws, name=cpu_cluster_name)\n",
-        "    print(\"Found existing cpucluster\")\n",
+        "    print(\"Found existing cpu-cluster\")\n",
         "except ComputeTargetException:\n",
-        "    print(\"Creating new cpucluster\")\n",
+        "    print(\"Creating new cpu-cluster\")\n",
         "    \n",
         "    # Specify the configuration for the new cluster\n",
         "    compute_config = AmlCompute.provisioning_configuration(vm_size=\"STANDARD_D2_V2\",\n",
@@ -306,14 +317,14 @@
         "from azureml.core.compute_target import ComputeTargetException\n",
         "\n",
         "# Choose a name for your GPU cluster\n",
-        "gpu_cluster_name = \"gpucluster\"\n",
+        "gpu_cluster_name = \"gpu-cluster\"\n",
         "\n",
         "# Verify that cluster does not exist already\n",
         "try:\n",
         "    gpu_cluster = ComputeTarget(workspace=ws, name=gpu_cluster_name)\n",
         "    print(\"Found existing gpu cluster\")\n",
         "except ComputeTargetException:\n",
-        "    print(\"Creating new gpucluster\")\n",
+        "    print(\"Creating new gpu-cluster\")\n",
         "    \n",
         "    # Specify the configuration for the new cluster\n",
         "    compute_config = AmlCompute.provisioning_configuration(vm_size=\"STANDARD_NC6\",\n",
@@ -350,7 +361,7 @@
   "metadata": {
     "authors": [
       {
-        "name": "roastala"
+        "name": "ninhu"
       }
     ],
     "kernelspec": {

configuration.yml

@@ -0,0 +1,4 @@
+name: configuration
+dependencies:
+- pip:
+  - azureml-sdk

contrib/RAPIDS/process_data.py

@@ -15,21 +15,6 @@ from glob import glob
 import os
 import argparse
 
-def initialize_rmm_pool():
-    from librmm_cffi import librmm_config as rmm_cfg
-
-    rmm_cfg.use_pool_allocator = True
-    #rmm_cfg.initial_pool_size = 2<<30 # set to 2GiB. Default is 1/2 total GPU memory
-    import cudf
-    return cudf._gdf.rmm_initialize()
-
-def initialize_rmm_no_pool():
-    from librmm_cffi import librmm_config as rmm_cfg
-    
-    rmm_cfg.use_pool_allocator = False
-    import cudf
-    return cudf._gdf.rmm_initialize()
-
 def run_dask_task(func, **kwargs):
     task = func(**kwargs)
     return task
@@ -207,26 +192,26 @@ def gpu_load_names(col_path):
 
 def create_ever_features(gdf, **kwargs):
     everdf = gdf[['loan_id', 'current_loan_delinquency_status']]
-    everdf = everdf.groupby('loan_id', method='hash').max()
+    everdf = everdf.groupby('loan_id', method='hash').max().reset_index()
     del(gdf)
-    everdf['ever_30'] = (everdf['max_current_loan_delinquency_status'] >= 1).astype('int8')
-    everdf['ever_90'] = (everdf['max_current_loan_delinquency_status'] >= 3).astype('int8')
-    everdf['ever_180'] = (everdf['max_current_loan_delinquency_status'] >= 6).astype('int8')
-    everdf.drop_column('max_current_loan_delinquency_status')
+    everdf['ever_30'] = (everdf['current_loan_delinquency_status'] >= 1).astype('int8')
+    everdf['ever_90'] = (everdf['current_loan_delinquency_status'] >= 3).astype('int8')
+    everdf['ever_180'] = (everdf['current_loan_delinquency_status'] >= 6).astype('int8')
+    everdf.drop_column('current_loan_delinquency_status')
     return everdf
 
 def create_delinq_features(gdf, **kwargs):
     delinq_gdf = gdf[['loan_id', 'monthly_reporting_period', 'current_loan_delinquency_status']]
     del(gdf)
-    delinq_30 = delinq_gdf.query('current_loan_delinquency_status >= 1')[['loan_id', 'monthly_reporting_period']].groupby('loan_id', method='hash').min()
-    delinq_30['delinquency_30'] = delinq_30['min_monthly_reporting_period']
-    delinq_30.drop_column('min_monthly_reporting_period')
-    delinq_90 = delinq_gdf.query('current_loan_delinquency_status >= 3')[['loan_id', 'monthly_reporting_period']].groupby('loan_id', method='hash').min()
-    delinq_90['delinquency_90'] = delinq_90['min_monthly_reporting_period']
-    delinq_90.drop_column('min_monthly_reporting_period')
-    delinq_180 = delinq_gdf.query('current_loan_delinquency_status >= 6')[['loan_id', 'monthly_reporting_period']].groupby('loan_id', method='hash').min()
-    delinq_180['delinquency_180'] = delinq_180['min_monthly_reporting_period']
-    delinq_180.drop_column('min_monthly_reporting_period')
+    delinq_30 = delinq_gdf.query('current_loan_delinquency_status >= 1')[['loan_id', 'monthly_reporting_period']].groupby('loan_id', method='hash').min().reset_index()
+    delinq_30['delinquency_30'] = delinq_30['monthly_reporting_period']
+    delinq_30.drop_column('monthly_reporting_period')
+    delinq_90 = delinq_gdf.query('current_loan_delinquency_status >= 3')[['loan_id', 'monthly_reporting_period']].groupby('loan_id', method='hash').min().reset_index()
+    delinq_90['delinquency_90'] = delinq_90['monthly_reporting_period']
+    delinq_90.drop_column('monthly_reporting_period')
+    delinq_180 = delinq_gdf.query('current_loan_delinquency_status >= 6')[['loan_id', 'monthly_reporting_period']].groupby('loan_id', method='hash').min().reset_index()
+    delinq_180['delinquency_180'] = delinq_180['monthly_reporting_period']
+    delinq_180.drop_column('monthly_reporting_period')
     del(delinq_gdf)
     delinq_merge = delinq_30.merge(delinq_90, how='left', on=['loan_id'], type='hash')
     delinq_merge['delinquency_90'] = delinq_merge['delinquency_90'].fillna(np.dtype('datetime64[ms]').type('1970-01-01').astype('datetime64[ms]'))
@@ -279,16 +264,15 @@ def create_joined_df(gdf, everdf, **kwargs):
 def create_12_mon_features(joined_df, **kwargs):
     testdfs = []
     n_months = 12
+
     for y in range(1, n_months + 1):
         tmpdf = joined_df[['loan_id', 'timestamp_year', 'timestamp_month', 'delinquency_12', 'upb_12']]
         tmpdf['josh_months'] = tmpdf['timestamp_year'] * 12 + tmpdf['timestamp_month']
         tmpdf['josh_mody_n'] = ((tmpdf['josh_months'].astype('float64') - 24000 - y) / 12).floor()
-        tmpdf = tmpdf.groupby(['loan_id', 'josh_mody_n'], method='hash').agg({'delinquency_12': 'max','upb_12': 'min'})
-        tmpdf['delinquency_12'] = (tmpdf['max_delinquency_12']>3).astype('int32')
-        tmpdf['delinquency_12'] +=(tmpdf['min_upb_12']==0).astype('int32')
-        tmpdf.drop_column('max_delinquency_12')
-        tmpdf['upb_12'] = tmpdf['min_upb_12']
-        tmpdf.drop_column('min_upb_12')
+        tmpdf = tmpdf.groupby(['loan_id', 'josh_mody_n'], method='hash').agg({'delinquency_12': 'max','upb_12': 'min'}).reset_index()
+        tmpdf['delinquency_12'] = (tmpdf['delinquency_12']>3).astype('int32')
+        tmpdf['delinquency_12'] +=(tmpdf['upb_12']==0).astype('int32')
+        tmpdf['upb_12'] = tmpdf['upb_12']
         tmpdf['timestamp_year'] = (((tmpdf['josh_mody_n'] * n_months) + 24000 + (y - 1)) / 12).floor().astype('int16')
         tmpdf['timestamp_month'] = np.int8(y)
         tmpdf.drop_column('josh_mody_n')
@@ -329,6 +313,7 @@ def last_mile_cleaning(df, **kwargs):
         'delinquency_30', 'delinquency_90', 'delinquency_180', 'upb_12',
         'zero_balance_effective_date','foreclosed_after', 'disposition_date','timestamp'
     ]
+
     for column in drop_list:
         df.drop_column(column)
     for col, dtype in df.dtypes.iteritems():
@@ -342,7 +327,6 @@ def last_mile_cleaning(df, **kwargs):
     return df.to_arrow(preserve_index=False)
 
 def main():
-    #print('XGBOOST_BUILD_DOC is ' + os.environ['XGBOOST_BUILD_DOC'])
     parser = argparse.ArgumentParser("rapidssample")
     parser.add_argument("--data_dir", type=str, help="location of data")
     parser.add_argument("--num_gpu", type=int, help="Number of GPUs to use", default=1)
@@ -364,7 +348,6 @@ def main():
     print('data_dir = {0}'.format(data_dir))
     print('num_gpu = {0}'.format(num_gpu))
     print('part_count = {0}'.format(part_count))
-    #part_count = part_count + 1 # adding one because the usage below is not inclusive
     print('end_year = {0}'.format(end_year))
     print('cpu_predictor = {0}'.format(cpu_predictor))
     
@@ -380,19 +363,17 @@ def main():
     client
     print(client.ncores())
 
-# to download data for this notebook, visit https://rapidsai.github.io/demos/datasets/mortgage-data and update the following paths accordingly
+    # to download data for this notebook, visit https://rapidsai.github.io/demos/datasets/mortgage-data and update the following paths accordingly
     acq_data_path = "{0}/acq".format(data_dir) #"/rapids/data/mortgage/acq"
     perf_data_path = "{0}/perf".format(data_dir) #"/rapids/data/mortgage/perf"
     col_names_path = "{0}/names.csv".format(data_dir) # "/rapids/data/mortgage/names.csv"
     start_year = 2000
-#end_year = 2000 # end_year is inclusive -- converted to parameter
-#part_count = 2 # the number of data files to train against -- converted to parameter
 
-    client.run(initialize_rmm_pool)
     client
-    print(client.ncores())
-# NOTE: The ETL calculates additional features which are then dropped before creating the XGBoost DMatrix.
-# This can be optimized to avoid calculating the dropped features.
+    print('--->>> Workers used: {0}'.format(client.ncores()))
+
+    # NOTE: The ETL calculates additional features which are then dropped before creating the XGBoost DMatrix.
+    # This can be optimized to avoid calculating the dropped features.
     print("Reading ...")
     t1 = datetime.datetime.now()
     gpu_dfs = []
@@ -414,14 +395,9 @@ def main():
             
     wait(gpu_dfs)
     t2 = datetime.datetime.now()
-    print("Reading time ...")
-    print(t2-t1)
-    print('len(gpu_dfs) is {0}'.format(len(gpu_dfs)))
-    
-    client.run(cudf._gdf.rmm_finalize)
-    client.run(initialize_rmm_no_pool)
-    client
-    print(client.ncores())
+    print("Reading time: {0}".format(str(t2-t1)))
+    print('--->>> Number of data parts: {0}'.format(len(gpu_dfs)))
+
     dxgb_gpu_params = {
         'nround':            100,
         'max_depth':         8,
@@ -438,7 +414,7 @@ def main():
         'n_gpus':            1, 
         'distributed_dask':  True,
         'loss':              'ls',
-        'objective':         'gpu:reg:linear',
+        'objective':         'reg:squarederror',
         'max_features':      'auto',
         'criterion':         'friedman_mse',
         'grow_policy':       'lossguide',
@@ -446,13 +422,13 @@ def main():
     }
       
     if cpu_predictor:
-        print('Training using CPUs')
+        print('\n---->>>> Training using CPUs <<<<----\n')
         dxgb_gpu_params['predictor'] = 'cpu_predictor'
         dxgb_gpu_params['tree_method'] = 'hist'
         dxgb_gpu_params['objective'] = 'reg:linear'
         
     else:
-        print('Training using GPUs')
+        print('\n---->>>> Training using GPUs <<<<----\n')
     
     print('Training parameters are {0}'.format(dxgb_gpu_params))
     
@@ -481,14 +457,13 @@ def main():
     gpu_dfs = [gpu_df.persist() for gpu_df in gpu_dfs]
     gc.collect()
     wait(gpu_dfs)
-    
+
+    # TRAIN THE MODEL
     labels = None
     t1 = datetime.datetime.now()
     bst = dxgb_gpu.train(client, dxgb_gpu_params, gpu_dfs, labels, num_boost_round=dxgb_gpu_params['nround'])
     t2 = datetime.datetime.now()
-    print("Training time ...")
-    print(t2-t1)
-    print('str(bst) is {0}'.format(str(bst)))
+    print('\n---->>>> Training time: {0} <<<<----\n'.format(str(t2-t1)))
     print('Exiting script')
 
 if __name__ == '__main__':

contrib/RAPIDS/rapids.yml

@@ -1,35 +0,0 @@
-name: rapids
-channels:
-- nvidia
-- numba
-- conda-forge
-- rapidsai
-- defaults
-- pytorch
-
-dependencies:
-- arrow-cpp=0.12.0
-- bokeh
-- cffi=1.11.5
-- cmake=3.12
-- cuda92
-- cython==0.29
-- dask=1.1.1
-- distributed=1.25.3
-- faiss-gpu=1.5.0
-- numba=0.42
-- numpy=1.15.4
-- nvstrings
-- pandas=0.23.4
-- pyarrow=0.12.0
-- scikit-learn
-- scipy
-- cudf
-- cuml
-- python=3.6.2
-- jupyterlab
-- pip:
-  - file:/rapids/xgboost/python-package/dist/xgboost-0.81-py3-none-any.whl
-  - git+https://github.com/rapidsai/dask-xgboost@dask-cudf
-  - git+https://github.com/rapidsai/dask-cudf@master
-  - git+https://github.com/rapidsai/dask-cuda@master

contrib/batch_inferencing/Code/digit_identification.py

@@ -0,0 +1,41 @@
+
+# Copyright (c) Microsoft. All rights reserved.
+# Licensed under the MIT license.
+
+import os
+import numpy as np
+import tensorflow as tf
+from PIL import Image
+from azureml.core import Model
+
+
+def init():
+    global g_tf_sess
+
+    # pull down model from workspace
+    model_path = Model.get_model_path("mnist")
+
+    # contruct graph to execute
+    tf.reset_default_graph()
+    saver = tf.train.import_meta_graph(os.path.join(model_path, 'mnist-tf.model.meta'))
+    g_tf_sess = tf.Session(config=tf.ConfigProto(device_count={'GPU': 0}))
+    saver.restore(g_tf_sess, os.path.join(model_path, 'mnist-tf.model'))
+
+
+def run(mini_batch):
+    print(f'run method start: {__file__}, run({mini_batch})')
+    resultList = []
+    in_tensor = g_tf_sess.graph.get_tensor_by_name("network/X:0")
+    output = g_tf_sess.graph.get_tensor_by_name("network/output/MatMul:0")
+
+    for image in mini_batch:
+        # prepare each image
+        data = Image.open(image)
+        np_im = np.array(data).reshape((1, 784))
+        # perform inference
+        inference_result = output.eval(feed_dict={in_tensor: np_im}, session=g_tf_sess)
+        # find best probability, and add to result list
+        best_result = np.argmax(inference_result)
+        resultList.append("{}: {}".format(os.path.basename(image), best_result))
+
+    return resultList

contrib/batch_inferencing/Code/iris_score.py

@@ -0,0 +1,31 @@
+import io
+import pickle
+import argparse
+import numpy as np
+
+from azureml.core.model import Model
+from sklearn.linear_model import LogisticRegression
+
+
+def init():
+    global iris_model
+
+    parser = argparse.ArgumentParser(description="Iris model serving")
+    parser.add_argument('--model_name', dest="model_name", required=True)
+    args, unknown_args = parser.parse_known_args()
+
+    model_path = Model.get_model_path(args.model_name)
+    with open(model_path, 'rb') as model_file:
+        iris_model = pickle.load(model_file)
+
+
+def run(input_data):
+    # make inference
+    num_rows, num_cols = input_data.shape
+    pred = iris_model.predict(input_data).reshape((num_rows, 1))
+
+    # cleanup output
+    result = input_data.drop(input_data.columns[4:], axis=1)
+    result['variety'] = pred
+
+    return result

contrib/batch_inferencing/README.md

@@ -0,0 +1,127 @@
+# Azure Machine Learning Batch Inference
+
+Azure Machine Learning Batch Inference targets large inference jobs that are not time-sensitive. Batch Inference provides cost-effective inference compute scaling, with unparalleled throughput for asynchronous applications. It is optimized for high-throughput, fire-and-forget inference over large collections of data.
+
+# Getting Started with Batch Inference Public Preview
+
+Batch inference public preview offers a platform in which to do large inference or generic parallel map-style operations. Below introduces the major steps to use this new functionality. For a quick try, please follow the prerequisites and simply run the sample notebooks provided in this directory.
+
+## Prerequisites
+
+### Python package installation
+Following the convention of most AzureML Public Preview features, Batch Inference SDK is currently available as a contrib package.
+
+If you're unfamiliar with creating a new Python environment, you may follow this example for [creating a conda environment](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-configure-environment#local).  Batch Inference package can be installed through the following pip command.
+```
+pip install azureml-contrib-pipeline-steps
+```
+
+### Creation of Azure Machine Learning Workspace
+If you do not already have a Azure ML Workspace, please run the [configuration Notebook](../../configuration.ipynb).
+
+## Configure a Batch Inference job
+
+To run a Batch Inference job, you will need to gather some configuration data.
+
+1. **ParallelRunConfig**
+    - **entry_script**: the local file path to the scoring script. If source_directory is specified, use relative path, otherwise use any path accessible on machine.
+    - **error_threshold**: the number of record failures for TabularDataset and file failures for FileDataset that should be ignored during processing. If the aggregated error count (across all workers) goes above this value, then the job will be aborted. Set to -1 to ignore all failures during processing.
+    - **output_action**: one of the following values
+        - **"append_row"**: all values output by run() method invocations will be aggregated into one unique file named parallel_run_step.txt that is created in the output location.
+        - **"summary_only"** – scoring script will handle the output by itself.  The script still needs to return one output row per successfully-processed input item. This is used for error threshold calculation (the actual value of the output row is ignored).
+    - **source_directory**: supporting files for scoring (optional)
+    - **compute_target**: only **AmlCompute** is supported currently
+    - **node_count**: number of compute nodes to use.
+    - **process_count_per_node**: number of processes per node (optional, default value is 1).
+    - **mini_batch_size**: the approximate amount of input data passed to each run() invocation.  For FileDataset input, this is number of files user script can process in one run() call. For TabularDataset input it is approximate size of data user script can process in one run() call. E.g. 1024, 1024KB, 10MB, 1GB (optional, default value 10 files for FileDataset and 1MB for TabularDataset.)
+    - **logging_level**: log verbosity. Values in increasing verbosity are: 'WARNING', 'INFO', 'DEBUG' (optional, default value is 'INFO').
+    - **run_invocation_timeout**: run method invocation timeout period in seconds (optional, default value is 60).
+    - **environment**: The environment definition. This field configures the Python environment. It can be configured to use an existing Python environment or to set up a temp environment for the experiment. The definition is also responsible for setting the required application dependencies.
+    - **description**: name given to batch service.
+
+2. **Scoring (entry) script**: entry point for execution, scoring script should contain two functions:
+    - **init()**: this function should be used for any costly or common preparation for subsequent inferences, e.g., deserializing and loading the model into a global object.
+    - **run(mini_batch)**: The method to be parallelized. Each invocation will have one minibatch.
+        - **mini_batch**: Batch inference will invoke run method and pass either a list or Pandas DataFrame as an argument to the method. Each entry in min_batch will be - a filepath if input is a FileDataset, a Pandas DataFrame if input is a TabularDataset.
+        - **return value**: run() method should return a Pandas DataFrame or an array. For append_row output_action, these returned elements are appended into the common output file. For summary_only, the contents of the elements are ignored. For all output actions, each returned output element indicates one successful inference of input element in the input mini-batch.
+
+3. **Base image** (optional)
+    - if GPU is required, use DEFAULT_GPU_IMAGE as base image in environment. [Example GPU environment](./file-dataset-image-inference-mnist.ipynb#specify-the-environment-to-run-the-script)
+
+Example image pull:
+```python
+from azureml.core.runconfig import ContainerRegistry
+
+# use an image available in public Container Registry without authentication
+public_base_image = "mcr.microsoft.com/azureml/o16n-sample-user-base/ubuntu-miniconda"
+
+# or use an image available in a private Container Registry
+base_image = "myregistry.azurecr.io/mycustomimage:1.0"
+base_image_registry = ContainerRegistry()
+base_image_registry.address = "myregistry.azurecr.io"
+base_image_registry.username = "username"
+base_image_registry.password = "password"
+```
+
+
+## Create a batch inference job
+
+**ParallelRunStep** is a newly added step in the azureml.contrib.pipeline.steps package. You will use it to add a step to create a batch inference job with your Azure machine learning pipeline. (Use batch inference without an Azure machine learning pipeline is not supported yet). ParallelRunStep has all the following parameters:
+  - **name**: this name will be used to register batch inference service, has the following naming restrictions: (unique, 3-32 chars and regex ^\[a-z\]([-a-z0-9]*[a-z0-9])?$)
+  - **models**: zero or more model names already registered in Azure Machine Learning model registry.
+  - **parallel_run_config**: ParallelRunConfig as defined above.
+  - **inputs**: one or more Dataset objects.
+  - **output**: this should be a PipelineData object encapsulating an Azure BLOB container path. 
+  - **arguments**: list of custom arguments passed to scoring script (optional)
+  - **allow_reuse**: optional, default value is True. If the inputs remain the same as a previous run, it will make the previous run results immediately available (skips re-computing the step).
+
+## Passing arguments from pipeline submission to script
+
+Many tasks require arguments to be passed from job submission to the distributed runs. Below is an example to pass such information.
+```
+# from script which creates pipeline job
+parallelrun_step = ParallelRunStep(
+  ...
+  arguments=["--model_name", "mosaic"]     # name of the model we want to use, in case we have more than one option
+)
+```
+```
+# from driver.py/score.py/task.py
+import argparse
+
+parser.add_argument('--model_name', dest="model_name")
+
+args, unknown_args = parser.parse_known_args()
+
+# to access values
+args.model_name # "mosaic"
+```
+
+## Submit a batch inference job
+
+You can submit a batch inference job by pipeline_run, or through REST calls with a published pipeline. To control node count using REST API/experiment, please use aml_node_count(special) pipeline parameter. A typical use case follows:
+
+```python
+pipeline = Pipeline(workspace=ws, steps=[parallelrun_step])
+pipeline_run = Experiment(ws, 'name_of_pipeline_run').submit(pipeline)
+```
+
+## Monitor your batch inference job
+
+A batch inference job can take a long time to finish. You can monitor your job's progress from Azure portal, using Azure ML widgets, view console output through SDK, or check out overview.txt in log/azureml directory.
+
+```python
+# view with widgets (will display GUI inside a browser)
+from azureml.widgets import RunDetails
+RunDetails(pipeline_run).show()
+
+# simple console output
+pipeline_run.wait_for_completion(show_output=True)
+```
+
+# Sample notebooks
+
+-  [file-dataset-image-inference-mnist.ipynb](./file-dataset-image-inference-mnist.ipynb) demonstrates how to run batch inference on an MNIST dataset.
+-  [tabular-dataset-inference-iris.ipynb](./tabular-dataset-inference-iris.ipynb) demonstrates how to run batch inference on an IRIS dataset.
+
+![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/contrib/batch_inferencing/README.png) 

contrib/batch_inferencing/file-dataset-image-inference-mnist.ipynb

@@ -0,0 +1,563 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Copyright (c) Microsoft Corporation. All rights reserved. \n",
+        "Licensed under the MIT License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/machine-learning-pipelines/contrib/batch_inferencing/file-dataset-image-inference-mnist.png)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# Using Azure Machine Learning Pipelines for Batch Inference\n",
+        "\n",
+        "In this notebook, we will demonstrate how to make predictions on large quantities of data asynchronously using the ML pipelines with Azure Machine Learning. Batch inference (or batch scoring) provides cost-effective inference, with unparalleled throughput for asynchronous applications. Batch prediction pipelines can scale to perform inference on terabytes of production data. Batch prediction is optimized for high throughput, fire-and-forget predictions for a large collection of data.\n",
+        "\n",
+        "> **Tip**\n",
+        "If your system requires low-latency processing (to process a single document or small set of documents quickly), use [real-time scoring](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-consume-web-service) instead of batch prediction.\n",
+        "\n",
+        "In this example will be take a digit identification model already-trained on MNIST dataset using the [AzureML training with deep learning example notebook](https://github.com/Azure/MachineLearningNotebooks/blob/master/how-to-use-azureml/training-with-deep-learning/train-hyperparameter-tune-deploy-with-keras/train-hyperparameter-tune-deploy-with-keras.ipynb), and run that trained model on some of the MNIST test images in batch.  \n",
+        "\n",
+        "The input dataset used for this notebook differs from a standard MNIST dataset in that it has been converted to PNG images to demonstrate use of files as inputs to Batch Inference. A sample of PNG-converted images of the MNIST dataset were take from [this repository](https://github.com/myleott/mnist_png). \n",
+        "\n",
+        "The outline of this notebook is as follows:\n",
+        "\n",
+        "- Create a DataStore referencing MNIST images stored in a blob container.\n",
+        "- Register the pretrained MNIST model into the model registry. \n",
+        "- Use the registered model to do batch inference on the images in the data blob container.\n",
+        "\n",
+        "## Prerequisites\n",
+        "If you are using an Azure Machine Learning Notebook VM, you are all set. Otherwise, make sure you go through the configuration Notebook located at https://github.com/Azure/MachineLearningNotebooks first. This sets you up with a working config file that has information on your workspace, subscription id, etc. "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Connect to workspace\n",
+        "Create a workspace object from the existing workspace. Workspace.from_config() reads the file config.json and loads the details into an object named ws."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core import Workspace\n",
+        "\n",
+        "ws = Workspace.from_config()\n",
+        "print('Workspace name: ' + ws.name, \n",
+        "      'Azure region: ' + ws.location, \n",
+        "      'Subscription id: ' + ws.subscription_id, \n",
+        "      'Resource group: ' + ws.resource_group, sep = '\\n')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Create or Attach existing compute resource\n",
+        "By using Azure Machine Learning Compute, a managed service, data scientists can train machine learning models on clusters of Azure virtual machines. Examples include VMs with GPU support. In this tutorial, you create Azure Machine Learning Compute as your training environment. The code below creates the compute clusters for you if they don't already exist in your workspace.\n",
+        "\n",
+        "**Creation of compute takes approximately 5 minutes. If the AmlCompute with that name is already in your workspace the code will skip the creation process.**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "from azureml.core.compute import AmlCompute, ComputeTarget\n",
+        "from azureml.core.compute_target import ComputeTargetException\n",
+        "\n",
+        "# choose a name for your cluster\n",
+        "compute_name = os.environ.get(\"AML_COMPUTE_CLUSTER_NAME\", \"cpu-cluster\")\n",
+        "compute_min_nodes = os.environ.get(\"AML_COMPUTE_CLUSTER_MIN_NODES\", 0)\n",
+        "compute_max_nodes = os.environ.get(\"AML_COMPUTE_CLUSTER_MAX_NODES\", 4)\n",
+        "\n",
+        "# This example uses CPU VM. For using GPU VM, set SKU to STANDARD_NC6\n",
+        "vm_size = os.environ.get(\"AML_COMPUTE_CLUSTER_SKU\", \"STANDARD_D2_V2\")\n",
+        "\n",
+        "\n",
+        "if compute_name in ws.compute_targets:\n",
+        "    compute_target = ws.compute_targets[compute_name]\n",
+        "    if compute_target and type(compute_target) is AmlCompute:\n",
+        "        print('found compute target. just use it. ' + compute_name)\n",
+        "else:\n",
+        "    print('creating a new compute target...')\n",
+        "    provisioning_config = AmlCompute.provisioning_configuration(vm_size = vm_size,\n",
+        "                                                                min_nodes = compute_min_nodes, \n",
+        "                                                                max_nodes = compute_max_nodes)\n",
+        "\n",
+        "    # create the cluster\n",
+        "    compute_target = ComputeTarget.create(ws, compute_name, provisioning_config)\n",
+        "    \n",
+        "    # can poll for a minimum number of nodes and for a specific timeout. \n",
+        "    # if no min node count is provided it will use the scale settings for the cluster\n",
+        "    compute_target.wait_for_completion(show_output=True, min_node_count=None, timeout_in_minutes=20)\n",
+        "    \n",
+        "     # For a more detailed view of current AmlCompute status, use get_status()\n",
+        "    print(compute_target.get_status().serialize())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Create a datastore containing sample images\n",
+        "The input dataset used for this notebook differs from a standard MNIST dataset in that it has been converted to PNG images to demonstrate use of files as inputs to Batch Inference. A sample of PNG-converted images of the MNIST dataset were take from [this repository](https://github.com/myleott/mnist_png).\n",
+        "\n",
+        "We have created a public blob container `sampledata` on an account named `pipelinedata`, containing these images from the MNIST dataset. In the next step, we create a datastore with the name `images_datastore`, which points to this blob container. In the call to `register_azure_blob_container` below, setting the `overwrite` flag to `True` overwrites any datastore that was created previously with that name. \n",
+        "\n",
+        "This step can be changed to point to your blob container by providing your own `datastore_name`, `container_name`, and `account_name`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.datastore import Datastore\n",
+        "\n",
+        "account_name = \"pipelinedata\"\n",
+        "datastore_name = \"mnist_datastore\"\n",
+        "container_name = \"sampledata\"\n",
+        "\n",
+        "mnist_data = Datastore.register_azure_blob_container(ws, \n",
+        "                      datastore_name=datastore_name, \n",
+        "                      container_name= container_name, \n",
+        "                      account_name=account_name,\n",
+        "                      overwrite=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Next, let's specify the default datastore for the outputs."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "def_data_store = ws.get_default_datastore()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Create a FileDataset\n",
+        "A [FileDataset](https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.data.filedataset?view=azure-ml-py) references single or multiple files in your datastores or public urls. The files can be of any format. FileDataset provides you with the ability to download or mount the files to your compute. By creating a dataset, you create a reference to the data source location. If you applied any subsetting transformations to the dataset, they will be stored in the dataset as well. The data remains in its existing location, so no extra storage cost is incurred."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.dataset import Dataset\n",
+        "\n",
+        "mnist_ds_name = 'mnist_sample_data'\n",
+        "\n",
+        "path_on_datastore = mnist_data.path('mnist')\n",
+        "input_mnist_ds = Dataset.File.from_files(path=path_on_datastore, validate=False)\n",
+        "registered_mnist_ds = input_mnist_ds.register(ws, mnist_ds_name, create_new_version=True)\n",
+        "named_mnist_ds = registered_mnist_ds.as_named_input(mnist_ds_name)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Intermediate/Output Data\n",
+        "Intermediate data (or output of a Step) is represented by [PipelineData](https://docs.microsoft.com/en-us/python/api/azureml-pipeline-core/azureml.pipeline.core.pipelinedata?view=azure-ml-py) object. PipelineData can be produced by one step and consumed in another step by providing the PipelineData object as an output of one step and the input of one or more steps.\n",
+        "\n",
+        "**Constructing PipelineData**\n",
+        "- name: [Required] Name of the data item within the pipeline graph\n",
+        "- datastore_name: Name of the Datastore to write this output to\n",
+        "- output_name: Name of the output\n",
+        "- output_mode: Specifies \"upload\" or \"mount\" modes for producing output (default: mount)\n",
+        "- output_path_on_compute: For \"upload\" mode, the path to which the module writes this output during execution\n",
+        "- output_overwrite: Flag to overwrite pre-existing data"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.pipeline.core import Pipeline, PipelineData\n",
+        "\n",
+        "output_dir = PipelineData(name=\"inferences\", \n",
+        "                          datastore=def_data_store, \n",
+        "                          output_path_on_compute=\"mnist/results\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Download the Model\n",
+        "\n",
+        "Download and extract the model from https://pipelinedata.blob.core.windows.net/mnist-model/mnist-tf.tar.gz to \"models\" directory"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import tarfile\n",
+        "import urllib.request\n",
+        "\n",
+        "# create directory for model\n",
+        "model_dir = 'models'\n",
+        "if not os.path.isdir(model_dir):\n",
+        "    os.mkdir(model_dir)\n",
+        "\n",
+        "url=\"https://pipelinedata.blob.core.windows.net/mnist-model/mnist-tf.tar.gz\"\n",
+        "response = urllib.request.urlretrieve(url, \"model.tar.gz\")\n",
+        "tar = tarfile.open(\"model.tar.gz\", \"r:gz\")\n",
+        "tar.extractall(model_dir)\n",
+        "\n",
+        "os.listdir(model_dir)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Register the model with Workspace\n",
+        "A registered model is a logical container for one or more files that make up your model. For example, if you have a model that's stored in multiple files, you can register them as a single model in the workspace. After you register the files, you can then download or deploy the registered model and receive all the files that you registered.\n",
+        "\n",
+        "Using tags, you can track useful information such as the name and version of the machine learning library used to train the model. Note that tags must be alphanumeric. Learn more about registering models [here](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-deploy-and-where#registermodel) "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.model import Model\n",
+        "\n",
+        "# register downloaded model \n",
+        "model = Model.register(model_path = \"models/\",\n",
+        "                       model_name = \"mnist\", # this is the name the model is registered as\n",
+        "                       tags = {'pretrained': \"mnist\"},\n",
+        "                       description = \"Mnist trained tensorflow model\",\n",
+        "                       workspace = ws)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Using your model to make batch predictions\n",
+        "To use the model to make batch predictions, you need an **entry script** and a list of **dependencies**:\n",
+        "\n",
+        "#### An entry script\n",
+        "This script accepts requests, scores the requests by using the model, and returns the results.\n",
+        "- __init()__ - Typically this function loads the model into a global object. This function is run only once at the start of batch processing per worker node/process. Init method can make use of following environment variables (ParallelRunStep input):\n",
+        "    1.\tAZUREML_BI_OUTPUT_PATH \u00e2\u20ac\u201c output folder path\n",
+        "- __run(mini_batch)__ - The method to be parallelized. Each invocation will have one minibatch.<BR>\n",
+        "__mini_batch__: Batch inference will invoke run method and pass either a list or Pandas DataFrame as an argument to the method. Each entry in min_batch will be - a filepath if input is a FileDataset, a Pandas DataFrame if input is a TabularDataset.<BR>\n",
+        "__run__ method response: run() method should return a Pandas DataFrame or an array. For append_row output_action, these returned elements are appended into the common output file. For summary_only, the contents of the elements are ignored. For all output actions, each returned output element indicates one successful inference of input element in the input mini-batch.\n",
+        "    User should make sure that enough data is included in inference result to map input to inference. Inference output will be written in output file and not guaranteed to be in order, user should use some key in the output to map it to input.\n",
+        "    \n",
+        "\n",
+        "#### Dependencies\n",
+        "Helper scripts or Python/Conda packages required to run the entry script or model.\n",
+        "\n",
+        "The deployment configuration for the compute target that hosts the deployed model. This configuration describes things like memory and CPU requirements needed to run the model.\n",
+        "\n",
+        "These items are encapsulated into an inference configuration and a deployment configuration. The inference configuration references the entry script and other dependencies. You define these configurations programmatically when you use the SDK to perform the deployment. You define them in JSON files when you use the CLI."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "scripts_folder = \"Code\"\n",
+        "script_file = \"digit_identification.py\"\n",
+        "\n",
+        "# peek at contents\n",
+        "with open(os.path.join(scripts_folder, script_file)) as inference_file:\n",
+        "    print(inference_file.read())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Build and run the batch inference pipeline\n",
+        "The data, models, and compute resource are now available. Let's put all these together in a pipeline."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "###  Specify the environment to run the script\n",
+        "Specify the conda dependencies for your script. This will allow us to install pip packages as well as configure the inference environment."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core import Environment\n",
+        "from azureml.core.runconfig import CondaDependencies, DEFAULT_CPU_IMAGE\n",
+        "\n",
+        "batch_conda_deps = CondaDependencies.create(pip_packages=[\"tensorflow==1.13.1\", \"pillow\"])\n",
+        "\n",
+        "batch_env = Environment(name=\"batch_environment\")\n",
+        "batch_env.python.conda_dependencies = batch_conda_deps\n",
+        "batch_env.docker.enabled = True\n",
+        "batch_env.docker.base_image = DEFAULT_CPU_IMAGE"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "###  Create the configuration to wrap the inference script"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.contrib.pipeline.steps import ParallelRunStep, ParallelRunConfig\n",
+        "\n",
+        "parallel_run_config = ParallelRunConfig(\n",
+        "    source_directory=scripts_folder,\n",
+        "    entry_script=script_file,\n",
+        "    mini_batch_size=\"5\",\n",
+        "    error_threshold=10,\n",
+        "    output_action=\"append_row\",\n",
+        "    environment=batch_env,\n",
+        "    compute_target=compute_target,\n",
+        "    node_count=2)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Create the pipeline step\n",
+        "Create the pipeline step using the script, environment configuration, and parameters. Specify the compute target you already attached to your workspace as the target of execution of the script. We will use ParallelRunStep to create the pipeline step."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "parallelrun_step = ParallelRunStep(\n",
+        "    name=\"predict-digits-mnist\",\n",
+        "    parallel_run_config=parallel_run_config,\n",
+        "    inputs=[ named_mnist_ds ],\n",
+        "    output=output_dir,\n",
+        "    models=[ model ],\n",
+        "    arguments=[ ],\n",
+        "    allow_reuse=True\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Run the pipeline\n",
+        "At this point you can run the pipeline and examine the output it produced. The Experiment object is used to track the run of the pipeline"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core import Experiment\n",
+        "\n",
+        "pipeline = Pipeline(workspace=ws, steps=[parallelrun_step])\n",
+        "experiment = Experiment(ws, 'digit_identification')\n",
+        "pipeline_run = experiment.submit(pipeline)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Monitor the run"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.widgets import RunDetails\n",
+        "RunDetails(pipeline_run).show()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Optional: View detailed logs (streaming) "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "pipeline_run.wait_for_completion(show_output=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### View the prediction results per input image\n",
+        "In the score.py file above you can see that the ResultList with the filename and the prediction result gets returned. These are written to the DataStore specified in the PipelineData object as the output data, which in this case is called *inferences*. This containers the outputs from  all of the worker nodes used in the compute cluster. You can download this data to view the results ... below just filters to the first 10 rows"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import pandas as pd\n",
+        "import shutil\n",
+        "\n",
+        "# remove previous run results, if present\n",
+        "shutil.rmtree(\"mnist_results\", ignore_errors=True)\n",
+        "\n",
+        "batch_run = next(pipeline_run.get_children())\n",
+        "batch_output = batch_run.get_output_data(\"inferences\")\n",
+        "batch_output.download(local_path=\"mnist_results\")\n",
+        "\n",
+        "for root, dirs, files in os.walk(\"mnist_results\"):\n",
+        "    for file in files:\n",
+        "        if file.endswith('parallel_run_step.txt'):\n",
+        "            result_file = os.path.join(root,file)\n",
+        "\n",
+        "df = pd.read_csv(result_file, delimiter=\":\", header=None)\n",
+        "df.columns = [\"Filename\", \"Prediction\"]\n",
+        "print(\"Prediction has \", df.shape[0], \" rows\")\n",
+        "df.head(10) "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Cleanup Compute resources\n",
+        "\n",
+        "For re-occurring jobs, it may be wise to keep compute the compute resources and allow compute nodes to scale down to 0. However, since this is just a single-run job, we are free to release the allocated compute resources."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# uncomment below and run if compute resources are no longer needed \n",
+        "# compute_target.delete() "
+      ]
+    }
+  ],
+  "metadata": {
+    "authors": [
+      {
+        "name": "joringer"
+      },
+      {
+        "name": "asraniwa"
+      },
+      {
+        "name": "pansav"
+      },
+      {
+        "name": "tracych"
+      }
+    ],
+    "friendly_name": "MNIST data inferencing using ParallelRunStep",
+    "exclude_from_index": false,
+    "index_order": 1,
+    "category": "Other notebooks",
+    "compute": [
+      "AML Compute"
+    ],
+    "datasets": [
+      "MNIST"
+    ],
+    "deployment": [
+      "None"
+    ],
+    "framework": [
+      "None"
+    ],
+    "tags": [
+      "Batch Inferencing",
+      "Pipeline"
+    ],
+    "task": "Digit identification",
+    "kernelspec": {
+      "display_name": "Python 3.6",
+      "language": "python",
+      "name": "python36"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.6.9"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 2
+}

contrib/batch_inferencing/file-dataset-image-inference-mnist.yml

@@ -0,0 +1,6 @@
+name: file-dataset-image-inference-mnist
+dependencies:
+- pip:
+  - azureml-sdk
+  - azureml-contrib-pipeline-steps
+  - azureml-widgets

contrib/batch_inferencing/tabular-dataset-inference-iris.ipynb

@@ -0,0 +1,538 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Copyright (c) Microsoft Corporation. All rights reserved.  \n",
+        "Licensed under the MIT License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/machine-learning-pipelines/contrib/batch_inferencing/tabular-dataset-inference-iris.png)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# Using Azure Machine Learning Pipelines for Batch Inference for CSV Files\n",
+        "\n",
+        "In this notebook, we will demonstrate how to make predictions on large quantities of data asynchronously using the ML pipelines with Azure Machine Learning. Batch inference (or batch scoring) provides cost-effective inference, with unparalleled throughput for asynchronous applications. Batch prediction pipelines can scale to perform inference on terabytes of production data. Batch prediction is optimized for high throughput, fire-and-forget predictions for a large collection of data.\n",
+        "\n",
+        "> **Tip**\n",
+        "If your system requires low-latency processing (to process a single document or small set of documents quickly), use [real-time scoring](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-consume-web-service) instead of batch prediction.\n",
+        "\n",
+        "In this example we will take use a machine learning model already trained to predict different types of iris flowers and run that trained model on some of the data in a CSV file which has characteristics of different iris flowers.  However, the same example can be extended to manipulating data to any embarrassingly-parallel processing through a python script.\n",
+        "\n",
+        "The outline of this notebook is as follows:\n",
+        "\n",
+        "- Create a DataStore referencing the CSV files stored in a blob container.\n",
+        "- Register the pretrained model into the model registry. \n",
+        "- Use the registered model to do batch inference on the CSV files in the data blob container.\n",
+        "\n",
+        "## Prerequisites\n",
+        "If you are using an Azure Machine Learning Notebook VM, you are all set. Otherwise, make sure you go through the configuration Notebook located at https://github.com/Azure/MachineLearningNotebooks first. This sets you up with a working config file that has information on your workspace, subscription id, etc. \n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Connect to workspace\n",
+        "Create a workspace object from the existing workspace. Workspace.from_config() reads the file config.json and loads the details into an object named ws."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core import Workspace\n",
+        "\n",
+        "ws = Workspace.from_config()\n",
+        "print('Workspace name: ' + ws.name, \n",
+        "      'Azure region: ' + ws.location, \n",
+        "      'Subscription id: ' + ws.subscription_id, \n",
+        "      'Resource group: ' + ws.resource_group, sep = '\\n')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Create or Attach existing compute resource\n",
+        "By using Azure Machine Learning Compute, a managed service, data scientists can train machine learning models on clusters of Azure virtual machines. Examples include VMs with GPU support. In this tutorial, you create Azure Machine Learning Compute as your training environment. The code below creates the compute clusters for you if they don't already exist in your workspace.\n",
+        "\n",
+        "**Creation of compute takes approximately 5 minutes. If the AmlCompute with that name is already in your workspace the code will skip the creation process.**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "from azureml.core.compute import AmlCompute, ComputeTarget\n",
+        "from azureml.core.compute_target import ComputeTargetException\n",
+        "\n",
+        "# choose a name for your cluster\n",
+        "compute_name = os.environ.get(\"AML_COMPUTE_CLUSTER_NAME\", \"cpu-cluster\")\n",
+        "compute_min_nodes = os.environ.get(\"AML_COMPUTE_CLUSTER_MIN_NODES\", 0)\n",
+        "compute_max_nodes = os.environ.get(\"AML_COMPUTE_CLUSTER_MAX_NODES\", 4)\n",
+        "\n",
+        "# This example uses CPU VM. For using GPU VM, set SKU to STANDARD_NC6\n",
+        "vm_size = os.environ.get(\"AML_COMPUTE_CLUSTER_SKU\", \"STANDARD_D2_V2\")\n",
+        "\n",
+        "\n",
+        "if compute_name in ws.compute_targets:\n",
+        "    compute_target = ws.compute_targets[compute_name]\n",
+        "    if compute_target and type(compute_target) is AmlCompute:\n",
+        "        print('found compute target. just use it. ' + compute_name)\n",
+        "else:\n",
+        "    print('creating a new compute target...')\n",
+        "    provisioning_config = AmlCompute.provisioning_configuration(vm_size = vm_size,\n",
+        "                                                                min_nodes = compute_min_nodes, \n",
+        "                                                                max_nodes = compute_max_nodes)\n",
+        "\n",
+        "    # create the cluster\n",
+        "    compute_target = ComputeTarget.create(ws, compute_name, provisioning_config)\n",
+        "    \n",
+        "    # can poll for a minimum number of nodes and for a specific timeout.\n",
+        "    # if no min node count is provided it will use the scale settings for the cluster\n",
+        "    compute_target.wait_for_completion(show_output=True, min_node_count=None, timeout_in_minutes=20)\n",
+        "    \n",
+        "     # For a more detailed view of current AmlCompute status, use get_status()\n",
+        "    print(compute_target.get_status().serialize())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Create a datastore containing sample images\n",
+        "The input dataset used for this notebook is CSV data which has attributes of different iris flowers. We have created a public blob container `sampledata` on an account named `pipelinedata`, containing iris data set. In the next step, we create a datastore with the name `iris_datastore`, which points to this container. In the call to `register_azure_blob_container` below, setting the `overwrite` flag to `True` overwrites any datastore that was created previously with that name. \n",
+        "\n",
+        "This step can be changed to point to your blob container by providing your own `datastore_name`, `container_name`, and `account_name`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.datastore import Datastore\n",
+        "\n",
+        "account_name = \"pipelinedata\"\n",
+        "datastore_name=\"iris_datastore_data\"\n",
+        "container_name=\"sampledata\"\n",
+        "\n",
+        "iris_data = Datastore.register_azure_blob_container(ws, \n",
+        "                      datastore_name=datastore_name, \n",
+        "                      container_name= container_name, \n",
+        "                      account_name=account_name, \n",
+        "                      overwrite=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Create a TabularDataset\n",
+        "A [TabularDataSet](https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.data.tabulardataset?view=azure-ml-py) references single or multiple files which contain data in a tabular structure (ie like CSV files) in your datastores or public urls. TabularDatasets provides you with the ability to download or mount the files to your compute. By creating a dataset, you create a reference to the data source location. If you applied any subsetting transformations to the dataset, they will be stored in the dataset as well. The data remains in its existing location, so no extra storage cost is incurred."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.dataset import Dataset\n",
+        "\n",
+        "iris_ds_name = 'iris_data'\n",
+        "\n",
+        "path_on_datastore = iris_data.path('iris/')\n",
+        "input_iris_ds = Dataset.Tabular.from_delimited_files(path=path_on_datastore, validate=False)\n",
+        "registered_iris_ds = input_iris_ds.register(ws, iris_ds_name, create_new_version=True)\n",
+        "named_iris_ds = registered_iris_ds.as_named_input(iris_ds_name)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Intermediate/Output Data\n",
+        "Intermediate data (or output of a Step) is represented by [PipelineData](https://docs.microsoft.com/en-us/python/api/azureml-pipeline-core/azureml.pipeline.core.pipelinedata?view=azure-ml-py) object. PipelineData can be produced by one step and consumed in another step by providing the PipelineData object as an output of one step and the input of one or more steps.\n",
+        "\n",
+        "**Constructing PipelineData**\n",
+        "- name: [Required] Name of the data item within the pipeline graph\n",
+        "- datastore_name: Name of the Datastore to write this output to\n",
+        "- output_name: Name of the output\n",
+        "- output_mode: Specifies \"upload\" or \"mount\" modes for producing output (default: mount)\n",
+        "- output_path_on_compute: For \"upload\" mode, the path to which the module writes this output during execution\n",
+        "- output_overwrite: Flag to overwrite pre-existing data"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.pipeline.core import PipelineData\n",
+        "\n",
+        "datastore = ws.get_default_datastore()\n",
+        "output_folder = PipelineData(name='inferences', datastore=datastore)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Registering the Model with the Workspace\n",
+        "Get the pretrained model from a publicly available Azure Blob container, then register it to use in your workspace"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "model_container_name=\"iris-model\"\n",
+        "model_datastore_name=\"iris_model_datastore\"\n",
+        "\n",
+        "model_datastore = Datastore.register_azure_blob_container(ws, \n",
+        "                      datastore_name=model_datastore_name, \n",
+        "                      container_name= model_container_name, \n",
+        "                      account_name=account_name, \n",
+        "                      overwrite=True)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.model import Model\n",
+        "\n",
+        "model_datastore.download('iris_model.pkl')\n",
+        "\n",
+        "# register downloaded model\n",
+        "model = Model.register(model_path = \"iris_model.pkl/iris_model.pkl\",\n",
+        "                       model_name = \"iris\", # this is the name the model is registered as\n",
+        "                       tags = {'pretrained': \"iris\"},\n",
+        "                       workspace = ws)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Using your model to make batch predictions\n",
+        "To use the model to make batch predictions, you need an **entry script** and a list of **dependencies**:\n",
+        "\n",
+        "#### An entry script\n",
+        "This script accepts requests, scores the requests by using the model, and returns the results.\n",
+        "- __init()__ - Typically this function loads the model into a global object. This function is run only once at the start of batch processing per worker node/process. init method can make use of following environment variables (ParallelRunStep input):\n",
+        "    1.\tAZUREML_BI_OUTPUT_PATH \u00e2\u20ac\u201c output folder path\n",
+        "- __run(mini_batch)__ - The method to be parallelized. Each invocation will have one minibatch.<BR>\n",
+        "__mini_batch__: Batch inference will invoke run method and pass either a list or Pandas DataFrame as an argument to the method. Each entry in min_batch will be - a filepath if input is a FileDataset, a Pandas DataFrame if input is a TabularDataset.<BR>\n",
+        "__run__ method response: run() method should return a Pandas DataFrame or an array. For append_row output_action, these returned elements are appended into the common output file. For summary_only, the contents of the elements are ignored. For all output actions, each returned output element indicates one successful inference of input element in the input mini-batch.\n",
+        "    User should make sure that enough data is included in inference result to map input to inference. Inference output will be written in output file and not guaranteed to be in order, user should use some key in the output to map it to input.\n",
+        "    \n",
+        "\n",
+        "#### Dependencies\n",
+        "Helper scripts or Python/Conda packages required to run the entry script or model.\n",
+        "\n",
+        "The deployment configuration for the compute target that hosts the deployed model. This configuration describes things like memory and CPU requirements needed to run the model.\n",
+        "\n",
+        "These items are encapsulated into an inference configuration and a deployment configuration. The inference configuration references the entry script and other dependencies. You define these configurations programmatically when you use the SDK to perform the deployment. You define them in JSON files when you use the CLI.\n",
+        "\n",
+        "## Print inferencing script"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "scripts_folder = \"Code\"\n",
+        "script_file = \"iris_score.py\"\n",
+        "\n",
+        "# peek at contents\n",
+        "with open(os.path.join(scripts_folder, script_file)) as inference_file:\n",
+        "    print(inference_file.read())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Build and run the batch inference pipeline\n",
+        "The data, models, and compute resource are now available. Let's put all these together in a pipeline."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "###  Specify the environment to run the script\n",
+        "Specify the conda dependencies for your script. This will allow us to install pip packages as well as configure the inference environment."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core import Environment\n",
+        "from azureml.core.runconfig import CondaDependencies\n",
+        "\n",
+        "predict_conda_deps = CondaDependencies.create(pip_packages=[ \"scikit-learn==0.20.3\" ])\n",
+        "\n",
+        "predict_env = Environment(name=\"predict_environment\")\n",
+        "predict_env.python.conda_dependencies = predict_conda_deps\n",
+        "predict_env.docker.enabled = True\n",
+        "predict_env.spark.precache_packages = False"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "###  Create the configuration to wrap the inference script"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.contrib.pipeline.steps import ParallelRunStep, ParallelRunConfig\n",
+        "\n",
+        "# In a real-world scenario, you'll want to shape your process per node and nodes to fit your problem domain.\n",
+        "parallel_run_config = ParallelRunConfig(\n",
+        "                    source_directory=scripts_folder,\n",
+        "                    entry_script=script_file,  # the user script to run against each input\n",
+        "                    mini_batch_size='5MB',\n",
+        "                    error_threshold=5,\n",
+        "                    output_action='append_row',\n",
+        "                    environment=predict_env,\n",
+        "                    compute_target=compute_target, \n",
+        "                    node_count=3,\n",
+        "                    run_invocation_timeout=600)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Create the pipeline step\n",
+        "Create the pipeline step using the script, environment configuration, and parameters. Specify the compute target you already attached to your workspace as the target of execution of the script. We will use ParallelRunStep to create the pipeline step."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "distributed_csv_iris_step = ParallelRunStep(\n",
+        "    name='example-iris',\n",
+        "    inputs=[named_iris_ds],\n",
+        "    output=output_folder,\n",
+        "    parallel_run_config=parallel_run_config,\n",
+        "    models=[model],\n",
+        "    arguments=['--model_name', 'iris'],\n",
+        "    allow_reuse=True\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Run the pipeline\n",
+        "At this point you can run the pipeline and examine the output it produced. The Experiment object is used to track the run of the pipeline"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core import Experiment\n",
+        "from azureml.pipeline.core import Pipeline\n",
+        "\n",
+        "pipeline = Pipeline(workspace=ws, steps=[distributed_csv_iris_step])\n",
+        "\n",
+        "pipeline_run = Experiment(ws, 'iris').submit(pipeline)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# this will output a table with link to the run details in azure portal\n",
+        "pipeline_run"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## View progress of Pipeline run\n",
+        "\n",
+        "The progress of the pipeline is able to be viewed either through azureml.widgets or a console feed from PipelineRun.wait_for_completion()."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# GUI\n",
+        "from azureml.widgets import RunDetails\n",
+        "RunDetails(pipeline_run).show() "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Console logs\n",
+        "pipeline_run.wait_for_completion(show_output=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## View Results\n",
+        "In the iris_score.py file above you can see that the Result with the prediction of the iris variety gets returned and then appended to the original input of the row from the csv file. These results are written to the DataStore specified in the PipelineData object as the output data, which in this case is called *inferences*. This contains the outputs from  all of the worker nodes used in the compute cluster. You can download this data to view the results ... below just filters to a random 20 rows"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import pandas as pd\n",
+        "import shutil\n",
+        "\n",
+        "shutil.rmtree(\"iris_results\", ignore_errors=True)\n",
+        "\n",
+        "prediction_run = next(pipeline_run.get_children())\n",
+        "prediction_output = prediction_run.get_output_data(\"inferences\")\n",
+        "prediction_output.download(local_path=\"iris_results\")\n",
+        "\n",
+        "\n",
+        "for root, dirs, files in os.walk(\"iris_results\"):\n",
+        "    for file in files:\n",
+        "        if file.endswith('parallel_run_step.txt'):\n",
+        "            result_file = os.path.join(root,file)\n",
+        "\n",
+        "# cleanup output format\n",
+        "df = pd.read_csv(result_file, delimiter=\" \", header=None)\n",
+        "df.columns = [\"sepal.length\", \"sepal.width\", \"petal.length\", \"petal.width\", \"variety\"]\n",
+        "print(\"Prediction has \", df.shape[0], \" rows\")\n",
+        "\n",
+        "random_subset = df.sample(n=20)\n",
+        "random_subset.head(20)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Cleanup compute resources\n",
+        "For re-occurring jobs, it may be wise to keep compute the compute resources and allow compute nodes to scale down to 0. However, since this is just a single run job, we are free to release the allocated compute resources."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# uncomment below and run if compute resources are no longer needed \n",
+        "# compute_target.delete()"
+      ]
+    }
+  ],
+  "metadata": {
+    "authors": [
+      {
+        "name": "joringer"
+      },
+      {
+        "name": "asraniwa"
+      },
+      {
+        "name": "pansav"
+      },
+      {
+        "name": "tracych"
+      }
+    ],
+    "friendly_name": "IRIS data inferencing using ParallelRunStep",
+    "exclude_from_index": false,
+    "index_order": 1,
+    "category": "Other notebooks",
+    "compute": [
+      "AML Compute"
+    ],
+    "datasets": [
+      "IRIS"
+    ],
+    "deployment": [
+      "None"
+    ],
+    "framework": [
+      "None"
+    ],
+    "tags": [
+      "Batch Inferencing",
+      "Pipeline"
+    ],
+    "task": "Recognize flower type",
+    "kernelspec": {
+      "display_name": "Python 3.6",
+      "language": "python",
+      "name": "python36"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.6.2"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 2
+}

contrib/batch_inferencing/tabular-dataset-inference-iris.yml

@@ -0,0 +1,6 @@
+name: tabular-dataset-inference-iris
+dependencies:
+- pip:
+  - azureml-sdk
+  - azureml-contrib-pipeline-steps
+  - azureml-widgets

contrib/gbdt/lightgbm/binary0.test

@@ -0,0 +1,500 @@
+1	0.644	0.247	-0.447	0.862	0.374	0.854	-1.126	-0.790	2.173	1.015	-0.201	1.400	0.000	1.575	1.807	1.607	0.000	1.585	-0.190	-0.744	3.102	0.958	1.061	0.980	0.875	0.581	0.905	0.796
+0	0.385	1.800	1.037	1.044	0.349	1.502	-0.966	1.734	0.000	0.966	-1.960	-0.249	0.000	1.501	0.465	-0.354	2.548	0.834	-0.440	0.638	3.102	0.695	0.909	0.981	0.803	0.813	1.149	1.116
+0	1.214	-0.166	0.004	0.505	1.434	0.628	-1.174	-1.230	1.087	0.579	-1.047	-0.118	0.000	0.835	0.340	1.234	2.548	0.711	-1.383	1.355	0.000	0.848	0.911	1.043	0.931	1.058	0.744	0.696
+1	0.420	1.111	0.137	1.516	-1.657	0.854	0.623	1.605	1.087	1.511	-1.297	0.251	0.000	0.872	-0.368	-0.721	0.000	0.543	0.731	1.424	3.102	1.597	1.282	1.105	0.730	0.148	1.231	1.234
+0	0.897	-1.703	-1.306	1.022	-0.729	0.836	0.859	-0.333	2.173	1.336	-0.965	0.972	2.215	0.671	1.021	-1.439	0.000	0.493	-2.019	-0.289	0.000	0.805	0.930	0.984	1.430	2.198	1.934	1.684
+0	0.756	1.126	-0.945	2.355	-0.555	0.889	0.800	1.440	0.000	0.585	0.271	0.631	2.215	0.722	1.744	1.051	0.000	0.618	0.924	0.698	1.551	0.976	0.864	0.988	0.803	0.234	0.822	0.911
+0	1.141	-0.741	0.953	1.478	-0.524	1.197	-0.871	1.689	2.173	0.875	1.321	-0.518	1.107	0.540	0.037	-0.987	0.000	0.879	1.187	0.245	0.000	0.888	0.701	1.747	1.358	2.479	1.491	1.223
+1	0.606	-0.936	-0.384	1.257	-1.162	2.719	-0.600	0.100	2.173	3.303	-0.284	1.561	1.107	0.689	1.786	-0.326	0.000	0.780	-0.532	1.216	0.000	0.936	2.022	0.985	1.574	4.323	2.263	1.742
+1	0.603	0.429	-0.279	1.448	1.301	1.008	2.423	-1.295	0.000	0.452	1.305	0.533	0.000	1.076	1.011	1.256	2.548	2.021	1.260	-0.343	0.000	0.890	0.969	1.281	0.763	0.652	0.827	0.785
+0	1.171	-0.962	0.521	0.841	-0.315	1.196	-0.744	-0.882	2.173	0.726	-1.305	1.377	1.107	0.643	-1.790	-1.264	0.000	1.257	0.222	0.817	0.000	0.862	0.911	0.987	0.846	1.293	0.899	0.756
+1	1.392	-0.358	0.235	1.494	-0.461	0.895	-0.848	1.549	2.173	0.841	-0.384	0.666	1.107	1.199	2.509	-0.891	0.000	1.109	-0.364	-0.945	0.000	0.693	2.135	1.170	1.362	0.959	2.056	1.842
+1	1.024	1.076	-0.886	0.851	1.530	0.673	-0.449	0.187	1.087	0.628	-0.895	1.176	2.215	0.696	-0.232	-0.875	0.000	0.411	1.501	0.048	0.000	0.842	0.919	1.063	1.193	0.777	0.964	0.807
+1	0.890	-0.760	1.182	1.369	0.751	0.696	-0.959	-0.710	1.087	0.775	-0.130	-1.409	2.215	0.701	-0.110	-0.739	0.000	0.508	-0.451	0.390	0.000	0.762	0.738	0.998	1.126	0.788	0.940	0.790
+1	0.460	0.537	0.636	1.442	-0.269	0.585	0.323	-1.731	2.173	0.503	1.034	-0.927	0.000	0.928	-1.024	1.006	2.548	0.513	-0.618	-1.336	0.000	0.802	0.831	0.992	1.019	0.925	1.056	0.833
+1	0.364	1.648	0.560	1.720	0.829	1.110	0.811	-0.588	0.000	0.408	1.045	1.054	2.215	0.319	-1.138	1.545	0.000	0.423	1.025	-1.265	3.102	1.656	0.928	1.003	0.544	0.327	0.670	0.746
+1	0.525	-0.096	1.206	0.948	-1.103	1.519	-0.582	0.606	2.173	1.274	-0.572	-0.934	0.000	0.855	-1.028	-1.222	0.000	0.578	-1.000	-1.725	3.102	0.896	0.878	0.981	0.498	0.909	0.772	0.668
+0	0.536	-0.821	-1.029	0.703	1.113	0.363	-0.711	0.022	1.087	0.325	1.503	1.249	2.215	0.673	1.041	-0.401	0.000	0.480	2.127	1.681	0.000	0.767	1.034	0.990	0.671	0.836	0.669	0.663
+1	1.789	-0.583	1.641	0.897	0.799	0.515	-0.100	-1.483	0.000	1.101	0.031	-0.326	2.215	1.195	0.001	0.126	2.548	0.768	-0.148	0.601	0.000	0.916	0.921	1.207	1.069	0.483	0.934	0.795
+1	1.332	-0.571	0.986	0.580	1.508	0.582	0.634	-0.746	1.087	1.084	-0.964	-0.489	0.000	0.785	0.274	0.343	2.548	0.779	0.721	1.489	0.000	1.733	1.145	0.990	1.270	0.715	0.897	0.915
+0	1.123	0.629	-1.708	0.597	-0.882	0.752	0.195	1.522	2.173	1.671	1.515	-0.003	0.000	0.778	0.514	0.139	1.274	0.801	1.260	1.600	0.000	1.495	0.976	0.988	0.676	0.921	1.010	0.943
+0	1.816	-0.515	0.171	0.980	-0.454	0.870	0.202	-1.399	2.173	1.130	1.066	-1.593	0.000	0.844	0.735	1.275	2.548	1.125	-1.133	0.348	0.000	0.837	0.693	0.988	1.112	0.784	1.009	0.974
+1	0.364	0.694	0.445	1.862	0.159	0.963	-1.356	1.260	1.087	0.887	-0.540	-1.533	2.215	0.658	-2.544	-1.236	0.000	0.516	-0.807	0.039	0.000	0.891	1.004	0.991	1.092	0.976	1.000	0.953
+1	0.790	-1.175	0.475	1.846	0.094	0.999	-1.090	0.257	0.000	1.422	0.854	1.112	2.215	1.302	1.004	-1.702	1.274	2.557	-0.787	-1.048	0.000	0.890	1.429	0.993	2.807	0.840	2.248	1.821
+1	0.765	-0.500	-0.603	1.843	-0.560	1.068	0.007	0.746	2.173	1.154	-0.017	1.329	0.000	1.165	1.791	-1.585	0.000	1.116	0.441	-0.886	0.000	0.774	0.982	0.989	1.102	0.633	1.178	1.021
+1	1.407	1.293	-1.418	0.502	-1.527	2.005	-2.122	0.622	0.000	1.699	1.508	-0.649	2.215	1.665	0.748	-0.755	0.000	2.555	0.811	1.423	1.551	7.531	5.520	0.985	1.115	1.881	4.487	3.379
+1	0.772	-0.186	-1.372	0.823	-0.140	0.781	0.763	0.046	2.173	1.128	0.516	1.380	0.000	0.797	-0.640	-0.134	2.548	2.019	-0.972	-1.670	0.000	2.022	1.466	0.989	0.856	0.808	1.230	0.991
+1	0.546	-0.954	0.715	1.335	-1.689	0.783	-0.443	-1.735	2.173	1.081	0.185	-0.435	0.000	1.433	-0.662	-0.389	0.000	0.969	0.924	1.099	0.000	0.910	0.879	0.988	0.683	0.753	0.878	0.865
+1	0.596	0.276	-1.054	1.358	1.355	1.444	1.813	-0.208	0.000	1.175	-0.949	-1.573	0.000	0.855	-1.228	-0.925	2.548	1.837	-0.400	0.913	0.000	0.637	0.901	1.028	0.553	0.790	0.679	0.677
+0	0.458	2.292	1.530	0.291	1.283	0.749	-0.930	-0.198	0.000	0.300	-1.560	0.990	0.000	0.811	-0.176	0.995	2.548	1.085	-0.178	-1.213	3.102	0.891	0.648	0.999	0.732	0.655	0.619	0.620
+0	0.638	-0.575	-1.048	0.125	0.178	0.846	-0.753	-0.339	1.087	0.799	-0.727	1.182	0.000	0.888	0.283	0.717	0.000	1.051	-1.046	-1.557	3.102	0.889	0.871	0.989	0.884	0.923	0.836	0.779
+1	0.434	-1.119	-0.313	2.427	0.461	0.497	0.261	-1.177	2.173	0.618	-0.737	-0.688	0.000	1.150	-1.237	-1.652	2.548	0.757	-0.054	1.700	0.000	0.809	0.741	0.982	1.450	0.936	1.086	0.910
+1	0.431	-1.144	-1.030	0.778	-0.655	0.490	0.047	-1.546	0.000	1.583	-0.014	0.891	2.215	0.516	0.956	0.567	2.548	0.935	-1.123	-0.082	0.000	0.707	0.995	0.995	0.700	0.602	0.770	0.685
+1	1.894	0.222	1.224	1.578	1.715	0.966	2.890	-0.013	0.000	0.922	-0.703	-0.844	0.000	0.691	2.056	1.039	0.000	0.900	-0.733	-1.240	3.102	1.292	1.992	1.026	0.881	0.684	1.759	1.755
+0	0.985	-0.316	0.141	1.067	-0.946	0.819	-1.177	1.307	2.173	1.080	-0.429	0.557	1.107	1.726	1.435	-1.075	0.000	1.100	1.547	-0.647	0.000	0.873	1.696	1.179	1.146	1.015	1.538	1.270
+0	0.998	-0.187	-0.236	0.882	0.755	0.468	0.950	-0.439	2.173	0.579	-0.550	-0.624	0.000	1.847	1.196	1.384	1.274	0.846	1.273	-1.072	0.000	1.194	0.797	1.013	1.319	1.174	0.963	0.898
+0	0.515	0.246	-0.593	1.082	1.591	0.912	-0.623	-0.957	2.173	0.858	0.418	0.844	0.000	0.948	2.519	1.599	0.000	1.158	1.385	-0.095	3.102	0.973	1.033	0.988	0.998	1.716	1.054	0.901
+0	0.919	-1.001	1.506	1.389	0.653	0.507	-0.616	-0.689	2.173	0.808	0.536	-0.467	2.215	0.496	2.187	-0.859	0.000	0.822	0.807	1.163	0.000	0.876	0.861	1.088	0.947	0.614	0.911	1.087
+0	0.794	0.051	1.477	1.504	-1.695	0.716	0.315	0.264	1.087	0.879	-0.135	-1.094	2.215	1.433	-0.741	0.201	0.000	1.566	0.534	-0.989	0.000	0.627	0.882	0.974	0.807	1.130	0.929	0.925
+1	0.455	-0.946	-1.175	1.453	-0.580	0.763	-0.856	0.840	0.000	0.829	1.223	1.174	2.215	0.714	0.638	-0.466	0.000	1.182	0.223	-1.333	0.000	0.977	0.938	0.986	0.713	0.714	0.796	0.843
+1	0.662	-0.296	-1.287	1.212	-0.707	0.641	1.457	0.222	0.000	0.600	0.525	-1.700	2.215	0.784	-0.835	-0.961	2.548	0.865	1.131	1.162	0.000	0.854	0.877	0.978	0.740	0.734	0.888	0.811
+0	0.390	0.698	-1.629	1.888	0.298	0.990	1.614	-1.572	0.000	1.666	0.170	0.719	2.215	1.590	1.064	-0.886	1.274	0.952	0.305	-1.216	0.000	1.048	0.897	1.173	0.891	1.936	1.273	1.102
+0	1.014	0.117	1.384	0.686	-1.047	0.609	-1.245	-0.850	0.000	1.076	-1.158	0.814	1.107	1.598	-0.389	-0.111	0.000	0.907	1.688	-1.673	0.000	1.333	0.866	0.989	0.975	0.442	0.797	0.788
+0	1.530	-1.408	-0.207	0.440	-1.357	0.902	-0.647	1.325	1.087	1.320	-0.819	0.246	1.107	0.503	1.407	-1.683	0.000	1.189	-0.972	-0.925	0.000	0.386	1.273	0.988	0.829	1.335	1.173	1.149
+1	1.689	-0.590	0.915	2.076	1.202	0.644	-0.478	-0.238	0.000	0.809	-1.660	-1.184	0.000	1.227	-0.224	-0.808	2.548	1.655	1.047	-0.623	0.000	0.621	1.192	0.988	1.309	0.866	0.924	1.012
+0	1.102	0.402	-1.622	1.262	1.022	0.576	0.271	-0.269	0.000	0.591	0.495	-1.278	0.000	1.271	0.209	0.575	2.548	0.941	0.964	-0.685	3.102	0.989	0.963	1.124	0.857	0.858	0.716	0.718
+0	2.491	0.825	0.581	1.593	0.205	0.782	-0.815	1.499	0.000	1.179	-0.999	-1.509	0.000	0.926	0.920	-0.522	2.548	2.068	-1.021	-1.050	3.102	0.874	0.943	0.980	0.945	1.525	1.570	1.652
+0	0.666	0.254	1.601	1.303	-0.250	1.236	-1.929	0.793	0.000	1.074	0.447	-0.871	0.000	0.991	1.059	-0.342	0.000	1.703	-0.393	-1.419	3.102	0.921	0.945	1.285	0.931	0.462	0.770	0.729
+0	0.937	-1.126	1.424	1.395	1.743	0.760	0.428	-0.238	2.173	0.846	0.494	1.320	2.215	0.872	-1.826	-0.507	0.000	0.612	1.860	1.403	0.000	3.402	2.109	0.985	1.298	1.165	1.404	1.240
+1	0.881	-1.086	-0.870	0.513	0.266	2.049	-1.870	1.160	0.000	2.259	-0.428	-0.935	2.215	1.321	-0.655	-0.449	2.548	1.350	-1.766	-0.108	0.000	0.911	1.852	0.987	1.167	0.820	1.903	1.443
+0	0.410	0.835	-0.819	1.257	1.112	0.871	-1.737	-0.401	0.000	0.927	0.158	1.253	0.000	1.183	0.405	-1.570	0.000	0.807	-0.704	-0.438	3.102	0.932	0.962	0.987	0.653	0.315	0.616	0.648
+1	0.634	0.196	-1.679	1.379	-0.967	2.260	-0.273	1.114	0.000	1.458	1.070	-0.278	1.107	1.195	0.110	-0.688	2.548	0.907	0.298	-1.359	0.000	0.949	1.129	0.984	0.675	0.877	0.938	0.824
+1	0.632	-1.254	1.201	0.496	-0.106	0.235	2.731	-0.955	0.000	0.615	-0.805	0.600	0.000	0.633	-0.934	1.641	0.000	1.407	-0.483	-0.962	1.551	0.778	0.797	0.989	0.578	0.722	0.576	0.539
+0	0.714	1.122	1.566	2.399	-1.431	1.665	0.299	0.323	0.000	1.489	1.087	-0.861	2.215	1.174	0.140	1.083	2.548	0.404	-0.968	1.105	0.000	0.867	0.969	0.981	1.039	1.552	1.157	1.173
+1	0.477	-0.321	-0.471	1.966	1.034	2.282	1.359	-0.874	0.000	1.672	-0.258	1.109	0.000	1.537	0.604	0.231	2.548	1.534	-0.640	0.827	0.000	0.746	1.337	1.311	0.653	0.721	0.795	0.742
+1	1.351	0.460	0.031	1.194	-1.185	0.670	-1.157	-1.637	2.173	0.599	-0.823	0.680	0.000	0.478	0.373	1.716	0.000	0.809	-0.919	0.010	1.551	0.859	0.839	1.564	0.994	0.777	0.971	0.826
+1	0.520	-1.442	-0.348	0.840	1.654	1.273	-0.760	1.317	0.000	0.861	2.579	-0.791	0.000	1.779	0.257	-0.703	0.000	2.154	1.928	0.457	0.000	1.629	3.194	0.992	0.730	1.107	2.447	2.747
+0	0.700	-0.308	0.920	0.438	-0.879	0.516	1.409	1.101	0.000	0.960	0.701	-0.049	2.215	1.442	-0.416	-1.439	2.548	0.628	1.009	-0.364	0.000	0.848	0.817	0.987	0.759	1.421	0.937	0.920
+1	0.720	1.061	-0.546	0.798	-1.521	1.066	0.173	0.271	1.087	1.453	0.114	1.336	1.107	0.702	0.616	-0.367	0.000	0.543	-0.386	-1.301	0.000	0.653	0.948	0.989	1.031	1.500	0.965	0.790
+1	0.735	-0.416	0.588	1.308	-0.382	1.042	0.344	1.609	0.000	0.926	0.163	-0.520	1.107	1.050	-0.427	1.159	2.548	0.834	0.613	0.948	0.000	0.848	1.189	1.042	0.844	1.099	0.829	0.843
+1	0.777	-0.396	1.540	1.608	0.638	0.955	0.040	0.918	2.173	1.315	1.116	-0.823	0.000	0.781	-0.762	0.564	2.548	0.945	-0.573	1.379	0.000	0.679	0.706	1.124	0.608	0.593	0.515	0.493
+1	0.934	0.319	-0.257	0.970	-0.980	0.726	0.774	0.731	0.000	0.896	0.038	-1.465	1.107	0.773	-0.055	-0.831	2.548	1.439	-0.229	0.698	0.000	0.964	1.031	0.995	0.845	0.480	0.810	0.762
+0	0.461	0.771	0.019	2.055	-1.288	1.043	0.147	0.261	2.173	0.833	-0.156	1.425	0.000	0.832	0.805	-0.491	2.548	0.589	1.252	1.414	0.000	0.850	0.906	1.245	1.364	0.850	0.908	0.863
+1	0.858	-0.116	-0.937	0.966	1.167	0.825	-0.108	1.111	1.087	0.733	1.163	-0.634	0.000	0.894	0.771	0.020	0.000	0.846	-1.124	-1.195	3.102	0.724	1.194	1.195	0.813	0.969	0.985	0.856
+0	0.720	-0.335	-0.307	1.445	0.540	1.108	-0.034	-1.691	1.087	0.883	-1.356	-0.678	2.215	0.440	1.093	0.253	0.000	0.389	-1.582	-1.097	0.000	1.113	1.034	0.988	1.256	1.572	1.062	0.904
+1	0.750	-0.811	-0.542	0.985	0.408	0.471	0.477	0.355	0.000	1.347	-0.875	-1.556	2.215	0.564	1.082	-0.724	0.000	0.793	-0.958	-0.020	3.102	0.836	0.825	0.986	1.066	0.924	0.927	0.883
+0	0.392	-0.468	-0.216	0.680	1.565	1.086	-0.765	-0.581	1.087	1.264	-1.035	1.189	2.215	0.986	-0.338	0.747	0.000	0.884	-1.328	-0.965	0.000	1.228	0.988	0.982	1.135	1.741	1.108	0.956
+1	0.434	-1.269	0.643	0.713	0.608	0.597	0.832	1.627	0.000	0.708	-0.422	0.079	2.215	1.533	-0.823	-1.127	2.548	0.408	-1.357	-0.828	0.000	1.331	1.087	0.999	1.075	1.015	0.875	0.809
+0	0.828	-1.803	0.342	0.847	-0.162	1.585	-1.128	-0.272	2.173	1.974	0.039	-1.717	0.000	0.900	0.764	-1.741	0.000	1.349	-0.079	1.035	3.102	0.984	0.815	0.985	0.780	1.661	1.403	1.184
+1	1.089	-0.350	-0.747	1.472	0.792	1.087	-0.069	-1.192	0.000	0.512	-0.841	-1.284	0.000	2.162	-0.821	0.545	2.548	1.360	2.243	-0.183	0.000	0.977	0.628	1.725	1.168	0.635	0.823	0.822
+1	0.444	0.451	-1.332	1.176	-0.247	0.898	0.194	0.007	0.000	1.958	0.576	-1.618	2.215	0.584	1.203	0.268	0.000	0.939	1.033	1.264	3.102	0.829	0.886	0.985	1.265	0.751	1.032	0.948
+0	0.629	0.114	1.177	0.917	-1.204	0.845	0.828	-0.088	0.000	0.962	-1.302	0.823	2.215	0.732	0.358	-1.334	2.548	0.538	0.582	1.561	0.000	1.028	0.834	0.988	0.904	1.205	1.039	0.885
+1	1.754	-1.259	-0.573	0.959	-1.483	0.358	0.448	-1.452	0.000	0.711	0.313	0.499	2.215	1.482	-0.390	1.474	2.548	1.879	-1.540	0.668	0.000	0.843	0.825	1.313	1.315	0.939	1.048	0.871
+1	0.549	0.706	-1.437	0.894	0.891	0.680	-0.762	-1.568	0.000	0.981	0.499	-0.425	2.215	1.332	0.678	0.485	1.274	0.803	0.022	-0.893	0.000	0.793	1.043	0.987	0.761	0.899	0.915	0.794
+0	0.475	0.542	-0.987	1.569	0.069	0.551	1.543	-1.488	0.000	0.608	0.301	1.734	2.215	0.277	0.499	-0.522	0.000	1.375	1.212	0.696	3.102	0.652	0.756	0.987	0.828	0.830	0.715	0.679
+1	0.723	0.049	-1.153	1.300	0.083	0.723	-0.749	0.630	0.000	1.126	0.412	-0.384	0.000	1.272	1.256	1.358	2.548	3.108	0.777	-1.486	3.102	0.733	1.096	1.206	1.269	0.899	1.015	0.903
+1	1.062	0.296	0.725	0.285	-0.531	0.819	1.277	-0.667	0.000	0.687	0.829	-0.092	0.000	1.158	0.447	1.047	2.548	1.444	-0.186	-1.491	3.102	0.863	1.171	0.986	0.769	0.828	0.919	0.840
+0	0.572	-0.349	1.396	2.023	0.795	0.577	0.457	-0.533	0.000	1.351	0.701	-1.091	0.000	0.724	-1.012	-0.182	2.548	0.923	-0.012	0.789	3.102	0.936	1.025	0.985	1.002	0.600	0.828	0.909
+1	0.563	0.387	0.412	0.553	1.050	0.723	-0.992	-0.447	0.000	0.748	0.948	0.546	2.215	1.761	-0.559	-1.183	0.000	1.114	-0.251	1.192	3.102	0.936	0.912	0.976	0.578	0.722	0.829	0.892
+1	1.632	1.577	-0.697	0.708	-1.263	0.863	0.012	1.197	2.173	0.498	0.990	-0.806	0.000	0.627	2.387	-1.283	0.000	0.607	1.290	-0.174	3.102	0.916	1.328	0.986	0.557	0.971	0.935	0.836
+1	0.562	-0.360	0.399	0.803	-1.334	1.443	-0.116	1.628	2.173	0.750	0.987	0.135	1.107	0.795	0.298	-0.556	0.000	1.150	-0.113	-0.093	0.000	0.493	1.332	0.985	1.001	1.750	1.013	0.886
+1	0.987	0.706	-0.492	0.861	0.607	0.593	0.088	-0.184	0.000	0.802	0.894	1.608	2.215	0.782	-0.471	1.500	2.548	0.521	0.772	-0.960	0.000	0.658	0.893	1.068	0.877	0.664	0.709	0.661
+1	1.052	0.883	-0.581	1.566	0.860	0.931	1.515	-0.873	0.000	0.493	0.145	-0.672	0.000	1.133	0.935	1.581	2.548	1.630	0.695	0.923	3.102	1.105	1.087	1.713	0.948	0.590	0.872	0.883
+1	2.130	-0.516	-0.291	0.776	-1.230	0.689	-0.257	0.800	2.173	0.730	-0.274	-1.437	0.000	0.615	0.241	1.083	0.000	0.834	0.757	1.613	3.102	0.836	0.806	1.333	1.061	0.730	0.889	0.783
+1	0.742	0.797	1.628	0.311	-0.418	0.620	0.685	-1.457	0.000	0.683	1.774	-1.082	0.000	1.700	1.104	0.225	2.548	0.382	-2.184	-1.307	0.000	0.945	1.228	0.984	0.864	0.931	0.988	0.838
+0	0.311	-1.249	-0.927	1.272	-1.262	0.642	-1.228	-0.136	0.000	1.220	-0.804	-1.558	2.215	0.950	-0.828	0.495	1.274	2.149	-1.672	0.634	0.000	1.346	0.887	0.981	0.856	1.101	1.001	1.106
+0	0.660	-1.834	-0.667	0.601	1.236	0.932	-0.933	-0.135	2.173	1.373	-0.122	1.429	0.000	0.654	-0.034	-0.847	2.548	0.711	0.911	0.703	0.000	1.144	0.942	0.984	0.822	0.739	0.992	0.895
+0	3.609	-0.590	0.851	0.615	0.455	1.280	0.003	-0.866	1.087	1.334	0.708	-1.131	0.000	0.669	0.480	0.092	0.000	0.975	0.983	-1.429	3.102	1.301	1.089	0.987	1.476	0.934	1.469	1.352
+1	0.905	-0.403	1.567	2.651	0.953	1.194	-0.241	-0.567	1.087	0.308	-0.384	-0.007	0.000	0.608	-0.175	-1.163	2.548	0.379	0.941	1.662	0.000	0.580	0.721	1.126	0.895	0.544	1.097	0.836
+1	0.983	0.255	1.093	0.905	-0.874	0.863	0.060	-0.368	0.000	0.824	-0.747	-0.633	0.000	0.614	0.961	1.052	0.000	0.792	-0.260	1.632	3.102	0.874	0.883	1.280	0.663	0.406	0.592	0.645
+1	1.160	-1.027	0.274	0.460	0.322	2.085	-1.623	-0.840	0.000	1.634	-1.046	1.182	2.215	0.492	-0.367	1.174	0.000	0.824	-0.998	1.617	0.000	0.943	0.884	1.001	1.209	1.313	1.034	0.866
+0	0.299	0.028	-1.372	1.930	-0.661	0.840	-0.979	0.664	1.087	0.535	-2.041	1.434	0.000	1.087	-1.797	0.344	0.000	0.485	-0.560	-1.105	3.102	0.951	0.890	0.980	0.483	0.684	0.730	0.706
+0	0.293	1.737	-1.418	2.074	0.794	0.679	1.024	-1.457	0.000	1.034	1.094	-0.168	1.107	0.506	1.680	-0.661	0.000	0.523	-0.042	-1.274	3.102	0.820	0.944	0.987	0.842	0.694	0.761	0.750
+0	0.457	-0.393	1.560	0.738	-0.007	0.475	-0.230	0.246	0.000	0.776	-1.264	-0.606	2.215	0.865	-0.731	-1.576	2.548	1.153	0.343	1.436	0.000	1.060	0.883	0.988	0.972	0.703	0.758	0.720
+0	0.935	-0.582	0.240	2.401	0.818	1.231	-0.618	-1.289	0.000	0.799	0.544	-0.228	2.215	0.525	-1.494	-0.969	0.000	0.609	-1.123	1.168	3.102	0.871	0.767	1.035	1.154	0.919	0.868	1.006
+1	0.902	-0.745	-1.215	1.174	-0.501	1.215	0.167	1.162	0.000	0.896	1.217	-0.976	0.000	0.585	-0.429	1.036	0.000	1.431	-0.416	0.151	3.102	0.524	0.952	0.990	0.707	0.271	0.592	0.826
+1	0.653	0.337	-0.320	1.118	-0.934	1.050	0.745	0.529	1.087	1.075	1.742	-1.538	0.000	0.585	1.090	0.973	0.000	1.091	-0.187	1.160	1.551	1.006	1.108	0.978	1.121	0.838	0.947	0.908
+0	1.157	1.401	0.340	0.395	-1.218	0.945	1.928	-0.876	0.000	1.384	0.320	1.002	1.107	1.900	1.177	-0.462	2.548	1.122	1.316	1.720	0.000	1.167	1.096	0.989	0.937	1.879	1.307	1.041
+0	0.960	0.355	-0.152	0.872	-0.338	0.391	0.348	0.956	1.087	0.469	2.664	1.409	0.000	0.756	-1.561	1.500	0.000	0.525	1.436	1.728	3.102	1.032	0.946	0.996	0.929	0.470	0.698	0.898
+1	1.038	0.274	0.825	1.198	0.963	1.078	-0.496	-1.014	2.173	0.739	-0.727	-0.151	2.215	1.035	-0.799	0.398	0.000	1.333	-0.872	-1.498	0.000	0.849	1.033	0.985	0.886	0.936	0.975	0.823
+0	0.490	0.277	0.318	1.303	0.694	1.333	-1.620	-0.563	0.000	1.459	-1.326	1.140	0.000	0.779	-0.673	-1.324	2.548	0.860	-1.247	0.043	0.000	0.857	0.932	0.992	0.792	0.278	0.841	1.498
+0	1.648	-0.688	-1.386	2.790	0.995	1.087	1.359	-0.687	0.000	1.050	-0.223	-0.261	2.215	0.613	-0.889	1.335	0.000	1.204	0.827	0.309	3.102	0.464	0.973	2.493	1.737	0.827	1.319	1.062
+0	1.510	-0.662	1.668	0.860	0.280	0.705	0.974	-1.647	1.087	0.662	-0.393	-0.225	0.000	0.610	-0.996	0.532	2.548	0.464	1.305	0.102	0.000	0.859	1.057	1.498	0.799	1.260	0.946	0.863
+1	0.850	-1.185	-0.117	0.943	-0.449	1.142	0.875	-0.030	0.000	2.223	-0.461	1.627	2.215	0.767	-1.761	-1.692	0.000	1.012	-0.727	0.639	3.102	3.649	2.062	0.985	1.478	1.087	1.659	1.358
+0	0.933	1.259	0.130	0.326	-0.890	0.306	1.136	1.142	0.000	0.964	0.705	-1.373	2.215	0.546	-0.196	-0.001	0.000	0.578	-1.169	1.004	3.102	0.830	0.836	0.988	0.837	1.031	0.749	0.655
+0	0.471	0.697	1.570	1.109	0.201	1.248	0.348	-1.448	0.000	2.103	0.773	0.686	2.215	1.451	-0.087	-0.453	2.548	1.197	-0.045	-1.026	0.000	0.793	1.094	0.987	0.851	1.804	1.378	1.089
+1	2.446	-0.701	-1.568	0.059	0.822	1.401	-0.600	-0.044	2.173	0.324	-0.001	1.344	2.215	0.913	-0.818	1.049	0.000	0.442	-1.088	-0.005	0.000	0.611	1.062	0.979	0.562	0.988	0.998	0.806
+0	0.619	2.029	0.933	0.528	-0.903	0.974	0.760	-0.311	2.173	0.825	0.658	-1.466	1.107	0.894	1.594	0.370	0.000	0.882	-0.258	1.661	0.000	1.498	1.088	0.987	0.867	1.139	0.900	0.779
+1	0.674	-0.131	-0.362	0.518	-1.574	0.876	0.442	0.145	1.087	0.497	-1.526	-1.704	0.000	0.680	2.514	-1.374	0.000	0.792	-0.479	0.773	1.551	0.573	1.198	0.984	0.800	0.667	0.987	0.832
+1	1.447	1.145	-0.937	0.307	-1.458	0.478	1.264	0.816	1.087	0.558	1.015	-0.101	2.215	0.937	-0.190	1.177	0.000	0.699	0.954	-1.512	0.000	0.877	0.838	0.990	0.873	0.566	0.646	0.713
+1	0.976	0.308	-0.844	0.436	0.610	1.253	0.149	-1.585	2.173	1.415	0.568	0.096	2.215	0.953	-0.855	0.441	0.000	0.867	-0.650	1.643	0.000	0.890	1.234	0.988	0.796	2.002	1.179	0.977
+0	0.697	0.401	-0.718	0.920	0.735	0.958	-0.172	0.168	2.173	0.872	-0.097	-1.335	0.000	0.513	-1.192	-1.710	1.274	0.426	-1.637	1.368	0.000	0.997	1.227	1.072	0.800	1.013	0.786	0.749
+1	1.305	-2.157	1.740	0.661	-0.912	0.705	-0.516	0.759	2.173	0.989	-0.716	-0.300	2.215	0.627	-1.052	-1.736	0.000	0.467	-2.467	0.568	0.000	0.807	0.964	0.988	1.427	1.012	1.165	0.926
+0	1.847	1.663	-0.618	0.280	1.258	1.462	-0.054	1.371	0.000	0.900	0.309	-0.544	0.000	0.331	-2.149	-0.341	0.000	1.091	-0.833	0.710	3.102	1.496	0.931	0.989	1.549	0.115	1.140	1.150
+0	0.410	-0.323	1.069	2.160	0.010	0.892	0.942	-1.640	2.173	0.946	0.938	1.314	0.000	1.213	-1.099	-0.794	2.548	0.650	0.053	0.056	0.000	1.041	0.916	1.063	0.985	1.910	1.246	1.107
+1	0.576	1.092	-0.088	0.777	-1.579	0.757	0.271	0.109	0.000	0.819	0.827	-1.554	2.215	1.313	2.341	-1.568	0.000	2.827	0.239	-0.338	0.000	0.876	0.759	0.986	0.692	0.457	0.796	0.791
+1	0.537	0.925	-1.406	0.306	-0.050	0.906	1.051	0.037	0.000	1.469	-0.177	-1.320	2.215	1.872	0.723	1.158	0.000	1.313	0.227	-0.501	3.102	0.953	0.727	0.978	0.755	0.892	0.932	0.781
+0	0.716	-0.065	-0.484	1.313	-1.563	0.596	-0.242	0.678	2.173	0.426	-1.909	0.616	0.000	0.885	-0.406	-1.343	2.548	0.501	-1.327	-0.340	0.000	0.470	0.728	1.109	0.919	0.881	0.665	0.692
+1	0.624	-0.389	0.128	1.636	-1.110	1.025	0.573	-0.843	2.173	0.646	-0.697	1.064	0.000	0.632	-1.442	0.961	0.000	0.863	-0.106	1.717	0.000	0.825	0.917	1.257	0.983	0.713	0.890	0.824
+0	0.484	2.101	1.714	1.131	-0.823	0.750	0.583	-1.304	1.087	0.894	0.421	0.559	2.215	0.921	-0.063	0.282	0.000	0.463	-0.474	-1.387	0.000	0.742	0.886	0.995	0.993	1.201	0.806	0.754
+0	0.570	0.339	-1.478	0.528	0.439	0.978	1.479	-1.411	2.173	0.763	1.541	-0.734	0.000	1.375	0.840	0.903	0.000	0.965	1.599	0.364	0.000	0.887	1.061	0.992	1.322	1.453	1.013	0.969
+0	0.940	1.303	1.636	0.851	-1.732	0.803	-0.030	-0.177	0.000	0.480	-0.125	-0.954	0.000	0.944	0.709	0.296	2.548	1.342	-0.418	1.197	3.102	0.853	0.989	0.979	0.873	0.858	0.719	0.786
+1	0.599	0.544	-0.238	0.816	1.043	0.857	0.660	1.128	2.173	0.864	-0.624	-0.843	0.000	1.159	0.367	0.174	0.000	1.520	-0.543	-1.508	0.000	0.842	0.828	0.984	0.759	0.895	0.918	0.791
+1	1.651	1.897	-0.914	0.423	0.315	0.453	0.619	-1.607	2.173	0.532	-0.424	0.209	1.107	0.369	2.479	0.034	0.000	0.701	0.217	0.984	0.000	0.976	0.951	1.035	0.879	0.825	0.915	0.798
+1	0.926	-0.574	-0.763	0.285	1.094	0.672	2.314	1.545	0.000	1.124	0.415	0.809	0.000	1.387	0.270	-0.949	2.548	1.547	-0.631	-0.200	3.102	0.719	0.920	0.986	0.889	0.933	0.797	0.777
+0	0.677	1.698	-0.890	0.641	-0.449	0.607	1.754	1.720	0.000	0.776	0.372	0.782	2.215	0.511	1.491	-0.480	0.000	0.547	-0.341	0.853	3.102	0.919	1.026	0.997	0.696	0.242	0.694	0.687
+0	1.266	0.602	0.958	0.487	1.256	0.709	0.843	-1.196	0.000	0.893	1.303	-0.594	1.107	1.090	1.320	0.354	0.000	0.797	1.846	1.139	0.000	0.780	0.896	0.986	0.661	0.709	0.790	0.806
+1	0.628	-0.616	-0.329	0.764	-1.150	0.477	-0.715	1.187	2.173	1.250	0.607	1.026	2.215	0.983	-0.023	-0.583	0.000	0.377	1.344	-1.015	0.000	0.744	0.954	0.987	0.837	0.841	0.795	0.694
+1	1.035	-0.828	-1.358	1.870	-1.060	1.075	0.130	0.448	2.173	0.660	0.697	0.641	0.000	0.425	1.006	-1.035	0.000	0.751	1.055	1.364	3.102	0.826	0.822	0.988	0.967	0.901	1.077	0.906
+1	0.830	0.265	-0.150	0.660	1.105	0.592	-0.557	0.908	2.173	0.670	-1.419	-0.671	0.000	1.323	-0.409	1.644	2.548	0.850	-0.033	-0.615	0.000	0.760	0.967	0.984	0.895	0.681	0.747	0.770
+1	1.395	1.100	1.167	1.088	0.218	0.400	-0.132	0.024	2.173	0.743	0.530	-1.361	2.215	0.341	-0.691	-0.238	0.000	0.396	-1.426	-0.933	0.000	0.363	0.472	1.287	0.922	0.810	0.792	0.656
+1	1.070	1.875	-1.298	1.215	-0.106	0.767	0.795	0.514	1.087	0.401	2.780	1.276	0.000	0.686	1.127	1.721	2.548	0.391	-0.259	-1.167	0.000	1.278	1.113	1.389	0.852	0.824	0.838	0.785
+0	1.114	-0.071	1.719	0.399	-1.383	0.849	0.254	0.481	0.000	0.958	-0.579	0.742	0.000	1.190	-0.140	-0.862	2.548	0.479	1.390	0.856	0.000	0.952	0.988	0.985	0.764	0.419	0.835	0.827
+0	0.714	0.376	-0.568	1.578	-1.165	0.648	0.141	0.639	2.173	0.472	0.569	1.449	1.107	0.783	1.483	0.361	0.000	0.540	-0.790	0.032	0.000	0.883	0.811	0.982	0.775	0.572	0.760	0.745
+0	0.401	-1.731	0.765	0.974	1.648	0.652	-1.024	0.191	0.000	0.544	-0.366	-1.246	2.215	0.627	0.140	1.008	2.548	0.810	0.409	0.429	0.000	0.950	0.934	0.977	0.621	0.580	0.677	0.650
+1	0.391	1.679	-1.298	0.605	-0.832	0.549	1.338	0.522	2.173	1.244	0.884	1.070	0.000	1.002	0.846	-1.345	2.548	0.783	-2.464	-0.237	0.000	4.515	2.854	0.981	0.877	0.939	1.942	1.489
+1	0.513	-0.220	-0.444	1.699	0.479	1.109	0.181	-0.999	2.173	0.883	-0.335	-1.716	2.215	1.075	-0.380	1.352	0.000	0.857	0.048	0.147	0.000	0.937	0.758	0.986	1.206	0.958	0.949	0.876
+0	1.367	-0.388	0.798	1.158	1.078	0.811	-1.024	-1.628	0.000	1.504	0.097	-0.999	2.215	1.652	-0.860	0.054	2.548	0.573	-0.142	-1.401	0.000	0.869	0.833	1.006	1.412	1.641	1.214	1.041
+1	1.545	-0.533	-1.517	1.177	1.289	2.331	-0.370	-0.073	0.000	1.295	-0.358	-0.891	2.215	0.476	0.756	0.985	0.000	1.945	-0.016	-1.651	3.102	1.962	1.692	1.073	0.656	0.941	1.312	1.242
+0	0.858	0.978	-1.258	0.286	0.161	0.729	1.230	1.087	2.173	0.561	2.670	-0.109	0.000	0.407	2.346	0.938	0.000	1.078	0.729	-0.658	3.102	0.597	0.921	0.982	0.579	0.954	0.733	0.769
+1	1.454	-1.384	0.870	0.067	0.394	1.033	-0.673	0.318	0.000	1.166	-0.763	-1.533	2.215	2.848	-0.045	-0.856	2.548	0.697	-0.140	1.134	0.000	0.931	1.293	0.977	1.541	1.326	1.201	1.078
+1	0.559	-0.913	0.486	1.104	-0.321	1.073	-0.348	1.345	0.000	0.901	-0.827	-0.842	0.000	0.739	0.047	-0.415	2.548	0.433	-1.132	1.268	0.000	0.797	0.695	0.985	0.868	0.346	0.674	0.623
+1	1.333	0.780	-0.964	0.916	1.202	1.822	-0.071	0.742	2.173	1.486	-0.399	-0.824	0.000	0.740	0.568	-0.134	0.000	0.971	-0.070	-1.589	3.102	1.278	0.929	1.421	1.608	1.214	1.215	1.137
+1	2.417	0.631	-0.317	0.323	0.581	0.841	1.524	-1.738	0.000	0.543	1.176	-0.325	0.000	0.827	0.700	0.866	0.000	0.834	-0.262	-1.702	3.102	0.932	0.820	0.988	0.646	0.287	0.595	0.589
+0	0.955	-1.242	0.938	1.104	0.474	0.798	-0.743	1.535	0.000	1.356	-1.357	-1.080	2.215	1.320	-1.396	-0.132	2.548	0.728	-0.529	-0.633	0.000	0.832	0.841	0.988	0.923	1.077	0.988	0.816
+1	1.305	-1.918	0.391	1.161	0.063	0.724	2.593	1.481	0.000	0.592	-1.207	-0.329	0.000	0.886	-0.836	-1.168	2.548	1.067	-1.481	-1.440	0.000	0.916	0.688	0.991	0.969	0.550	0.665	0.638
+0	1.201	0.071	-1.123	2.242	-1.533	0.702	-0.256	0.688	0.000	0.967	0.491	1.040	2.215	1.271	-0.558	0.095	0.000	1.504	0.676	-0.383	3.102	0.917	1.006	0.985	1.017	1.057	0.928	1.057
+0	0.994	-1.607	1.596	0.774	-1.391	0.625	-0.134	-0.862	2.173	0.746	-0.765	-0.316	2.215	1.131	-0.320	0.869	0.000	0.607	0.826	0.301	0.000	0.798	0.967	0.999	0.880	0.581	0.712	0.774
+1	0.482	-0.467	0.729	1.419	1.458	0.824	0.376	-0.242	0.000	1.368	0.023	1.459	2.215	0.826	0.669	-1.079	2.548	0.936	2.215	-0.309	0.000	1.883	1.216	0.997	1.065	0.946	1.224	1.526
+1	0.383	1.588	1.611	0.748	1.194	0.866	-0.279	-0.636	0.000	0.707	0.536	0.801	2.215	1.647	-1.155	0.367	0.000	1.292	0.303	-1.681	3.102	2.016	1.581	0.986	0.584	0.684	1.107	0.958
+0	0.629	0.203	0.736	0.671	-0.271	1.350	-0.486	0.761	2.173	0.496	-0.805	-1.718	0.000	2.393	0.044	-1.046	1.274	0.651	-0.116	-0.541	0.000	0.697	1.006	0.987	1.069	2.317	1.152	0.902
+0	0.905	-0.564	-0.570	0.263	1.096	1.219	-1.397	-1.414	1.087	1.164	-0.533	-0.208	0.000	1.459	1.965	0.784	0.000	2.220	-1.421	0.452	0.000	0.918	1.360	0.993	0.904	0.389	2.118	1.707
+1	1.676	1.804	1.171	0.529	1.175	1.664	0.354	-0.530	0.000	1.004	0.691	-1.280	2.215	0.838	0.373	0.626	2.548	1.094	1.774	0.501	0.000	0.806	1.100	0.991	0.769	0.976	0.807	0.740
+1	1.364	-1.936	0.020	1.327	0.428	1.021	-1.665	-0.907	2.173	0.818	-2.701	1.303	0.000	0.716	-0.590	-1.629	2.548	0.895	-2.280	-1.602	0.000	1.211	0.849	0.989	1.320	0.864	1.065	0.949
+0	0.629	-0.626	0.609	1.828	1.280	0.644	-0.856	-0.873	2.173	0.555	1.066	-0.640	0.000	0.477	-1.364	-1.021	2.548	1.017	0.036	0.380	0.000	0.947	0.941	0.994	1.128	0.241	0.793	0.815
+1	1.152	-0.843	0.926	1.802	0.800	2.493	-1.449	-1.127	0.000	1.737	0.833	0.488	0.000	1.026	0.929	-0.990	2.548	1.408	0.689	1.142	3.102	1.171	0.956	0.993	2.009	0.867	1.499	1.474
+0	2.204	0.081	0.008	1.021	-0.679	2.676	0.090	1.163	0.000	2.210	-1.686	-1.195	0.000	1.805	0.891	-0.148	2.548	0.450	-0.502	-1.295	3.102	6.959	3.492	1.205	0.908	0.845	2.690	2.183
+1	0.957	0.954	1.702	0.043	-0.503	1.113	0.033	-0.308	0.000	0.757	-0.363	-1.129	2.215	1.635	0.068	1.048	1.274	0.415	-2.098	0.061	0.000	1.010	0.979	0.992	0.704	1.125	0.761	0.715
+0	1.222	0.418	1.059	1.303	1.442	0.282	-1.499	-1.286	0.000	1.567	0.016	-0.164	2.215	0.451	2.229	-1.229	0.000	0.660	-0.513	-0.296	3.102	2.284	1.340	0.985	1.531	0.314	1.032	1.094
+1	0.603	1.675	-0.973	0.703	-1.709	1.023	0.652	1.296	2.173	1.078	0.363	-0.263	0.000	0.734	-0.457	-0.745	1.274	0.561	1.434	-0.042	0.000	0.888	0.771	0.984	0.847	1.234	0.874	0.777
+0	0.897	0.949	-0.848	1.115	-0.085	0.522	-1.267	-1.418	0.000	0.684	-0.599	1.474	0.000	1.176	0.922	0.641	2.548	0.470	0.103	0.148	3.102	0.775	0.697	0.984	0.839	0.358	0.847	1.008
+1	0.987	1.013	-1.504	0.468	-0.259	1.160	0.476	-0.971	2.173	1.266	0.919	0.780	0.000	0.634	1.695	0.233	0.000	0.487	-0.082	0.719	3.102	0.921	0.641	0.991	0.730	0.828	0.952	0.807
+1	0.847	1.581	-1.397	1.629	1.529	1.053	0.816	-0.344	2.173	0.895	0.779	0.332	0.000	0.750	1.311	0.419	2.548	1.604	0.844	1.367	0.000	1.265	0.798	0.989	1.328	0.783	0.930	0.879
+1	0.805	1.416	-1.327	0.397	0.589	0.488	0.982	0.843	0.000	0.664	-0.999	0.129	0.000	0.624	0.613	-0.558	0.000	1.431	-0.667	-1.561	3.102	0.959	1.103	0.989	0.590	0.632	0.926	0.798
+0	1.220	-0.313	-0.489	1.759	0.201	1.698	-0.220	0.241	2.173	1.294	1.390	-1.682	0.000	1.447	-1.623	-1.296	0.000	1.710	0.872	-1.356	3.102	1.198	0.981	1.184	0.859	2.165	1.807	1.661
+0	0.772	-0.611	-0.549	0.465	-1.528	1.103	-0.140	0.001	2.173	0.854	-0.406	1.655	0.000	0.733	-1.250	1.072	0.000	0.883	0.627	-1.132	3.102	0.856	0.927	0.987	1.094	1.013	0.938	0.870
+1	1.910	0.771	0.828	0.231	1.267	1.398	1.455	-0.295	2.173	0.837	-2.564	0.770	0.000	0.540	2.189	1.287	0.000	1.345	1.311	-1.151	0.000	0.861	0.869	0.984	1.359	1.562	1.105	0.963
+1	0.295	0.832	1.399	1.222	-0.517	2.480	0.013	1.591	0.000	2.289	0.436	0.287	2.215	1.995	-0.367	-0.409	1.274	0.375	1.367	-1.716	0.000	1.356	2.171	0.990	1.467	1.664	1.855	1.705
+1	1.228	0.339	-0.575	0.417	1.474	0.480	-1.416	-1.498	2.173	0.614	-0.933	-0.961	0.000	1.189	1.690	1.003	0.000	1.690	-1.065	0.106	3.102	0.963	1.147	0.987	1.086	0.948	0.930	0.866
+0	2.877	-1.014	1.440	0.782	0.483	1.134	-0.735	-0.196	2.173	1.123	0.084	-0.596	0.000	1.796	-0.356	1.044	2.548	1.406	1.582	-0.991	0.000	0.939	1.178	1.576	0.996	1.629	1.216	1.280
+1	2.178	0.259	1.107	0.256	1.222	0.979	-0.440	-0.538	1.087	0.496	-0.760	-0.049	0.000	1.471	1.683	-1.486	0.000	0.646	0.695	-1.577	3.102	1.093	1.070	0.984	0.608	0.889	0.962	0.866
+1	0.604	0.592	1.295	0.964	0.348	1.178	-0.016	0.832	2.173	1.626	-0.420	-0.760	0.000	0.748	0.461	-0.906	0.000	0.728	0.309	-1.269	1.551	0.852	0.604	0.989	0.678	0.949	1.021	0.878
+0	0.428	-1.352	-0.912	1.713	0.797	1.894	-1.452	0.191	2.173	2.378	2.113	-1.190	0.000	0.860	2.174	0.949	0.000	1.693	0.759	1.426	3.102	0.885	1.527	1.186	1.090	3.294	4.492	3.676
+0	0.473	0.485	0.154	1.433	-1.504	0.766	1.257	-1.302	2.173	0.414	0.119	0.238	0.000	0.805	0.242	-0.691	2.548	0.734	0.749	0.753	0.000	0.430	0.893	1.137	0.686	0.724	0.618	0.608
+1	0.763	-0.601	0.876	0.182	-1.678	0.818	0.599	0.481	2.173	0.658	-0.737	-0.553	0.000	0.857	-1.138	-1.435	0.000	1.540	-1.466	-0.447	0.000	0.870	0.566	0.989	0.728	0.658	0.821	0.726
+0	0.619	-0.273	-0.143	0.992	-1.267	0.566	0.876	-1.396	2.173	0.515	0.892	0.618	0.000	0.434	-0.902	0.862	2.548	0.490	-0.539	0.549	0.000	0.568	0.794	0.984	0.667	0.867	0.597	0.578
+0	0.793	0.970	0.324	0.570	0.816	0.761	-0.550	1.519	2.173	1.150	0.496	-0.447	0.000	0.925	0.724	1.008	1.274	1.135	-0.275	-0.843	0.000	0.829	1.068	0.978	1.603	0.892	1.041	1.059
+1	0.480	0.364	-0.067	1.906	-1.582	1.397	1.159	0.140	0.000	0.639	0.398	-1.102	0.000	1.597	-0.668	1.607	2.548	1.306	-0.797	0.288	3.102	0.856	1.259	1.297	1.022	1.032	1.049	0.939
+0	0.514	1.304	1.490	1.741	-0.220	0.648	0.155	0.535	0.000	0.562	-1.016	0.837	0.000	0.863	-0.780	-0.815	2.548	1.688	-0.130	-1.545	3.102	0.887	0.980	1.309	1.269	0.654	1.044	1.035
+0	1.225	0.333	0.656	0.893	0.859	1.037	-0.876	1.603	1.087	1.769	0.272	-0.227	2.215	1.000	0.579	-1.690	0.000	1.385	0.471	-0.860	0.000	0.884	1.207	0.995	1.097	2.336	1.282	1.145
+0	2.044	-1.472	-0.294	0.392	0.369	0.927	0.718	1.492	1.087	1.619	-0.736	0.047	2.215	1.884	-0.101	-1.540	0.000	0.548	-0.441	1.117	0.000	0.798	0.877	0.981	0.750	2.272	1.469	1.276
+0	1.037	-0.276	0.735	3.526	1.156	2.498	0.401	-0.590	1.087	0.714	-1.203	1.393	2.215	0.681	0.629	1.534	0.000	0.719	-0.355	-0.706	0.000	0.831	0.857	0.988	2.864	2.633	1.988	1.466
+1	0.651	-1.218	-0.791	0.770	-1.449	0.610	-0.535	0.960	2.173	0.380	-1.072	-0.031	2.215	0.415	2.123	-1.100	0.000	0.776	0.217	0.420	0.000	0.986	1.008	1.001	0.853	0.588	0.799	0.776
+0	1.586	-0.409	0.085	3.258	0.405	1.647	-0.674	-1.519	0.000	0.640	-1.027	-1.681	0.000	1.452	-0.444	-0.957	2.548	0.927	-0.017	1.215	3.102	0.519	0.866	0.992	0.881	0.847	1.018	1.278
+0	0.712	0.092	-0.466	0.688	1.236	0.921	-1.217	-1.022	2.173	2.236	-1.167	0.868	2.215	0.851	-1.892	-0.753	0.000	0.475	-1.216	-0.383	0.000	0.668	0.758	0.988	1.180	2.093	1.157	0.934
+0	0.419	0.471	0.974	2.805	0.235	1.473	-0.198	1.255	1.087	0.931	1.083	-0.712	0.000	1.569	1.358	-1.179	2.548	2.506	0.199	-0.842	0.000	0.929	0.991	0.992	1.732	2.367	1.549	1.430
+1	0.667	1.003	1.504	0.368	1.061	0.885	-0.318	-0.353	0.000	1.438	-1.939	0.710	0.000	1.851	0.277	-1.460	2.548	1.403	0.517	-0.157	0.000	0.883	1.019	1.000	0.790	0.859	0.938	0.841
+1	1.877	-0.492	0.372	0.441	0.955	1.034	-1.220	-0.846	1.087	0.952	-0.320	1.125	0.000	0.542	0.308	-1.261	2.548	1.018	-1.415	-1.547	0.000	1.280	0.932	0.991	1.273	0.878	0.921	0.906
+0	1.052	0.901	1.176	1.280	1.517	0.562	-1.150	-0.079	2.173	1.228	-0.308	-0.354	0.000	0.790	-1.492	-0.963	0.000	0.942	-0.672	-1.588	3.102	1.116	0.902	0.988	1.993	0.765	1.375	1.325
+1	0.518	-0.254	1.642	0.865	0.725	0.980	0.734	0.023	0.000	1.448	0.780	-1.736	2.215	0.955	0.513	-0.519	0.000	0.365	-0.444	-0.243	3.102	0.833	0.555	0.984	0.827	0.795	0.890	0.786
+0	0.870	0.815	-0.506	0.663	-0.518	0.935	0.289	-1.675	2.173	1.188	0.005	0.635	0.000	0.580	0.066	-1.455	2.548	0.580	-0.634	-0.199	0.000	0.852	0.788	0.979	1.283	0.208	0.856	0.950
+0	0.628	1.382	0.135	0.683	0.571	1.097	0.564	-0.950	2.173	0.617	-0.326	0.371	0.000	1.093	0.918	1.667	2.548	0.460	1.221	0.708	0.000	0.743	0.861	0.975	1.067	1.007	0.843	0.762
+0	4.357	0.816	-1.609	1.845	-1.288	3.292	0.726	0.324	2.173	1.528	0.583	-0.801	2.215	0.605	0.572	1.406	0.000	0.794	-0.791	0.122	0.000	0.967	1.132	1.124	3.602	2.811	2.460	1.861
+0	0.677	-1.265	1.559	0.866	-0.618	0.823	0.260	0.185	0.000	1.133	0.337	1.589	2.215	0.563	-0.830	0.510	0.000	0.777	0.117	-0.941	3.102	0.839	0.763	0.986	1.182	0.649	0.796	0.851
+0	2.466	-1.838	-1.648	1.717	1.533	1.676	-1.553	-0.109	2.173	0.670	-0.666	0.284	0.000	0.334	-2.480	0.316	0.000	0.366	-0.804	-1.298	3.102	0.875	0.894	0.997	0.548	0.770	1.302	1.079
+1	1.403	0.129	-1.307	0.688	0.306	0.579	0.753	0.814	1.087	0.474	0.694	-1.400	0.000	0.520	1.995	0.185	0.000	0.929	-0.504	1.270	3.102	0.972	0.998	1.353	0.948	0.650	0.688	0.724
+1	0.351	1.188	-0.360	0.254	-0.346	1.129	0.545	1.691	0.000	0.652	-0.039	-0.258	2.215	1.089	0.655	0.472	2.548	0.554	-0.493	1.366	0.000	0.808	1.045	0.992	0.570	0.649	0.809	0.744
+0	1.875	-0.013	-0.128	0.236	1.163	0.902	0.426	0.590	2.173	1.251	-1.210	-0.616	0.000	1.035	1.534	0.912	0.000	1.944	1.789	-1.691	0.000	0.974	1.113	0.990	0.925	1.120	0.956	0.912
+0	0.298	0.750	-0.507	1.555	1.463	0.804	1.200	-0.665	0.000	0.439	-0.829	-0.252	1.107	0.770	-1.090	0.947	2.548	1.165	-0.166	-0.763	0.000	1.140	0.997	0.988	1.330	0.555	1.005	1.012
+0	0.647	0.342	0.245	4.340	-0.157	2.229	0.068	1.170	2.173	2.133	-0.201	-1.441	0.000	1.467	0.697	-0.532	1.274	1.457	0.583	-1.640	0.000	0.875	1.417	0.976	2.512	2.390	1.794	1.665
+1	1.731	-0.803	-1.013	1.492	-0.020	1.646	-0.541	1.121	2.173	0.459	-1.251	-1.495	2.215	0.605	-1.711	-0.232	0.000	0.658	0.634	-0.068	0.000	1.214	0.886	1.738	1.833	1.024	1.192	1.034
+0	0.515	1.416	-1.089	1.697	1.426	1.414	0.941	0.027	0.000	1.480	0.133	-1.595	2.215	1.110	0.752	0.760	2.548	1.062	0.697	-0.492	0.000	0.851	0.955	0.994	1.105	1.255	1.175	1.095
+0	1.261	0.858	1.465	0.757	0.305	2.310	0.679	1.080	2.173	1.544	2.518	-0.464	0.000	2.326	0.270	-0.841	0.000	2.163	0.839	-0.500	3.102	0.715	0.825	1.170	0.980	2.371	1.527	1.221
+1	1.445	1.509	1.471	0.414	-1.285	0.767	0.864	-0.677	2.173	0.524	1.388	0.171	0.000	0.826	0.190	0.121	2.548	0.572	1.691	-1.603	0.000	0.870	0.935	0.994	0.968	0.735	0.783	0.777
+1	0.919	-0.264	-1.245	0.681	-1.722	1.022	1.010	0.097	2.173	0.685	0.403	-1.351	0.000	1.357	-0.429	1.262	1.274	0.687	1.021	-0.563	0.000	0.953	0.796	0.991	0.873	1.749	1.056	0.917
+1	0.293	-2.258	-1.427	1.191	1.202	0.394	-2.030	1.438	0.000	0.723	0.596	-0.024	2.215	0.525	-1.678	-0.290	0.000	0.788	-0.824	-1.029	3.102	0.821	0.626	0.976	1.080	0.810	0.842	0.771
+0	3.286	0.386	1.688	1.619	-1.620	1.392	-0.009	0.280	0.000	1.179	-0.776	-0.110	2.215	1.256	0.248	-1.114	2.548	0.777	0.825	-0.156	0.000	1.026	1.065	0.964	0.909	1.249	1.384	1.395
+1	1.075	0.603	0.561	0.656	-0.685	0.985	0.175	0.979	2.173	1.154	0.584	-0.886	0.000	1.084	-0.354	-1.004	2.548	0.865	1.224	1.269	0.000	1.346	1.073	1.048	0.873	1.310	1.003	0.865
+1	1.098	-0.091	1.466	1.558	0.915	0.649	1.314	-1.182	2.173	0.791	0.073	0.351	0.000	0.517	0.940	1.195	0.000	1.150	1.187	-0.692	3.102	0.866	0.822	0.980	1.311	0.394	1.119	0.890
+1	0.481	-1.042	0.148	1.135	-1.249	1.202	-0.344	0.308	1.087	0.779	-1.431	1.581	0.000	0.860	-0.860	-1.125	0.000	0.785	0.303	1.199	3.102	0.878	0.853	0.988	1.072	0.827	0.936	0.815
+0	1.348	0.497	0.318	0.806	0.976	1.393	-0.152	0.632	2.173	2.130	0.515	-1.054	0.000	0.908	0.062	-0.780	0.000	1.185	0.687	1.668	1.551	0.720	0.898	0.985	0.683	1.292	1.320	1.131
+0	2.677	-0.420	-1.685	1.828	1.433	2.040	-0.718	-0.039	0.000	0.400	-0.873	0.472	0.000	0.444	0.340	-0.830	2.548	0.431	0.768	-1.417	3.102	0.869	0.917	0.996	0.707	0.193	0.728	1.154
+1	1.300	0.586	-0.122	1.306	0.609	0.727	-0.556	-1.652	2.173	0.636	0.720	1.393	2.215	0.328	1.280	-0.390	0.000	0.386	0.752	-0.905	0.000	0.202	0.751	1.106	0.864	0.799	0.928	0.717
+0	0.637	-0.176	1.737	1.322	-0.414	0.702	-0.964	-0.680	0.000	1.054	-0.461	0.889	2.215	0.861	-0.267	0.225	0.000	1.910	-1.888	1.027	0.000	0.919	0.899	1.186	0.993	1.109	0.862	0.775
+1	0.723	-0.104	1.572	0.428	-0.840	0.655	0.544	1.401	2.173	1.522	-0.154	-0.452	2.215	0.996	0.190	0.273	0.000	1.906	-0.176	0.966	0.000	0.945	0.894	0.990	0.981	1.555	0.988	0.893
+0	2.016	-0.570	1.612	0.798	0.441	0.334	0.191	-0.909	0.000	0.939	0.146	0.021	2.215	0.553	-0.444	1.156	2.548	0.781	-1.545	-0.520	0.000	0.922	0.956	1.528	0.722	0.699	0.778	0.901
+0	1.352	-0.707	1.284	0.665	0.580	0.694	-1.040	-0.899	2.173	0.692	-2.048	0.029	0.000	0.545	-2.042	1.259	0.000	0.661	-0.808	-1.251	3.102	0.845	0.991	0.979	0.662	0.225	0.685	0.769
+1	1.057	-1.561	-0.411	0.952	-0.681	1.236	-1.107	1.045	2.173	1.288	-2.521	-0.521	0.000	1.361	-1.239	1.546	0.000	0.373	-1.540	0.028	0.000	0.794	0.782	0.987	0.889	0.832	0.972	0.828
+0	1.118	-0.017	-1.227	1.077	1.256	0.714	0.624	-0.811	0.000	0.800	0.704	0.387	1.107	0.604	0.234	0.986	0.000	1.306	-0.456	0.094	3.102	0.828	0.984	1.195	0.987	0.672	0.774	0.748
+1	0.602	2.201	0.212	0.119	0.182	0.474	2.130	1.270	0.000	0.370	2.088	-0.573	0.000	0.780	-0.725	-1.033	0.000	1.642	0.598	0.303	3.102	0.886	0.988	0.985	0.644	0.756	0.651	0.599
+0	1.677	-0.844	1.581	0.585	0.887	1.012	-2.315	0.752	0.000	1.077	0.748	-0.195	0.000	0.718	0.832	-1.337	1.274	1.181	-0.557	-1.006	3.102	1.018	1.247	0.988	0.908	0.651	1.311	1.120
+1	1.695	0.259	1.224	1.344	1.067	0.718	-1.752	-0.215	0.000	0.473	0.991	-0.993	0.000	0.891	1.285	-1.500	2.548	0.908	-0.131	0.288	0.000	0.945	0.824	0.979	1.009	0.951	0.934	0.833
+0	0.793	0.628	0.432	1.707	0.302	0.919	1.045	-0.784	0.000	1.472	0.175	-1.284	2.215	1.569	0.155	0.971	2.548	0.435	0.735	1.625	0.000	0.801	0.907	0.992	0.831	1.446	1.082	1.051
+1	0.537	-0.664	-0.244	1.104	1.272	1.154	0.394	1.633	0.000	1.527	0.963	0.559	2.215	1.744	0.650	-0.912	0.000	1.097	0.730	-0.368	3.102	1.953	1.319	1.045	1.309	0.869	1.196	1.126
+1	0.585	-1.469	1.005	0.749	-1.060	1.224	-0.717	-0.323	2.173	1.012	-0.201	1.268	0.000	0.359	-0.567	0.476	0.000	1.117	-1.124	1.557	3.102	0.636	1.281	0.986	0.616	1.289	0.890	0.881
+1	0.354	-1.517	0.667	2.534	-1.298	1.020	-0.375	1.254	0.000	1.119	-0.060	-1.538	2.215	1.059	-0.395	-0.140	0.000	2.609	0.199	-0.778	1.551	0.957	0.975	1.286	1.666	1.003	1.224	1.135
+1	0.691	-1.619	-1.380	0.361	1.727	1.493	-1.093	-0.289	0.000	1.447	-0.640	1.341	0.000	1.453	-0.617	-1.456	1.274	1.061	-1.481	-0.091	0.000	0.744	0.649	0.987	0.596	0.727	0.856	0.797
+0	1.336	1.293	-1.359	0.357	0.067	1.110	-0.058	-0.515	0.000	0.976	1.498	1.207	0.000	1.133	0.437	1.053	2.548	0.543	1.374	0.171	0.000	0.764	0.761	0.984	0.827	0.553	0.607	0.612
+0	0.417	-1.111	1.661	2.209	-0.683	1.931	-0.642	0.959	1.087	1.514	-2.032	-0.686	0.000	1.521	-0.539	1.344	0.000	0.978	-0.866	0.363	1.551	2.813	1.850	1.140	1.854	0.799	1.600	1.556
+0	1.058	0.390	-0.591	0.134	1.149	0.346	-1.550	0.186	0.000	1.108	-0.999	0.843	1.107	1.124	0.415	-1.514	0.000	1.067	-0.426	-1.000	3.102	1.744	1.050	0.985	1.006	1.010	0.883	0.789
+1	1.655	0.253	1.216	0.270	1.703	0.500	-0.006	-1.418	2.173	0.690	-0.350	0.170	2.215	1.045	-0.924	-0.774	0.000	0.996	-0.745	-0.123	0.000	0.839	0.820	0.993	0.921	0.869	0.725	0.708
+0	1.603	-0.850	0.564	0.829	0.093	1.270	-1.113	-1.155	2.173	0.853	-1.021	1.248	2.215	0.617	-1.270	1.733	0.000	0.935	-0.092	0.136	0.000	1.011	1.074	0.977	0.823	1.269	1.054	0.878
+0	1.568	-0.792	1.005	0.545	0.896	0.895	-1.698	-0.988	0.000	0.608	-1.634	1.705	0.000	0.826	0.208	0.618	1.274	2.063	-1.743	-0.520	0.000	0.939	0.986	0.990	0.600	0.435	1.033	1.087
+0	0.489	-1.335	-1.102	1.738	1.028	0.628	-0.992	-0.627	0.000	0.652	-0.064	-0.215	0.000	1.072	0.173	-1.251	2.548	1.042	0.057	0.841	3.102	0.823	0.895	1.200	1.164	0.770	0.837	0.846
+1	1.876	0.870	1.234	0.556	-1.262	1.764	0.855	-0.467	2.173	1.079	1.351	0.852	0.000	0.773	0.383	0.874	0.000	1.292	0.829	-1.228	3.102	0.707	0.969	1.102	1.601	1.017	1.112	1.028
+0	1.033	0.407	-0.374	0.705	-1.254	0.690	-0.231	1.502	2.173	0.433	-2.009	-0.057	0.000	0.861	1.151	0.334	0.000	0.960	-0.839	1.299	3.102	2.411	1.480	0.982	0.995	0.377	1.012	0.994
+0	1.092	0.653	-0.801	0.463	0.426	0.529	-1.055	0.040	0.000	0.663	0.999	1.255	1.107	0.749	-1.106	1.185	2.548	0.841	-0.745	-1.029	0.000	0.841	0.743	0.988	0.750	1.028	0.831	0.868
+1	0.799	-0.285	-0.011	0.531	1.392	1.063	0.854	0.494	2.173	1.187	-1.065	-0.851	0.000	0.429	-0.296	1.072	0.000	0.942	-1.985	1.172	0.000	0.873	0.693	0.992	0.819	0.689	1.131	0.913
+0	0.503	1.973	-0.377	1.515	-1.514	0.708	1.081	-0.313	2.173	1.110	-0.417	0.839	0.000	0.712	-1.153	1.165	0.000	0.675	-0.303	-0.930	1.551	0.709	0.761	1.032	0.986	0.698	0.963	1.291
+0	0.690	-0.574	-1.608	1.182	1.118	0.557	-2.243	0.144	0.000	0.969	0.216	-1.383	1.107	1.054	0.888	-0.709	2.548	0.566	1.663	-0.550	0.000	0.752	1.528	0.987	1.408	0.740	1.290	1.123
+1	0.890	1.501	0.786	0.779	-0.615	1.126	0.716	1.541	2.173	0.887	0.728	-0.673	2.215	1.216	0.332	-0.020	0.000	0.965	1.828	0.101	0.000	0.827	0.715	1.099	1.088	1.339	0.924	0.878
+0	0.566	0.883	0.655	1.600	0.034	1.155	2.028	-1.499	0.000	0.723	-0.871	0.763	0.000	1.286	-0.696	-0.676	2.548	1.134	-0.113	1.207	3.102	4.366	2.493	0.984	0.960	0.962	1.843	1.511
+0	1.146	1.086	-0.911	0.838	1.298	0.821	0.127	-0.145	0.000	1.352	0.474	-1.580	2.215	1.619	-0.081	0.675	2.548	1.382	-0.748	0.127	0.000	0.958	0.976	1.239	0.876	1.481	1.116	1.076
+0	1.739	-0.326	-1.661	0.420	-1.705	1.193	-0.031	-1.212	2.173	1.783	-0.442	0.522	0.000	1.064	-0.692	0.027	0.000	1.314	0.359	-0.037	3.102	0.968	0.897	0.986	0.907	1.196	1.175	1.112
+1	0.669	0.194	-0.703	0.657	-0.260	0.899	-2.511	0.311	0.000	1.482	0.773	0.974	2.215	3.459	0.037	-1.299	1.274	2.113	0.067	1.516	0.000	0.740	0.871	0.979	1.361	2.330	1.322	1.046
+1	1.355	-1.033	-1.173	0.552	-0.048	0.899	-0.482	-1.287	2.173	1.422	-1.227	0.390	1.107	1.937	-0.028	0.914	0.000	0.849	-0.230	-1.734	0.000	0.986	1.224	1.017	1.051	1.788	1.150	1.009
+1	0.511	-0.202	1.029	0.780	1.154	0.816	0.532	-0.731	0.000	0.757	0.517	0.749	2.215	1.302	0.289	-1.188	0.000	0.584	1.211	-0.350	0.000	0.876	0.943	0.995	0.963	0.256	0.808	0.891
+1	1.109	0.572	1.484	0.753	1.543	1.711	-0.145	-0.746	1.087	1.759	0.631	0.845	2.215	0.945	0.542	0.003	0.000	0.378	-1.150	-0.044	0.000	0.764	1.042	0.992	1.045	2.736	1.441	1.140
+0	0.712	-0.025	0.553	0.928	-0.711	1.304	0.045	-0.300	0.000	0.477	0.720	0.969	0.000	1.727	-0.474	1.328	1.274	1.282	2.222	1.684	0.000	0.819	0.765	1.023	0.961	0.657	0.799	0.744
+1	1.131	-0.302	1.079	0.901	0.236	0.904	-0.249	1.694	2.173	1.507	-0.702	-1.128	0.000	0.774	0.565	0.284	2.548	1.802	1.446	-0.192	0.000	3.720	2.108	0.986	0.930	1.101	1.484	1.238
+0	1.392	1.253	0.118	0.864	-1.358	0.922	-0.447	-1.243	1.087	1.969	1.031	0.774	2.215	1.333	-0.359	-0.681	0.000	1.099	-0.257	1.473	0.000	1.246	0.909	1.475	1.234	2.531	1.449	1.306
+0	1.374	2.291	-0.479	1.339	-0.243	0.687	2.345	1.310	0.000	0.467	1.081	0.772	0.000	0.656	1.155	-1.636	2.548	0.592	0.536	-1.269	3.102	0.981	0.821	1.010	0.877	0.217	0.638	0.758
+1	0.401	-1.516	0.909	2.738	0.519	0.887	0.566	-1.202	0.000	0.909	-0.176	1.682	0.000	2.149	-0.878	-0.514	2.548	0.929	-0.563	-1.555	3.102	1.228	0.803	0.980	1.382	0.884	1.025	1.172
+1	0.430	-1.589	1.417	2.158	1.226	1.180	-0.829	-0.781	2.173	0.798	1.400	-0.111	0.000	0.939	-0.878	1.076	2.548	0.576	1.335	-0.826	0.000	0.861	0.970	0.982	1.489	1.308	1.015	0.992
+1	1.943	-0.391	-0.840	0.621	-1.613	2.026	1.734	1.025	0.000	0.930	0.573	-0.912	0.000	1.326	0.847	-0.220	1.274	1.181	0.079	0.709	3.102	1.164	1.007	0.987	1.094	0.821	0.857	0.786
+1	0.499	0.436	0.887	0.859	1.509	0.733	-0.559	1.111	1.087	1.011	-0.796	0.279	2.215	1.472	-0.510	-0.982	0.000	1.952	0.379	-0.733	0.000	1.076	1.358	0.991	0.589	0.879	1.068	0.922
+0	0.998	-0.407	-1.711	0.139	0.652	0.810	-0.331	-0.721	0.000	0.471	-0.533	0.442	0.000	0.531	-1.405	0.120	2.548	0.707	0.098	-1.176	1.551	1.145	0.809	0.988	0.529	0.612	0.562	0.609
+1	1.482	0.872	0.638	1.288	0.362	0.856	0.900	-0.511	1.087	1.072	1.061	-1.432	2.215	1.770	-2.292	-1.547	0.000	1.131	1.374	0.783	0.000	6.316	4.381	1.002	1.317	1.048	2.903	2.351
+1	2.084	-0.422	1.289	1.125	0.735	1.104	-0.518	-0.326	2.173	0.413	-0.719	-0.699	0.000	0.857	0.108	-1.631	0.000	0.527	0.641	-1.362	3.102	0.791	0.952	1.016	0.776	0.856	0.987	0.836
+0	0.464	0.674	0.025	0.430	-1.703	0.982	-1.311	-0.808	2.173	1.875	1.060	0.821	2.215	0.954	-0.480	-1.677	0.000	0.567	0.702	-0.939	0.000	0.781	1.076	0.989	1.256	3.632	1.652	1.252
+1	0.457	-1.944	-1.010	1.409	0.931	1.098	-0.742	-0.415	0.000	1.537	-0.834	0.945	2.215	1.752	-0.287	-1.269	2.548	0.692	-1.537	-0.223	0.000	0.801	1.192	1.094	1.006	1.659	1.175	1.122
+0	3.260	-0.943	1.737	0.920	1.309	0.946	-0.139	-0.271	2.173	0.994	-0.952	-0.311	0.000	0.563	-0.136	-0.881	0.000	1.236	-0.507	0.906	1.551	0.747	0.869	0.985	1.769	1.034	1.179	1.042
+0	0.615	-0.778	0.246	1.861	1.619	0.560	-0.943	-0.204	2.173	0.550	-0.759	-1.342	2.215	0.578	0.076	-0.973	0.000	0.939	0.035	0.680	0.000	0.810	0.747	1.401	0.772	0.702	0.719	0.662
+1	2.370	-0.064	-0.237	1.737	0.154	2.319	-1.838	-1.673	0.000	1.053	-1.305	-0.075	0.000	0.925	0.149	0.318	1.274	0.851	-0.922	0.981	3.102	0.919	0.940	0.989	0.612	0.598	1.219	1.626
+1	1.486	0.311	-1.262	1.354	-0.847	0.886	-0.158	1.213	2.173	1.160	-0.218	0.239	0.000	1.166	0.494	0.278	2.548	0.575	1.454	-1.701	0.000	0.429	1.129	0.983	1.111	1.049	1.006	0.920
+1	1.294	1.587	-0.864	0.487	-0.312	0.828	1.051	-0.031	1.087	2.443	1.216	1.609	2.215	1.167	0.813	0.921	0.000	1.751	-0.415	0.119	0.000	1.015	1.091	0.974	1.357	2.093	1.178	1.059
+1	0.984	0.465	-1.661	0.379	-0.554	0.977	0.237	0.365	0.000	0.510	0.143	1.101	0.000	1.099	-0.662	-1.593	2.548	1.104	-0.197	-0.648	3.102	0.925	0.922	0.986	0.642	0.667	0.806	0.722
+1	0.930	-0.009	0.047	0.667	1.367	1.065	-0.231	0.815	0.000	1.199	-1.114	-0.877	2.215	0.940	0.824	-1.583	0.000	1.052	-0.407	-0.076	1.551	1.843	1.257	1.013	1.047	0.751	1.158	0.941
+0	0.767	-0.011	-0.637	0.341	-1.437	1.438	-0.425	-0.450	2.173	1.073	-0.718	1.341	2.215	0.633	-1.394	0.486	0.000	0.603	-1.945	-1.626	0.000	0.703	0.790	0.984	1.111	1.848	1.129	1.072
+1	1.779	0.017	0.432	0.402	1.022	0.959	1.480	1.595	2.173	1.252	1.365	0.006	0.000	1.188	-0.174	-1.107	0.000	1.181	0.518	-0.258	0.000	1.057	0.910	0.991	1.616	0.779	1.158	1.053
+0	0.881	0.630	1.029	1.990	0.508	1.102	0.742	-1.298	2.173	1.565	1.085	0.686	2.215	2.691	1.391	-0.904	0.000	0.499	1.388	-1.199	0.000	0.347	0.861	0.997	0.881	1.920	1.233	1.310
+0	1.754	-0.266	0.389	0.347	-0.030	0.462	-1.408	-0.957	2.173	0.515	-2.341	-1.700	0.000	0.588	-0.797	1.355	2.548	0.608	0.329	-1.389	0.000	1.406	0.909	0.988	0.760	0.593	0.768	0.847
+0	1.087	0.311	-1.447	0.173	0.567	0.854	0.362	0.584	0.000	1.416	-0.716	-1.211	2.215	0.648	-0.358	-0.692	1.274	0.867	-0.513	0.206	0.000	0.803	0.813	0.984	1.110	0.491	0.921	0.873
+0	0.279	1.114	-1.190	3.004	-0.738	1.233	0.896	1.092	2.173	0.454	-0.374	0.117	2.215	0.357	0.119	1.270	0.000	0.458	1.343	0.316	0.000	0.495	0.540	0.988	1.715	1.139	1.618	1.183
+1	1.773	-0.694	-1.518	2.306	-1.200	3.104	0.749	0.362	0.000	1.871	0.230	-1.686	2.215	0.805	-0.179	-0.871	1.274	0.910	0.607	-0.246	0.000	1.338	1.598	0.984	1.050	0.919	1.678	1.807
+0	0.553	0.683	0.827	0.973	-0.706	1.488	0.149	1.140	2.173	1.788	0.447	-0.478	0.000	0.596	1.043	1.607	0.000	0.373	-0.868	-1.308	1.551	1.607	1.026	0.998	1.134	0.808	1.142	0.936
+1	0.397	1.101	-1.139	1.688	0.146	0.972	0.541	1.518	0.000	1.549	-0.873	-1.012	0.000	2.282	-0.151	0.314	2.548	1.174	0.033	-1.368	0.000	0.937	0.776	1.039	1.143	0.959	0.986	1.013
+1	0.840	1.906	-0.959	0.869	0.576	0.642	0.554	-1.351	0.000	0.756	0.923	-0.823	2.215	1.251	1.130	0.545	2.548	1.513	0.410	1.073	0.000	1.231	0.985	1.163	0.812	0.987	0.816	0.822
+1	0.477	1.665	0.814	0.763	-0.382	0.828	-0.008	0.280	2.173	1.213	-0.001	1.560	0.000	1.136	0.311	-1.289	0.000	0.797	1.091	-0.616	3.102	1.026	0.964	0.992	0.772	0.869	0.916	0.803
+0	2.655	0.020	0.273	1.464	0.482	1.709	-0.107	-1.456	2.173	0.825	0.141	-0.386	0.000	1.342	-0.592	1.635	1.274	0.859	-0.175	-0.874	0.000	0.829	0.946	1.003	2.179	0.836	1.505	1.176
+0	0.771	-1.992	-0.720	0.732	-1.464	0.869	-1.290	0.388	2.173	0.926	-1.072	-1.489	2.215	0.640	-1.232	0.840	0.000	0.528	-2.440	-0.446	0.000	0.811	0.868	0.993	0.995	1.317	0.809	0.714
+0	1.357	1.302	0.076	0.283	-1.060	0.783	1.559	-0.994	0.000	0.947	1.212	1.617	0.000	1.127	0.311	0.442	2.548	0.582	-0.052	1.186	1.551	1.330	0.995	0.985	0.846	0.404	0.858	0.815
+0	0.442	-0.381	-0.424	1.244	0.591	0.731	0.605	-0.713	2.173	0.629	2.762	1.040	0.000	0.476	2.693	-0.617	0.000	0.399	0.442	1.486	3.102	0.839	0.755	0.988	0.869	0.524	0.877	0.918
+0	0.884	0.422	0.055	0.818	0.624	0.950	-0.763	1.624	0.000	0.818	-0.609	-1.166	0.000	1.057	-0.528	1.070	2.548	1.691	-0.124	-0.335	3.102	1.104	0.933	0.985	0.913	1.000	0.863	1.056
+0	1.276	0.156	1.714	1.053	-1.189	0.672	-0.464	-0.030	2.173	0.469	-2.483	0.442	0.000	0.564	2.580	-0.253	0.000	0.444	-0.628	1.080	1.551	5.832	2.983	0.985	1.162	0.494	1.809	1.513
+0	1.106	-0.556	0.406	0.573	-1.400	0.769	-0.518	1.457	2.173	0.743	-0.352	-0.010	0.000	1.469	-0.550	-0.930	2.548	0.540	1.236	-0.571	0.000	0.962	0.970	1.101	0.805	1.107	0.873	0.773
+0	0.539	-0.964	-0.464	1.371	-1.606	0.667	-0.160	0.655	0.000	0.952	0.352	-0.740	2.215	0.952	0.007	1.123	0.000	1.061	-0.505	1.389	3.102	1.063	0.991	1.019	0.633	0.967	0.732	0.799
+1	0.533	-0.989	-1.608	0.462	-1.723	1.204	-0.598	-0.098	2.173	1.343	-0.460	1.632	2.215	0.577	0.221	-0.492	0.000	0.628	-0.073	0.472	0.000	0.518	0.880	0.988	1.179	1.874	1.041	0.813
+1	1.024	1.075	-0.795	0.286	-1.436	1.365	0.857	-0.309	2.173	0.804	1.532	1.435	0.000	1.511	0.722	1.494	0.000	1.778	0.903	0.753	1.551	0.686	0.810	0.999	0.900	1.360	1.133	0.978
+1	2.085	-0.269	-1.423	0.789	1.298	0.281	1.652	0.187	0.000	0.658	-0.760	-0.042	2.215	0.663	0.024	0.120	0.000	0.552	-0.299	-0.428	3.102	0.713	0.811	1.130	0.705	0.218	0.675	0.743
+1	0.980	-0.443	0.813	0.785	-1.253	0.719	0.448	-1.458	0.000	1.087	0.595	0.635	1.107	1.428	0.029	-0.995	0.000	1.083	1.562	-0.092	0.000	0.834	0.891	1.165	0.967	0.661	0.880	0.817
+1	0.903	-0.733	-0.980	0.634	-0.639	0.780	0.266	-0.287	2.173	1.264	-0.936	1.004	0.000	1.002	-0.056	-1.344	2.548	1.183	-0.098	1.169	0.000	0.733	1.002	0.985	0.711	0.916	0.966	0.875
+0	0.734	-0.304	-1.175	2.851	1.674	0.904	-0.634	0.412	2.173	1.363	-1.050	-0.282	0.000	1.476	-1.603	0.103	0.000	2.231	-0.718	1.708	3.102	0.813	0.896	1.088	0.686	1.392	1.033	1.078
+1	1.680	0.591	-0.243	0.111	-0.478	0.326	-0.079	-1.555	2.173	0.711	0.714	0.922	2.215	0.355	0.858	1.682	0.000	0.727	1.620	1.360	0.000	0.334	0.526	1.001	0.862	0.633	0.660	0.619
+1	1.163	0.225	-0.202	0.501	-0.979	1.609	-0.938	1.424	0.000	1.224	-0.118	-1.274	0.000	2.034	1.241	-0.254	0.000	1.765	0.536	0.237	3.102	0.894	0.838	0.988	0.693	0.579	0.762	0.726
+0	1.223	1.232	1.471	0.489	1.728	0.703	-0.111	0.411	0.000	1.367	1.014	-1.294	1.107	1.524	-0.414	-0.164	2.548	1.292	0.833	0.316	0.000	0.861	0.752	0.994	0.836	1.814	1.089	0.950
+0	0.816	1.637	-1.557	1.036	-0.342	0.913	1.333	0.949	2.173	0.812	0.756	-0.628	2.215	1.333	0.470	1.495	0.000	1.204	-2.222	-1.675	0.000	1.013	0.924	1.133	0.758	1.304	0.855	0.860
+0	0.851	-0.564	-0.691	0.692	1.345	1.219	1.014	0.318	0.000	1.422	-0.262	-1.635	2.215	0.531	1.802	0.008	0.000	0.508	0.515	-1.267	3.102	0.821	0.787	1.026	0.783	0.432	1.149	1.034
+0	0.800	-0.599	0.204	0.552	-0.484	0.974	0.413	0.961	2.173	1.269	-0.984	-1.039	2.215	0.380	-1.213	1.371	0.000	0.551	0.332	-0.659	0.000	0.694	0.852	0.984	1.057	2.037	1.096	0.846
+0	0.744	-0.071	-0.255	0.638	0.512	1.125	0.407	0.844	2.173	0.860	-0.481	-0.677	0.000	1.102	0.181	-1.194	0.000	1.011	-1.081	-1.713	3.102	0.854	0.862	0.982	1.111	1.372	1.042	0.920
+1	0.400	1.049	-0.625	0.880	-0.407	1.040	2.150	-1.359	0.000	0.747	-0.144	0.847	2.215	0.560	-1.829	0.698	0.000	1.663	-0.668	0.267	0.000	0.845	0.964	0.996	0.820	0.789	0.668	0.668
+0	1.659	-0.705	-1.057	1.803	-1.436	1.008	0.693	0.005	0.000	0.895	-0.007	0.681	1.107	1.085	0.125	1.476	2.548	1.214	1.068	0.486	0.000	0.867	0.919	0.986	1.069	0.692	1.026	1.313
+0	0.829	-0.153	0.861	0.615	-0.548	0.589	1.077	-0.041	2.173	1.056	0.763	-1.737	0.000	0.639	0.970	0.725	0.000	0.955	1.227	-0.799	3.102	1.020	1.024	0.985	0.750	0.525	0.685	0.671
+1	0.920	-0.806	-0.840	1.048	0.278	0.973	-0.077	-1.364	2.173	1.029	0.309	0.133	0.000	1.444	1.484	1.618	1.274	1.419	-0.482	0.417	0.000	0.831	1.430	1.151	1.829	1.560	1.343	1.224
+1	0.686	0.249	-0.905	0.343	-1.731	0.724	-2.823	-0.901	0.000	0.982	0.303	1.312	1.107	1.016	0.245	0.610	0.000	1.303	-0.557	-0.360	3.102	1.384	1.030	0.984	0.862	1.144	0.866	0.779
+0	1.603	0.444	0.508	0.586	0.401	0.610	0.467	-1.735	2.173	0.914	0.626	-1.019	0.000	0.812	0.422	-0.408	2.548	0.902	1.679	1.490	0.000	1.265	0.929	0.990	1.004	0.816	0.753	0.851
+1	0.623	0.780	-0.203	0.056	0.015	0.899	0.793	1.326	1.087	0.803	1.478	-1.499	2.215	1.561	1.492	-0.120	0.000	0.904	0.795	0.137	0.000	0.548	1.009	0.850	0.924	0.838	0.914	0.860
+0	1.654	-2.032	-1.160	0.859	-1.583	0.689	-1.965	0.891	0.000	0.646	-1.014	-0.288	2.215	0.630	-0.815	0.402	0.000	0.638	0.316	0.655	3.102	0.845	0.879	0.993	1.067	0.625	1.041	0.958
+1	0.828	-1.269	-1.203	0.744	-0.213	0.626	-1.017	-0.404	0.000	1.281	-0.931	1.733	2.215	0.699	-0.351	1.287	0.000	1.251	-1.171	0.197	0.000	0.976	1.186	0.987	0.646	0.655	0.733	0.671
+1	0.677	0.111	1.090	1.580	1.591	1.560	0.654	-0.341	2.173	0.794	-0.266	0.702	0.000	0.823	0.651	-1.239	2.548	0.730	1.467	-1.530	0.000	1.492	1.023	0.983	1.909	1.022	1.265	1.127
+1	0.736	0.882	-1.060	0.589	0.168	1.663	0.781	1.022	2.173	2.025	1.648	-1.292	0.000	1.240	0.924	-0.421	1.274	1.354	0.065	0.501	0.000	0.316	0.925	0.988	0.664	1.736	0.992	0.807
+1	1.040	-0.822	1.638	0.974	-0.674	0.393	0.830	0.011	2.173	0.770	-0.140	-0.402	0.000	0.294	-0.133	0.030	0.000	1.220	0.807	0.638	0.000	0.826	1.063	1.216	1.026	0.705	0.934	0.823
+1	0.711	0.602	0.048	1.145	0.966	0.934	0.263	-1.589	2.173	0.971	-0.496	-0.421	1.107	0.628	-0.865	0.845	0.000	0.661	-0.008	-0.565	0.000	0.893	0.705	0.988	0.998	1.339	0.908	0.872
+1	0.953	-1.651	-0.167	0.885	1.053	1.013	-1.239	0.133	0.000	1.884	-1.122	1.222	2.215	1.906	-0.860	-1.184	1.274	1.413	-0.668	-1.647	0.000	1.873	1.510	1.133	1.050	1.678	1.246	1.061
+1	0.986	-0.892	-1.380	0.917	1.134	0.950	-1.162	-0.469	0.000	0.569	-1.393	0.215	0.000	0.320	2.667	1.712	0.000	1.570	-0.375	1.457	3.102	0.925	1.128	1.011	0.598	0.824	0.913	0.833
+1	1.067	0.099	1.154	0.527	-0.789	1.085	0.623	-1.602	2.173	1.511	-0.230	0.022	2.215	0.269	-0.377	0.883	0.000	0.571	-0.540	-0.512	0.000	0.414	0.803	1.022	0.959	2.053	1.041	0.780
+0	0.825	-2.118	0.217	1.453	-0.493	0.819	0.313	-0.942	0.000	2.098	-0.725	1.096	2.215	0.484	1.336	1.458	0.000	0.482	0.100	1.163	0.000	0.913	0.536	0.990	1.679	0.957	1.095	1.143
+1	1.507	0.054	1.120	0.698	-1.340	0.912	0.384	0.015	1.087	0.720	0.247	-0.820	0.000	0.286	0.154	1.578	2.548	0.629	1.582	-0.576	0.000	0.828	0.893	1.136	0.514	0.632	0.699	0.709
+1	0.610	1.180	-0.993	0.816	0.301	0.932	0.758	1.539	0.000	0.726	-0.830	0.248	2.215	0.883	0.857	-1.305	0.000	1.338	1.009	-0.252	3.102	0.901	1.074	0.987	0.875	1.159	1.035	0.858
+1	1.247	-1.360	1.502	1.525	-1.332	0.618	1.063	0.755	0.000	0.582	-0.155	0.473	2.215	1.214	-0.422	-0.551	2.548	0.838	-1.171	-1.166	0.000	2.051	1.215	1.062	1.091	0.725	0.896	1.091
+0	0.373	-0.600	1.291	2.573	0.207	0.765	-0.209	1.667	0.000	0.668	0.724	-1.499	0.000	1.045	-0.338	-0.754	2.548	0.558	-0.469	0.029	3.102	0.868	0.939	1.124	0.519	0.383	0.636	0.838
+0	0.791	0.336	-0.307	0.494	1.213	1.158	0.336	1.081	2.173	0.918	1.289	-0.449	0.000	0.735	-0.521	-0.969	0.000	1.052	0.499	-1.188	3.102	0.699	1.013	0.987	0.622	1.050	0.712	0.661
+0	1.321	0.856	0.464	0.202	0.901	1.144	0.120	-1.651	0.000	0.803	0.577	-0.509	2.215	0.695	-0.114	0.423	2.548	0.621	1.852	-0.420	0.000	0.697	0.964	0.983	0.527	0.659	0.719	0.729
+0	0.563	2.081	0.913	0.982	-0.533	0.549	-0.481	-1.730	0.000	0.962	0.921	0.569	2.215	0.731	1.184	-0.679	1.274	0.918	0.931	-1.432	0.000	1.008	0.919	0.993	0.895	0.819	0.810	0.878
+1	1.148	0.345	0.953	0.921	0.617	0.991	1.103	-0.484	0.000	0.970	1.978	1.525	0.000	1.150	0.689	-0.757	2.548	0.517	0.995	1.245	0.000	1.093	1.140	0.998	1.006	0.756	0.864	0.838
+1	1.400	0.128	-1.695	1.169	1.070	1.094	-0.345	-0.249	0.000	1.224	0.364	-0.036	2.215	1.178	0.530	-1.544	0.000	1.334	0.933	1.604	0.000	0.560	1.267	1.073	0.716	0.780	0.832	0.792
+0	0.330	-2.133	1.403	0.628	0.379	1.686	-0.995	0.030	1.087	2.071	0.127	-0.457	0.000	4.662	-0.855	1.477	0.000	2.072	-0.917	-1.416	3.102	5.403	3.074	0.977	0.936	1.910	2.325	1.702
+0	0.989	0.473	0.968	1.970	1.368	0.844	0.574	-0.290	2.173	0.866	-0.345	-1.019	0.000	1.130	0.605	-0.752	0.000	0.956	-0.888	0.870	3.102	0.885	0.886	0.982	1.157	1.201	1.100	1.068
+1	0.773	0.418	0.753	1.388	1.070	1.104	-0.378	-0.758	0.000	1.027	0.397	-0.496	2.215	1.234	0.027	1.084	2.548	0.936	0.209	1.677	0.000	1.355	1.020	0.983	0.550	1.206	0.916	0.931
+0	0.319	2.015	1.534	0.570	-1.134	0.632	0.124	0.757	0.000	0.477	0.598	-1.109	1.107	0.449	0.438	-0.755	2.548	0.574	-0.659	0.691	0.000	0.440	0.749	0.985	0.517	0.158	0.505	0.522
+0	1.215	1.453	-1.386	1.276	1.298	0.643	0.570	-0.196	2.173	0.588	2.104	0.498	0.000	0.617	-0.296	-0.801	2.548	0.452	0.110	0.313	0.000	0.815	0.953	1.141	1.166	0.547	0.892	0.807
+1	1.257	-1.869	-0.060	0.265	0.653	1.527	-0.346	1.163	2.173	0.758	-2.119	-0.604	0.000	1.473	-1.133	-1.290	2.548	0.477	-0.428	-0.066	0.000	0.818	0.841	0.984	1.446	1.729	1.211	1.054
+1	1.449	0.464	1.585	1.418	-1.488	1.540	0.942	0.087	0.000	0.898	0.402	-0.631	2.215	0.753	0.039	-1.729	0.000	0.859	0.849	-1.054	0.000	0.791	0.677	0.995	0.687	0.527	0.703	0.606
+1	1.084	-1.997	0.900	1.333	1.024	0.872	-0.864	-1.500	2.173	1.072	-0.813	-0.421	2.215	0.924	0.478	0.304	0.000	0.992	-0.398	-1.022	0.000	0.741	1.085	0.980	1.221	1.176	1.032	0.961
+0	1.712	1.129	0.125	1.120	-1.402	1.749	0.951	-1.575	2.173	1.711	0.445	0.578	0.000	1.114	0.234	-1.011	0.000	1.577	-0.088	0.086	3.102	2.108	1.312	1.882	1.597	2.009	1.441	1.308
+0	0.530	0.248	1.622	1.450	-1.012	1.221	-1.154	-0.763	2.173	1.698	-0.586	0.733	0.000	0.889	1.042	1.038	1.274	0.657	0.008	0.701	0.000	0.430	1.005	0.983	0.930	2.264	1.357	1.146
+1	0.921	1.735	0.883	0.699	-1.614	0.821	1.463	0.319	1.087	1.099	0.814	-1.600	2.215	1.375	0.702	-0.691	0.000	0.869	1.326	-0.790	0.000	0.980	0.900	0.988	0.832	1.452	0.816	0.709
+0	2.485	-0.823	-0.297	0.886	-1.404	0.989	0.835	1.615	2.173	0.382	0.588	-0.224	0.000	1.029	-0.456	1.546	2.548	0.613	-0.359	-0.789	0.000	0.768	0.977	1.726	2.007	0.913	1.338	1.180
+1	0.657	-0.069	-0.078	1.107	1.549	0.804	1.335	-1.630	2.173	1.271	0.481	0.153	1.107	1.028	0.144	-0.762	0.000	1.098	0.132	1.570	0.000	0.830	0.979	1.175	1.069	1.624	1.000	0.868
+1	2.032	0.329	-1.003	0.493	-0.136	1.159	-0.224	0.750	1.087	0.396	0.546	0.587	0.000	0.620	1.805	0.982	0.000	1.236	0.744	-1.621	0.000	0.930	1.200	0.988	0.482	0.771	0.887	0.779
+0	0.524	-1.319	0.634	0.471	1.221	0.599	-0.588	-0.461	0.000	1.230	-1.504	-1.517	1.107	1.436	-0.035	0.104	2.548	0.629	1.997	-1.282	0.000	2.084	1.450	0.984	1.084	1.827	1.547	1.213
+1	0.871	0.618	-1.544	0.718	0.186	1.041	-1.180	0.434	2.173	1.133	1.558	-1.301	0.000	0.452	-0.595	0.522	0.000	0.665	0.567	0.130	3.102	1.872	1.114	1.095	1.398	0.979	1.472	1.168
+1	3.308	1.037	-0.634	0.690	-0.619	1.975	0.949	1.280	0.000	0.826	0.546	-0.139	2.215	0.635	-0.045	0.427	0.000	1.224	0.112	1.339	3.102	1.756	1.050	0.992	0.738	0.903	0.968	1.238
+0	0.588	2.104	-0.872	1.136	1.743	0.842	0.638	0.015	0.000	0.481	0.928	1.000	2.215	0.595	0.125	1.429	0.000	0.951	-1.140	-0.511	3.102	1.031	1.057	0.979	0.673	1.064	1.001	0.891
+0	0.289	0.823	0.013	0.615	-1.601	0.177	2.403	-0.015	0.000	0.258	1.151	1.036	2.215	0.694	0.553	-1.326	2.548	0.411	0.366	0.106	0.000	0.482	0.562	0.989	0.670	0.404	0.516	0.561
+1	0.294	-0.660	-1.162	1.752	0.384	0.860	0.513	1.119	0.000	2.416	0.107	-1.342	0.000	1.398	0.361	-0.350	2.548	1.126	-0.902	0.040	1.551	0.650	1.125	0.988	0.531	0.843	0.912	0.911
+0	0.599	-0.616	1.526	1.381	0.507	0.955	-0.646	-0.085	2.173	0.775	-0.533	1.116	2.215	0.789	-0.136	-1.176	0.000	2.449	1.435	-1.433	0.000	1.692	1.699	1.000	0.869	1.119	1.508	1.303
+1	1.100	-1.174	-1.114	1.601	-1.576	1.056	-1.343	0.547	2.173	0.555	0.367	0.592	2.215	0.580	-1.862	-0.914	0.000	0.904	0.508	-0.444	0.000	1.439	1.105	0.986	1.408	1.104	1.190	1.094
+1	2.237	-0.701	1.470	0.719	-0.199	0.745	-0.132	-0.737	1.087	0.976	-0.227	0.093	2.215	0.699	0.057	1.133	0.000	0.661	0.573	-0.679	0.000	0.785	0.772	1.752	1.235	0.856	0.990	0.825
+1	0.455	-0.880	-1.482	1.260	-0.178	1.499	0.158	1.022	0.000	1.867	-0.435	-0.675	2.215	1.234	0.783	1.586	0.000	0.641	-0.454	-0.409	3.102	1.002	0.964	0.986	0.761	0.240	1.190	0.995
+1	1.158	-0.778	-0.159	0.823	1.641	1.341	-0.830	-1.169	2.173	0.840	-1.554	0.934	0.000	0.693	0.488	-1.218	2.548	1.042	1.395	0.276	0.000	0.946	0.785	1.350	1.079	0.893	1.267	1.151
+1	0.902	-0.078	-0.055	0.872	-0.012	0.843	1.276	1.739	2.173	0.838	1.492	0.918	0.000	0.626	0.904	-0.648	2.548	0.412	-2.027	-0.883	0.000	2.838	1.664	0.988	1.803	0.768	1.244	1.280
+1	0.649	-1.028	-1.521	1.097	0.774	1.216	-0.383	-0.318	2.173	1.643	-0.285	-1.705	0.000	0.911	-0.091	0.341	0.000	0.592	0.537	0.732	3.102	0.911	0.856	1.027	1.160	0.874	0.986	0.893
+1	1.192	1.846	-0.781	1.326	-0.747	1.550	1.177	1.366	0.000	1.196	0.151	0.387	2.215	0.527	2.261	-0.190	0.000	0.390	1.474	0.381	0.000	0.986	1.025	1.004	1.392	0.761	0.965	1.043
+0	0.438	-0.358	-1.549	0.836	0.436	0.818	0.276	-0.708	2.173	0.707	0.826	0.392	0.000	1.050	1.741	-1.066	0.000	1.276	-1.583	0.842	0.000	1.475	1.273	0.986	0.853	1.593	1.255	1.226
+1	1.083	0.142	1.701	0.605	-0.253	1.237	0.791	1.183	2.173	0.842	2.850	-0.082	0.000	0.724	-0.464	-0.694	0.000	1.499	0.456	-0.226	3.102	0.601	0.799	1.102	0.995	1.389	1.013	0.851
+0	0.828	1.897	-0.615	0.572	-0.545	0.572	0.461	0.464	2.173	0.393	0.356	1.069	2.215	1.840	0.088	1.500	0.000	0.407	-0.663	-0.787	0.000	0.950	0.965	0.979	0.733	0.363	0.618	0.733
+0	0.735	1.438	1.197	1.123	-0.214	0.641	0.949	0.858	0.000	1.162	0.524	-0.896	2.215	0.992	0.454	-1.475	2.548	0.902	1.079	0.019	0.000	0.822	0.917	1.203	1.032	0.569	0.780	0.764
+0	0.437	-2.102	0.044	1.779	-1.042	1.231	-0.181	-0.515	1.087	2.666	0.863	1.466	2.215	1.370	0.345	-1.371	0.000	0.906	0.363	1.611	0.000	1.140	1.362	1.013	3.931	3.004	2.724	2.028
+1	0.881	1.814	-0.987	0.384	0.800	2.384	1.422	0.640	0.000	1.528	0.292	-0.962	1.107	2.126	-0.371	-1.401	2.548	0.700	0.109	0.203	0.000	0.450	0.813	0.985	0.956	1.013	0.993	0.774
+1	0.630	0.408	0.152	0.194	0.316	0.710	-0.824	-0.358	2.173	0.741	0.535	-0.851	2.215	0.933	0.406	1.148	0.000	0.523	-0.479	-0.625	0.000	0.873	0.960	0.988	0.830	0.921	0.711	0.661
+1	0.870	-0.448	-1.134	0.616	0.135	0.600	0.649	-0.622	2.173	0.768	0.709	-0.123	0.000	1.308	0.500	1.468	0.000	1.973	-0.286	1.462	3.102	0.909	0.944	0.990	0.835	1.250	0.798	0.776
+0	1.290	0.552	1.330	0.615	-1.353	0.661	0.240	-0.393	0.000	0.531	0.053	-1.588	0.000	0.675	0.839	-0.345	1.274	1.597	0.020	0.536	3.102	1.114	0.964	0.987	0.783	0.675	0.662	0.675
+1	0.943	0.936	1.068	1.373	0.671	2.170	-2.011	-1.032	0.000	0.640	0.361	-0.806	0.000	2.239	-0.083	0.590	2.548	1.224	0.646	-1.723	0.000	0.879	0.834	0.981	1.436	0.568	0.916	0.931
+1	0.431	1.686	-1.053	0.388	1.739	0.457	-0.471	-0.743	2.173	0.786	1.432	-0.547	2.215	0.537	-0.413	1.256	0.000	0.413	2.311	-0.408	0.000	1.355	1.017	0.982	0.689	1.014	0.821	0.715
+0	1.620	-0.055	-0.862	1.341	-1.571	0.634	-0.906	0.935	2.173	0.501	-2.198	-0.525	0.000	0.778	-0.708	-0.060	0.000	0.988	-0.621	0.489	3.102	0.870	0.956	1.216	0.992	0.336	0.871	0.889
+1	0.549	0.304	-1.443	1.309	-0.312	1.116	0.644	1.519	2.173	1.078	-0.303	-0.736	0.000	1.261	0.387	0.628	2.548	0.945	-0.190	0.090	0.000	0.893	1.043	1.000	1.124	1.077	1.026	0.886
+0	0.412	-0.618	-1.486	1.133	-0.665	0.646	0.436	1.520	0.000	0.993	0.976	0.106	2.215	0.832	0.091	0.164	2.548	0.672	-0.650	1.256	0.000	0.695	1.131	0.991	1.017	0.455	1.226	1.087
+0	1.183	-0.084	1.644	1.389	0.967	0.843	0.938	-0.670	0.000	0.480	0.256	0.123	2.215	0.437	1.644	0.491	0.000	0.501	-0.416	0.101	3.102	1.060	0.804	1.017	0.775	0.173	0.535	0.760
+0	1.629	-1.486	-0.683	2.786	-0.492	1.347	-2.638	1.453	0.000	1.857	0.208	0.873	0.000	0.519	-1.265	-1.602	1.274	0.903	-1.102	-0.329	1.551	6.892	3.522	0.998	0.570	0.477	2.039	2.006
+1	2.045	-0.671	-1.235	0.490	-0.952	0.525	-1.252	1.289	0.000	1.088	-0.993	0.648	2.215	0.975	-0.109	-0.254	2.548	0.556	-1.095	-0.194	0.000	0.803	0.861	0.980	1.282	0.945	0.925	0.811
+0	0.448	-0.058	-0.974	0.945	-1.633	1.181	-1.139	0.266	2.173	1.118	-0.761	1.502	1.107	1.706	0.585	-0.680	0.000	0.487	-1.951	0.945	0.000	2.347	1.754	0.993	1.161	1.549	1.414	1.176
+0	0.551	0.519	0.448	2.183	1.293	1.220	0.628	-0.627	2.173	1.019	-0.002	-0.652	0.000	1.843	-0.386	1.042	2.548	0.400	-1.102	-1.014	0.000	0.648	0.792	1.049	0.888	2.132	1.262	1.096
+0	1.624	0.488	1.403	0.760	0.559	0.812	0.777	-1.244	2.173	0.613	0.589	-0.030	2.215	0.692	1.058	0.683	0.000	1.054	1.165	-0.765	0.000	0.915	0.875	1.059	0.821	0.927	0.792	0.721
+1	0.774	0.444	1.257	0.515	-0.689	0.515	1.448	-1.271	0.000	0.793	0.118	0.811	1.107	0.679	0.326	-0.426	0.000	1.066	-0.865	-0.049	3.102	0.960	1.046	0.986	0.716	0.772	0.855	0.732
+1	2.093	-1.240	1.615	0.918	-1.202	1.412	-0.541	0.640	1.087	2.019	0.872	-0.639	0.000	0.672	-0.936	0.972	0.000	0.896	0.235	0.212	0.000	0.810	0.700	1.090	0.797	0.862	1.049	0.874
+1	0.908	1.069	0.283	0.400	1.293	0.609	1.452	-1.136	0.000	0.623	0.417	-0.098	2.215	1.023	0.775	1.054	1.274	0.706	2.346	-1.305	0.000	0.744	1.006	0.991	0.606	0.753	0.796	0.753
+0	0.403	-1.328	-0.065	0.901	1.052	0.708	-0.354	-0.718	2.173	0.892	0.633	1.684	2.215	0.999	-1.205	0.941	0.000	0.930	1.072	-0.809	0.000	2.105	1.430	0.989	0.838	1.147	1.042	0.883
+0	1.447	0.453	0.118	1.731	0.650	0.771	0.446	-1.564	0.000	0.973	-2.014	0.354	0.000	1.949	-0.643	-1.531	1.274	1.106	-0.334	-1.163	0.000	0.795	0.821	1.013	1.699	0.918	1.118	1.018
+1	1.794	0.123	-0.454	0.057	1.489	0.966	-1.190	1.090	1.087	0.539	-0.535	1.035	0.000	1.096	-1.069	-1.236	2.548	0.659	-1.196	-0.283	0.000	0.803	0.756	0.985	1.343	1.109	0.993	0.806
+0	1.484	-2.047	0.813	0.591	-0.295	0.923	0.312	-1.164	2.173	0.654	-0.316	0.752	2.215	0.599	1.966	-1.128	0.000	0.626	-0.304	-1.431	0.000	1.112	0.910	1.090	0.986	1.189	1.350	1.472
+0	0.417	-2.016	0.849	1.817	0.040	1.201	-1.676	-1.394	0.000	0.792	0.537	0.641	2.215	0.794	-1.222	0.187	0.000	0.825	-0.217	1.334	3.102	1.470	0.931	0.987	1.203	0.525	0.833	0.827
+1	0.603	1.009	0.033	0.486	1.225	0.884	-0.617	-1.058	0.000	0.500	-1.407	-0.567	0.000	1.476	-0.876	0.605	2.548	0.970	0.560	1.092	3.102	0.853	1.153	0.988	0.846	0.920	0.944	0.835
+1	1.381	-0.326	0.552	0.417	-0.027	1.030	-0.835	-1.287	2.173	0.941	-0.421	1.519	2.215	0.615	-1.650	0.377	0.000	0.606	0.644	0.650	0.000	1.146	0.970	0.990	1.191	0.884	0.897	0.826
+1	0.632	1.200	-0.703	0.438	-1.700	0.779	-0.731	0.958	1.087	0.605	0.393	-1.376	0.000	0.670	-0.827	-1.315	2.548	0.626	-0.501	0.417	0.000	0.904	0.903	0.998	0.673	0.803	0.722	0.640
+1	1.561	-0.569	1.580	0.329	0.237	1.059	0.731	0.415	2.173	0.454	0.016	-0.828	0.000	0.587	0.008	-0.291	1.274	0.597	1.119	1.191	0.000	0.815	0.908	0.988	0.733	0.690	0.892	0.764
+1	2.102	0.087	0.449	1.164	-0.390	1.085	-0.408	-1.116	2.173	0.578	0.197	-0.137	0.000	1.202	0.917	1.523	0.000	0.959	-0.832	1.404	3.102	1.380	1.109	1.486	1.496	0.886	1.066	1.025
+1	1.698	-0.489	-0.552	0.976	-1.009	1.620	-0.721	0.648	1.087	1.481	-1.860	-1.354	0.000	1.142	-1.140	1.401	2.548	1.000	-1.274	-0.158	0.000	1.430	1.130	0.987	1.629	1.154	1.303	1.223
+1	1.111	-0.249	-1.457	0.421	0.939	0.646	-2.076	0.362	0.000	1.315	0.796	-1.436	2.215	0.780	0.130	0.055	0.000	1.662	-0.834	0.461	0.000	0.920	0.948	0.990	1.046	0.905	1.493	1.169
+1	0.945	0.390	-1.159	1.675	0.437	0.356	0.261	0.543	1.087	0.574	0.838	1.599	2.215	0.496	-1.220	-0.022	0.000	0.558	-2.454	1.440	0.000	0.763	0.983	1.728	1.000	0.578	0.922	1.003
+1	2.076	0.014	-1.314	0.854	-0.306	3.446	1.341	0.598	0.000	2.086	0.227	-0.747	2.215	1.564	-0.216	1.649	2.548	0.965	-0.857	-1.062	0.000	0.477	0.734	1.456	1.003	1.660	1.001	0.908
+1	1.992	0.192	-0.103	0.108	-1.599	0.938	0.595	-1.360	2.173	0.869	-1.012	1.432	0.000	1.302	0.850	0.436	2.548	0.487	1.051	-1.027	0.000	0.502	0.829	0.983	1.110	1.394	0.904	0.836
+0	0.460	1.625	1.485	1.331	1.242	0.675	-0.329	-1.039	1.087	0.671	-1.028	-0.514	0.000	1.265	-0.788	0.415	1.274	0.570	-0.683	-1.738	0.000	0.725	0.758	1.004	1.024	1.156	0.944	0.833
+0	0.871	0.839	-1.536	0.428	1.198	0.875	-1.256	-0.466	1.087	0.684	-0.768	0.150	0.000	0.556	-1.793	0.389	0.000	0.942	-1.126	1.339	1.551	0.624	0.734	0.986	1.357	0.960	1.474	1.294
+1	0.951	1.651	0.576	1.273	1.495	0.834	0.048	-0.578	2.173	0.386	-0.056	-1.448	0.000	0.597	-0.196	0.162	2.548	0.524	1.649	1.625	0.000	0.737	0.901	1.124	1.014	0.556	1.039	0.845
+1	1.049	-0.223	0.685	0.256	-1.191	2.506	0.238	-0.359	0.000	1.510	-0.904	1.158	1.107	2.733	-0.902	1.679	2.548	0.407	-0.474	-1.572	0.000	1.513	2.472	0.982	1.238	0.978	1.985	1.510
+0	0.455	-0.028	0.265	1.286	1.373	0.459	0.331	-0.922	0.000	0.343	0.634	0.430	0.000	0.279	-0.084	-0.272	0.000	0.475	0.926	-0.123	3.102	0.803	0.495	0.987	0.587	0.211	0.417	0.445
+1	2.074	0.388	0.878	1.110	1.557	1.077	-0.226	-0.295	2.173	0.865	-0.319	-1.116	2.215	0.707	-0.835	0.722	0.000	0.632	-0.608	-0.728	0.000	0.715	0.802	1.207	1.190	0.960	1.143	0.926
+1	1.390	0.265	1.196	0.919	-1.371	1.858	0.506	0.786	0.000	1.280	-1.367	-0.720	2.215	1.483	-0.441	-0.675	2.548	1.076	0.294	-0.539	0.000	1.126	0.830	1.155	1.551	0.702	1.103	0.933
+1	1.014	-0.079	1.597	1.038	-0.281	1.135	-0.722	-0.177	2.173	0.544	-1.475	-1.501	0.000	1.257	-1.315	1.212	0.000	0.496	-0.060	1.180	1.551	0.815	0.611	1.411	1.110	0.792	0.846	0.853
+0	0.335	1.267	-1.154	2.011	-0.574	0.753	0.618	1.411	0.000	0.474	0.748	0.681	2.215	0.608	-0.446	-0.354	2.548	0.399	1.295	-0.581	0.000	0.911	0.882	0.975	0.832	0.598	0.580	0.678
+1	0.729	-0.189	1.182	0.293	1.310	0.412	0.459	-0.632	0.000	0.869	-1.128	-0.625	2.215	1.173	-0.893	0.478	2.548	0.584	-2.394	-1.727	0.000	2.016	1.272	0.995	1.034	0.905	0.966	1.038
+1	1.225	-1.215	-0.088	0.881	-0.237	0.600	-0.976	1.462	2.173	0.876	0.506	1.583	2.215	0.718	1.228	-0.031	0.000	0.653	-1.292	1.216	0.000	0.838	1.108	0.981	1.805	0.890	1.251	1.197
+1	2.685	-0.444	0.847	0.253	0.183	0.641	-1.541	-0.873	2.173	0.417	2.874	-0.551	0.000	0.706	-1.431	0.764	0.000	1.390	-0.596	-1.397	0.000	0.894	0.829	0.993	0.789	0.654	0.883	0.746
+0	0.638	-0.481	0.683	1.457	-1.024	0.707	-1.338	1.498	0.000	0.980	0.518	0.289	2.215	0.964	-0.531	-0.423	0.000	0.694	-0.654	-1.314	3.102	0.807	1.283	1.335	0.658	0.907	0.797	0.772
+1	1.789	-0.765	-0.732	0.421	-0.020	1.142	-1.353	1.439	2.173	0.725	-1.518	-1.261	0.000	0.812	-2.597	-0.463	0.000	1.203	-0.120	1.001	0.000	0.978	0.673	0.985	1.303	1.400	1.078	0.983
+1	0.784	-1.431	1.724	0.848	0.559	0.615	-1.643	-1.456	0.000	1.339	-0.513	0.040	2.215	0.394	-2.483	1.304	0.000	0.987	0.889	-0.339	0.000	0.732	0.713	0.987	0.973	0.705	0.875	0.759
+1	0.911	1.098	-1.289	0.421	0.823	1.218	-0.503	0.431	0.000	0.775	0.432	-1.680	0.000	0.855	-0.226	-0.460	2.548	0.646	-0.947	-1.243	1.551	2.201	1.349	0.985	0.730	0.451	0.877	0.825
+1	0.959	0.372	-0.269	1.255	0.702	1.151	0.097	0.805	2.173	0.993	1.011	0.767	2.215	1.096	0.185	0.381	0.000	1.001	-0.205	0.059	0.000	0.979	0.997	1.168	0.796	0.771	0.839	0.776
+0	0.283	-1.864	-1.663	0.219	1.624	0.955	-1.213	0.932	2.173	0.889	0.395	-0.268	0.000	0.597	-1.083	-0.921	2.548	0.584	1.325	-1.072	0.000	0.856	0.927	0.996	0.937	0.936	1.095	0.892
+0	2.017	-0.488	-0.466	1.029	-0.870	3.157	0.059	-0.343	2.173	3.881	0.872	1.502	1.107	3.631	1.720	0.963	0.000	0.633	-1.264	-1.734	0.000	4.572	3.339	1.005	1.407	5.590	3.614	3.110
+1	1.088	0.414	-0.841	0.485	0.605	0.860	1.110	-0.568	0.000	1.152	-0.325	1.203	2.215	0.324	1.652	-0.104	0.000	0.510	1.095	-1.728	0.000	0.880	0.722	0.989	0.977	0.711	0.888	0.762
+0	0.409	-1.717	0.712	0.809	-1.301	0.701	-1.529	-1.411	0.000	1.191	-0.582	0.438	2.215	1.147	0.813	-0.571	2.548	1.039	0.543	0.892	0.000	0.636	0.810	0.986	0.861	1.411	0.907	0.756
+1	1.094	1.577	-0.988	0.497	-0.149	0.891	-2.459	1.034	0.000	0.646	0.792	-1.022	0.000	1.573	0.254	-0.053	2.548	1.428	0.190	-1.641	3.102	4.322	2.687	0.985	0.881	1.135	1.907	1.831
+1	0.613	1.993	-0.280	0.544	0.931	0.909	1.526	1.559	0.000	0.840	1.473	-0.483	2.215	0.856	0.352	0.408	2.548	1.058	1.733	-1.396	0.000	0.801	1.066	0.984	0.639	0.841	0.871	0.748
+0	0.958	-1.202	0.600	0.434	0.170	0.783	-0.214	1.319	0.000	0.835	-0.454	-0.615	2.215	0.658	-1.858	-0.891	0.000	0.640	0.172	-1.204	3.102	1.790	1.086	0.997	0.804	0.403	0.793	0.756
+1	1.998	-0.238	0.972	0.058	0.266	0.759	1.576	-0.357	2.173	1.004	-0.349	-0.747	2.215	0.962	0.490	-0.453	0.000	1.592	0.661	-1.405	0.000	0.874	1.086	0.990	1.436	1.527	1.177	0.993
+1	0.796	-0.171	-0.818	0.574	-1.625	1.201	-0.737	1.451	2.173	0.651	0.404	-0.452	0.000	1.150	-0.652	-0.120	0.000	1.008	-0.093	0.531	3.102	0.884	0.706	0.979	1.193	0.937	0.943	0.881
+1	0.773	1.023	0.527	1.537	-0.201	2.967	-0.574	-1.534	2.173	2.346	-0.307	0.394	2.215	1.393	0.135	-0.027	0.000	3.015	0.187	0.516	0.000	0.819	1.260	0.982	2.552	3.862	2.179	1.786
+0	1.823	1.008	-1.489	0.234	-0.962	0.591	0.461	0.996	2.173	0.568	-1.297	-0.410	0.000	0.887	2.157	1.194	0.000	2.079	0.369	-0.085	3.102	0.770	0.945	0.995	1.179	0.971	0.925	0.983
+0	0.780	0.640	0.490	0.680	-1.301	0.715	-0.137	0.152	2.173	0.616	-0.831	1.668	0.000	1.958	0.528	-0.982	2.548	0.966	-1.551	0.462	0.000	1.034	1.079	1.008	0.827	1.369	1.152	0.983
+1	0.543	0.801	1.543	1.134	-0.772	0.954	-0.849	0.410	1.087	0.851	-1.988	1.686	0.000	0.799	-0.912	-1.156	0.000	0.479	0.097	1.334	0.000	0.923	0.597	0.989	1.231	0.759	0.975	0.867
+0	1.241	-0.014	0.129	1.158	0.670	0.445	-0.732	1.739	2.173	0.918	0.659	-1.340	2.215	0.557	2.410	-1.404	0.000	0.966	-1.545	-1.120	0.000	0.874	0.918	0.987	1.001	0.798	0.904	0.937
+0	1.751	-0.266	-1.575	0.489	1.292	1.112	1.533	0.137	2.173	1.204	-0.414	-0.928	0.000	0.879	1.237	-0.415	2.548	1.479	1.469	0.913	0.000	2.884	1.747	0.989	1.742	0.600	1.363	1.293
+1	1.505	1.208	-1.476	0.995	-0.836	2.800	-1.600	0.111	0.000	2.157	1.241	1.110	2.215	1.076	2.619	-0.913	0.000	1.678	2.204	-1.575	0.000	0.849	1.224	0.990	1.412	0.976	1.271	1.105
+0	0.816	0.611	0.779	1.694	0.278	0.575	-0.787	1.592	2.173	1.148	1.076	-0.831	2.215	0.421	1.316	0.632	0.000	0.589	0.452	-1.466	0.000	0.779	0.909	0.990	1.146	1.639	1.236	0.949
+1	0.551	-0.808	0.330	1.188	-0.294	0.447	-0.035	-0.993	0.000	0.432	-0.276	-0.481	2.215	1.959	-0.288	1.195	2.548	0.638	0.583	1.107	0.000	0.832	0.924	0.993	0.723	0.976	0.968	0.895
+0	1.316	-0.093	0.995	0.860	-0.621	0.593	-0.560	-1.599	2.173	0.524	-0.318	-0.240	2.215	0.566	0.759	-0.368	0.000	0.483	-2.030	-1.104	0.000	1.468	1.041	1.464	0.811	0.778	0.690	0.722
+1	1.528	0.067	-0.855	0.959	-1.464	1.143	-0.082	1.023	0.000	0.702	-0.763	-0.244	0.000	0.935	-0.881	0.206	2.548	0.614	-0.831	1.657	3.102	1.680	1.105	0.983	1.078	0.559	0.801	0.809
+0	0.558	-0.833	-0.598	1.436	-1.724	1.316	-0.661	1.593	2.173	1.148	-0.503	-0.132	1.107	1.584	-0.125	0.380	0.000	1.110	-1.216	-0.181	0.000	1.258	0.860	1.053	0.790	1.814	1.159	1.007
+1	0.819	0.879	1.221	0.598	-1.450	0.754	0.417	-0.369	2.173	0.477	1.199	0.274	0.000	1.073	0.368	0.273	2.548	1.599	2.047	1.690	0.000	0.933	0.984	0.983	0.788	0.613	0.728	0.717
+0	0.981	-1.007	0.489	0.923	1.261	0.436	-0.698	-0.506	2.173	0.764	-1.105	-1.241	2.215	0.577	-2.573	-0.036	0.000	0.565	-1.628	1.610	0.000	0.688	0.801	0.991	0.871	0.554	0.691	0.656
+0	2.888	0.568	-1.416	1.461	-1.157	1.756	-0.900	0.522	0.000	0.657	0.409	1.076	2.215	1.419	0.672	-0.019	0.000	1.436	-0.184	-0.980	3.102	0.946	0.919	0.995	1.069	0.890	0.834	0.856
+1	0.522	1.805	-0.963	1.136	0.418	0.727	-0.195	-1.695	2.173	0.309	2.559	-0.178	0.000	0.521	1.794	0.919	0.000	0.788	0.174	-0.406	3.102	0.555	0.729	1.011	1.385	0.753	0.927	0.832
+1	0.793	-0.162	-1.643	0.634	0.337	0.898	-0.633	1.689	0.000	0.806	-0.826	-0.356	2.215	0.890	-0.142	-1.268	0.000	1.293	0.574	0.725	0.000	0.833	1.077	0.988	0.721	0.679	0.867	0.753
+0	1.298	1.098	0.280	0.371	-0.373	0.855	-0.306	-1.186	0.000	0.977	-0.421	1.003	0.000	0.978	0.956	-1.249	2.548	0.735	0.577	-0.037	3.102	0.974	1.002	0.992	0.549	0.587	0.725	0.954
+1	0.751	-0.520	-1.653	0.168	-0.419	0.878	-1.023	-1.364	2.173	1.310	-0.667	0.863	0.000	1.196	-0.827	0.358	0.000	1.154	-0.165	-0.360	1.551	0.871	0.950	0.983	0.907	0.955	0.959	0.874
+0	1.730	0.666	-1.432	0.446	1.302	0.921	-0.203	0.621	0.000	1.171	-0.365	-0.611	1.107	0.585	0.807	1.150	0.000	0.415	-0.843	1.311	0.000	0.968	0.786	0.986	1.059	0.371	0.790	0.848
+1	0.596	-1.486	0.690	1.045	-1.344	0.928	0.867	0.820	2.173	0.610	0.999	-1.329	2.215	0.883	-0.001	-0.106	0.000	1.145	2.184	-0.808	0.000	2.019	1.256	1.056	1.751	1.037	1.298	1.518
+1	0.656	-1.993	-0.519	1.643	-0.143	0.815	0.256	1.220	1.087	0.399	-1.184	-1.458	0.000	0.738	1.361	-1.443	0.000	0.842	0.033	0.293	0.000	0.910	0.891	0.993	0.668	0.562	0.958	0.787
+1	1.127	-0.542	0.645	0.318	-1.496	0.661	-0.640	0.369	2.173	0.992	0.358	1.702	0.000	1.004	0.316	-1.109	0.000	1.616	-0.936	-0.707	1.551	0.875	1.191	0.985	0.651	0.940	0.969	0.834
+0	0.916	-1.423	-1.490	1.248	-0.538	0.625	-0.535	-0.174	0.000	0.769	-0.389	1.608	2.215	0.667	-1.138	-1.738	1.274	0.877	-0.019	0.482	0.000	0.696	0.917	1.121	0.678	0.347	0.647	0.722
+1	2.756	-0.637	-1.715	1.331	1.124	0.913	-0.296	-0.491	0.000	0.983	-0.831	0.000	2.215	1.180	-0.428	0.742	0.000	1.113	0.005	-1.157	1.551	1.681	1.096	1.462	0.976	0.917	1.009	1.040
+0	0.755	1.754	0.701	2.111	0.256	1.243	0.057	-1.502	2.173	0.565	-0.034	-1.078	1.107	0.529	1.696	-1.090	0.000	0.665	0.292	0.107	0.000	0.870	0.780	0.990	2.775	0.465	1.876	1.758
+1	0.593	-0.762	1.743	0.908	0.442	0.773	-1.357	-0.768	2.173	0.432	1.421	1.236	0.000	0.579	0.291	-0.403	0.000	0.966	-0.309	1.016	3.102	0.893	0.743	0.989	0.857	1.030	0.943	0.854
+1	0.891	-1.151	-1.269	0.504	-0.622	0.893	-0.549	0.700	0.000	0.828	-0.825	0.154	2.215	1.083	0.632	-1.141	0.000	1.059	-0.557	1.526	3.102	2.117	1.281	0.987	0.819	0.802	0.917	0.828
+1	2.358	-0.248	0.080	0.747	-0.975	1.019	1.374	1.363	0.000	0.935	0.127	-1.707	2.215	0.312	-0.827	0.017	0.000	0.737	1.059	-0.327	0.000	0.716	0.828	1.495	0.953	0.704	0.880	0.745
+0	0.660	-0.017	-1.138	0.453	1.002	0.645	0.518	0.703	2.173	0.751	0.705	-0.592	2.215	0.744	-0.909	-1.596	0.000	0.410	-1.135	0.481	0.000	0.592	0.922	0.989	0.897	0.948	0.777	0.701
+1	0.718	0.518	0.225	1.710	-0.022	1.888	-0.424	1.092	0.000	4.134	0.185	-1.366	0.000	1.415	1.293	0.242	2.548	2.351	0.264	-0.057	3.102	0.830	1.630	0.976	1.215	0.890	1.422	1.215
+1	1.160	0.203	0.941	0.594	0.212	0.636	-0.556	0.679	2.173	1.089	-0.481	-1.008	1.107	1.245	-0.056	-1.357	0.000	0.587	1.007	0.056	0.000	1.106	0.901	0.987	0.786	1.224	0.914	0.837
+1	0.697	0.542	0.619	0.985	1.481	0.745	0.415	1.644	2.173	0.903	0.495	-0.958	2.215	1.165	1.195	0.346	0.000	1.067	-0.881	-0.264	0.000	0.830	1.025	0.987	0.690	0.863	0.894	0.867
+0	1.430	0.190	-0.700	0.246	0.518	1.302	0.660	-0.247	2.173	1.185	-0.539	1.504	0.000	1.976	-0.401	1.079	0.000	0.855	-0.958	-1.110	3.102	0.886	0.953	0.993	0.889	1.400	1.376	1.119
+1	1.122	-0.795	0.202	0.397	-1.553	0.597	-1.459	-0.734	2.173	0.522	1.044	1.027	2.215	0.783	-1.243	1.701	0.000	0.371	1.737	0.199	0.000	1.719	1.176	0.988	0.723	1.583	1.063	0.914
+0	1.153	0.526	1.236	0.266	0.001	1.139	-1.236	-0.585	2.173	1.337	-0.215	-1.356	2.215	1.780	1.129	0.902	0.000	1.608	-0.391	-0.161	0.000	1.441	1.633	0.990	1.838	1.516	1.635	1.373
+1	0.760	1.012	0.758	0.937	0.051	0.941	0.687	-1.247	2.173	1.288	-0.743	0.822	0.000	1.552	1.782	-1.533	0.000	0.767	1.349	0.168	0.000	0.716	0.862	0.988	0.595	0.359	0.697	0.623
+1	1.756	-1.469	1.395	1.345	-1.595	0.817	0.017	-0.741	2.173	0.483	-0.008	0.293	0.000	1.768	-0.663	0.438	1.274	1.202	-1.387	-0.222	0.000	1.022	1.058	0.992	1.407	1.427	1.356	1.133
+0	0.397	0.582	-0.758	1.260	-1.735	0.889	-0.515	1.139	2.173	0.973	1.616	0.460	0.000	1.308	1.001	-0.709	2.548	0.858	0.995	-0.231	0.000	0.749	0.888	0.979	1.487	1.804	1.208	1.079
+0	0.515	-0.984	0.425	1.114	-0.439	1.999	0.818	1.561	0.000	1.407	0.009	-0.380	0.000	1.332	0.230	0.397	0.000	1.356	-0.616	-1.057	3.102	0.978	1.017	0.990	1.118	0.862	0.835	0.919
+1	1.368	-0.921	-0.866	0.842	-0.598	0.456	-1.176	1.219	1.087	0.419	-1.974	-0.819	0.000	0.791	-1.640	0.881	0.000	1.295	-0.782	0.442	3.102	0.945	0.761	0.974	0.915	0.535	0.733	0.651
+0	2.276	0.134	0.399	2.525	0.376	1.111	-1.078	-1.571	0.000	0.657	2.215	-0.900	0.000	1.183	-0.662	-0.508	2.548	1.436	-0.517	0.960	3.102	0.569	0.931	0.993	1.170	0.967	0.879	1.207
+0	0.849	0.907	0.124	0.652	1.585	0.715	0.355	-1.200	0.000	0.599	-0.892	1.301	0.000	1.106	1.151	0.582	0.000	1.895	-0.279	-0.568	3.102	0.881	0.945	0.998	0.559	0.649	0.638	0.660
+1	2.105	0.248	-0.797	0.530	0.206	1.957	-2.175	0.797	0.000	1.193	0.637	-1.646	2.215	0.881	1.111	-1.046	0.000	0.872	-0.185	1.085	1.551	0.986	1.343	1.151	1.069	0.714	2.063	1.951
+1	1.838	1.060	1.637	1.017	1.370	0.913	0.461	-0.609	1.087	0.766	-0.461	0.303	2.215	0.724	-0.061	0.886	0.000	0.941	1.123	-0.745	0.000	0.858	0.847	0.979	1.313	1.083	1.094	0.910
+0	0.364	1.274	1.066	1.570	-0.394	0.485	0.012	-1.716	0.000	0.317	-1.233	0.534	2.215	0.548	-2.165	0.762	0.000	0.729	0.169	-0.318	3.102	0.892	0.944	1.013	0.594	0.461	0.688	0.715
+1	0.503	1.343	-0.031	1.134	-1.204	0.590	-0.309	0.174	2.173	0.408	2.372	-0.628	0.000	1.850	0.400	1.147	2.548	0.664	-0.458	-0.885	0.000	1.445	1.283	0.989	1.280	1.118	1.127	1.026
+0	1.873	0.258	0.103	2.491	0.530	1.678	0.644	-1.738	2.173	1.432	0.848	-1.340	0.000	0.621	1.323	-1.316	0.000	0.628	0.789	-0.206	1.551	0.426	0.802	1.125	0.688	1.079	1.338	1.239
+1	0.826	-0.732	1.587	0.582	-1.236	0.495	0.757	-0.741	2.173	0.940	1.474	0.354	2.215	0.474	1.055	-1.657	0.000	0.415	1.758	0.841	0.000	0.451	0.578	0.984	0.757	0.922	0.860	0.696
+0	0.935	-1.614	-0.597	0.299	1.223	0.707	-0.853	-1.026	0.000	0.751	0.007	-1.691	0.000	1.062	-0.125	0.976	2.548	0.877	1.275	0.646	0.000	0.962	1.074	0.980	0.608	0.726	0.741	0.662
+1	0.643	0.542	-1.285	0.474	-0.366	0.667	-0.446	1.195	2.173	1.076	0.145	-0.126	0.000	0.970	-0.661	0.394	1.274	1.218	-0.184	-1.722	0.000	1.331	1.019	0.985	1.192	0.677	0.973	0.910
+0	0.713	0.164	1.080	1.427	-0.460	0.960	-0.152	-0.940	2.173	1.427	-0.901	1.036	1.107	0.440	-1.269	-0.194	0.000	0.452	1.932	-0.532	0.000	1.542	1.210	1.374	1.319	1.818	1.220	1.050
+0	0.876	-0.463	-1.224	2.458	-1.689	1.007	-0.752	0.398	0.000	2.456	-1.285	-0.152	1.107	1.641	1.838	1.717	0.000	0.458	0.194	0.488	3.102	4.848	2.463	0.986	1.981	0.974	2.642	2.258
+1	0.384	-0.275	0.387	1.403	-0.994	0.620	-1.529	1.685	0.000	1.091	-1.644	1.078	0.000	0.781	-1.311	0.326	2.548	1.228	-0.728	-0.633	1.551	0.920	0.854	0.987	0.646	0.609	0.740	0.884
+0	0.318	-1.818	-1.008	0.977	1.268	0.457	2.451	-1.522	0.000	0.881	1.351	0.461	2.215	0.929	0.239	-0.380	2.548	0.382	-0.613	1.330	0.000	1.563	1.193	0.994	0.829	0.874	0.901	1.026
+1	0.612	-1.120	1.098	0.402	-0.480	0.818	0.188	1.511	0.000	0.800	-0.253	0.977	0.000	1.175	0.271	-1.289	1.274	2.531	0.226	-0.409	3.102	0.889	0.947	0.979	1.486	0.940	1.152	1.119
+1	0.587	-0.737	-0.228	0.970	1.119	0.823	0.184	1.594	0.000	1.104	0.301	-0.818	2.215	0.819	0.712	-0.560	0.000	2.240	-0.419	0.340	3.102	1.445	1.103	0.988	0.715	1.363	1.019	0.926
+0	1.030	-0.694	-1.638	0.893	-1.074	1.160	-0.766	0.485	0.000	1.632	-0.698	-1.142	2.215	1.050	-1.092	0.952	0.000	1.475	0.286	0.125	3.102	0.914	1.075	0.982	0.732	1.493	1.219	1.079
+1	2.142	0.617	1.517	0.387	-0.862	0.345	1.203	-1.014	2.173	0.609	1.092	0.275	0.000	1.331	0.582	-0.183	2.548	0.557	1.540	-1.642	0.000	0.801	0.737	1.060	0.715	0.626	0.749	0.674
+0	1.076	0.240	-0.246	0.871	-1.241	0.496	0.282	0.746	2.173	1.095	-0.648	1.100	2.215	0.446	-1.756	0.764	0.000	0.434	0.788	-0.991	0.000	1.079	0.868	1.047	0.818	0.634	0.795	0.733
+0	1.400	0.901	-1.617	0.625	-0.163	0.661	-0.411	-1.616	2.173	0.685	0.524	0.425	0.000	0.881	-0.766	0.312	0.000	0.979	0.255	-0.667	3.102	0.898	1.105	1.253	0.730	0.716	0.738	0.795
+0	3.302	1.132	1.051	0.658	0.768	1.308	0.251	-0.374	1.087	1.673	0.015	-0.898	0.000	0.688	-0.535	1.363	1.274	0.871	1.325	-1.583	0.000	1.646	1.249	0.995	1.919	1.288	1.330	1.329
+0	1.757	0.202	0.750	0.767	-0.362	0.932	-1.033	-1.366	0.000	1.529	-1.012	-0.771	0.000	1.161	-0.287	0.059	0.000	2.185	1.147	1.099	3.102	0.795	0.529	1.354	1.144	1.491	1.319	1.161
+0	1.290	0.905	-1.711	1.017	-0.695	1.008	-1.038	0.693	2.173	1.202	-0.595	0.187	0.000	1.011	0.139	-1.607	0.000	0.789	-0.613	-1.041	3.102	1.304	0.895	1.259	1.866	0.955	1.211	1.200
+1	1.125	-0.004	1.694	0.373	0.329	0.978	0.640	-0.391	0.000	1.122	-0.376	1.521	2.215	0.432	2.413	-1.259	0.000	0.969	0.730	0.512	3.102	0.716	0.773	0.991	0.624	0.977	0.981	0.875
+0	1.081	0.861	1.252	1.621	1.474	1.293	0.600	0.630	0.000	1.991	-0.090	-0.675	2.215	0.861	1.105	-0.201	0.000	1.135	2.489	-1.659	0.000	1.089	0.657	0.991	2.179	0.412	1.334	1.071
+1	0.652	-0.294	1.241	1.034	0.490	1.033	0.551	-0.963	2.173	0.661	1.031	-1.654	2.215	1.376	-0.018	0.843	0.000	0.943	-0.329	-0.269	0.000	1.085	1.067	0.991	1.504	0.773	1.135	0.993
+1	1.408	-1.028	-1.018	0.252	-0.242	0.465	-0.364	-0.200	0.000	1.466	0.669	0.739	1.107	1.031	0.415	-1.468	2.548	0.457	-1.091	-1.722	0.000	0.771	0.811	0.979	1.459	1.204	1.041	0.866
+1	0.781	-1.143	-0.659	0.961	1.266	1.183	-0.686	0.119	2.173	1.126	-0.064	1.447	0.000	0.730	1.430	-1.535	0.000	1.601	0.513	1.658	0.000	0.871	1.345	1.184	1.058	0.620	1.107	0.978
+1	1.300	-0.616	1.032	0.751	-0.731	0.961	-0.716	1.592	0.000	2.079	-1.063	-0.271	2.215	0.475	0.518	1.695	1.274	0.395	-2.204	0.349	0.000	1.350	0.983	1.369	1.265	1.428	1.135	0.982
+1	0.833	0.809	1.657	1.637	1.019	0.705	1.077	-0.968	2.173	1.261	0.114	-0.298	1.107	1.032	0.017	0.236	0.000	0.640	-0.026	-1.598	0.000	0.894	0.982	0.981	1.250	1.054	1.018	0.853
+1	1.686	-1.090	-0.301	0.890	0.557	1.304	-0.284	-1.393	2.173	0.388	2.118	0.513	0.000	0.514	-0.015	0.891	0.000	0.460	0.547	0.627	3.102	0.942	0.524	1.186	1.528	0.889	1.015	1.122
+1	0.551	0.911	0.879	0.379	-0.796	1.154	-0.808	-0.966	0.000	1.168	-0.513	0.355	2.215	0.646	-1.309	0.773	0.000	0.544	-0.283	1.301	3.102	0.847	0.705	0.990	0.772	0.546	0.790	0.719
+1	1.597	0.793	-1.119	0.691	-1.455	0.370	0.337	1.354	0.000	0.646	-1.005	0.732	2.215	1.019	0.040	0.209	0.000	0.545	0.958	0.239	3.102	0.962	0.793	0.994	0.719	0.745	0.812	0.739
+0	1.033	-1.193	-0.452	0.247	0.970	0.503	-1.424	1.362	0.000	1.062	-0.416	-1.156	2.215	0.935	-0.023	0.555	2.548	0.410	-1.766	0.379	0.000	0.590	0.953	0.991	0.717	1.081	0.763	0.690
+1	0.859	-1.004	1.521	0.781	-0.993	0.677	0.643	-0.338	2.173	0.486	0.409	1.283	0.000	0.679	0.110	0.285	0.000	0.715	-0.735	-0.157	1.551	0.702	0.773	0.984	0.627	0.633	0.694	0.643
+0	0.612	-1.127	1.074	1.225	-0.426	0.927	-2.141	-0.473	0.000	1.290	-0.927	-1.085	2.215	1.183	1.981	-1.687	0.000	2.176	0.406	-1.581	0.000	0.945	0.651	1.170	0.895	1.604	1.179	1.142
+1	0.535	0.321	-1.095	0.281	-0.960	0.876	-0.709	-0.076	0.000	1.563	-0.666	1.536	2.215	0.773	-0.321	0.435	0.000	0.682	-0.801	-0.952	3.102	0.711	0.667	0.985	0.888	0.741	0.872	0.758
+1	0.745	1.586	1.578	0.863	-1.423	0.530	1.714	1.085	0.000	1.174	0.679	1.015	0.000	1.158	0.609	-1.186	2.548	1.851	0.832	-0.248	3.102	0.910	1.164	0.983	0.947	0.858	0.928	0.823
+0	0.677	-1.014	-1.648	1.455	1.461	0.596	-2.358	0.517	0.000	0.800	0.849	-0.743	2.215	1.024	-0.282	-1.004	0.000	1.846	-0.977	0.378	3.102	2.210	1.423	0.982	1.074	1.623	1.417	1.258
+1	0.815	-1.263	0.057	1.018	-0.208	0.339	-0.347	-1.646	2.173	1.223	0.600	-1.658	2.215	1.435	0.042	0.926	0.000	0.777	1.698	-0.698	0.000	1.022	1.058	1.000	0.784	0.477	0.886	0.836
+0	3.512	-1.094	-0.220	0.338	-0.328	1.962	-1.099	1.544	1.087	1.461	-1.305	-0.922	2.215	1.219	-1.289	0.400	0.000	0.731	0.155	1.249	0.000	1.173	1.366	0.993	2.259	2.000	1.626	1.349
+0	0.904	1.248	0.325	0.317	-1.624	0.685	-0.538	1.665	2.173	0.685	-2.145	-1.106	0.000	0.632	-1.460	1.017	0.000	1.085	-0.182	0.162	3.102	0.885	0.801	0.989	0.930	0.904	1.012	0.961

contrib/gbdt/lightgbm/binary1.test

@@ -0,0 +1,500 @@
+1	0.644	0.247	-0.447	0.862	0.374	0.854	-1.126	-0.790	2.173	1.015	-0.201	1.400	0.000	1.575	1.807	1.607	0.000	1.585	-0.190	-0.744	3.102	0.958	1.061	0.980	0.875	0.581	0.905	0.796
+0	0.385	1.800	1.037	1.044	0.349	1.502	-0.966	1.734	0.000	0.966	-1.960	-0.249	0.000	1.501	0.465	-0.354	2.548	0.834	-0.440	0.638	3.102	0.695	0.909	0.981	0.803	0.813	1.149	1.116
+0	1.214	-0.166	0.004	0.505	1.434	0.628	-1.174	-1.230	1.087	0.579	-1.047	-0.118	0.000	0.835	0.340	1.234	2.548	0.711	-1.383	1.355	0.000	0.848	0.911	1.043	0.931	1.058	0.744	0.696
+1	0.420	1.111	0.137	1.516	-1.657	0.854	0.623	1.605	1.087	1.511	-1.297	0.251	0.000	0.872	-0.368	-0.721	0.000	0.543	0.731	1.424	3.102	1.597	1.282	1.105	0.730	0.148	1.231	1.234
+0	0.897	-1.703	-1.306	1.022	-0.729	0.836	0.859	-0.333	2.173	1.336	-0.965	0.972	2.215	0.671	1.021	-1.439	0.000	0.493	-2.019	-0.289	0.000	0.805	0.930	0.984	1.430	2.198	1.934	1.684
+0	0.756	1.126	-0.945	2.355	-0.555	0.889	0.800	1.440	0.000	0.585	0.271	0.631	2.215	0.722	1.744	1.051	0.000	0.618	0.924	0.698	1.551	0.976	0.864	0.988	0.803	0.234	0.822	0.911
+0	1.141	-0.741	0.953	1.478	-0.524	1.197	-0.871	1.689	2.173	0.875	1.321	-0.518	1.107	0.540	0.037	-0.987	0.000	0.879	1.187	0.245	0.000	0.888	0.701	1.747	1.358	2.479	1.491	1.223
+1	0.606	-0.936	-0.384	1.257	-1.162	2.719	-0.600	0.100	2.173	3.303	-0.284	1.561	1.107	0.689	1.786	-0.326	0.000	0.780	-0.532	1.216	0.000	0.936	2.022	0.985	1.574	4.323	2.263	1.742
+1	0.603	0.429	-0.279	1.448	1.301	1.008	2.423	-1.295	0.000	0.452	1.305	0.533	0.000	1.076	1.011	1.256	2.548	2.021	1.260	-0.343	0.000	0.890	0.969	1.281	0.763	0.652	0.827	0.785
+0	1.171	-0.962	0.521	0.841	-0.315	1.196	-0.744	-0.882	2.173	0.726	-1.305	1.377	1.107	0.643	-1.790	-1.264	0.000	1.257	0.222	0.817	0.000	0.862	0.911	0.987	0.846	1.293	0.899	0.756
+1	1.392	-0.358	0.235	1.494	-0.461	0.895	-0.848	1.549	2.173	0.841	-0.384	0.666	1.107	1.199	2.509	-0.891	0.000	1.109	-0.364	-0.945	0.000	0.693	2.135	1.170	1.362	0.959	2.056	1.842
+1	1.024	1.076	-0.886	0.851	1.530	0.673	-0.449	0.187	1.087	0.628	-0.895	1.176	2.215	0.696	-0.232	-0.875	0.000	0.411	1.501	0.048	0.000	0.842	0.919	1.063	1.193	0.777	0.964	0.807
+1	0.890	-0.760	1.182	1.369	0.751	0.696	-0.959	-0.710	1.087	0.775	-0.130	-1.409	2.215	0.701	-0.110	-0.739	0.000	0.508	-0.451	0.390	0.000	0.762	0.738	0.998	1.126	0.788	0.940	0.790
+1	0.460	0.537	0.636	1.442	-0.269	0.585	0.323	-1.731	2.173	0.503	1.034	-0.927	0.000	0.928	-1.024	1.006	2.548	0.513	-0.618	-1.336	0.000	0.802	0.831	0.992	1.019	0.925	1.056	0.833
+1	0.364	1.648	0.560	1.720	0.829	1.110	0.811	-0.588	0.000	0.408	1.045	1.054	2.215	0.319	-1.138	1.545	0.000	0.423	1.025	-1.265	3.102	1.656	0.928	1.003	0.544	0.327	0.670	0.746
+1	0.525	-0.096	1.206	0.948	-1.103	1.519	-0.582	0.606	2.173	1.274	-0.572	-0.934	0.000	0.855	-1.028	-1.222	0.000	0.578	-1.000	-1.725	3.102	0.896	0.878	0.981	0.498	0.909	0.772	0.668
+0	0.536	-0.821	-1.029	0.703	1.113	0.363	-0.711	0.022	1.087	0.325	1.503	1.249	2.215	0.673	1.041	-0.401	0.000	0.480	2.127	1.681	0.000	0.767	1.034	0.990	0.671	0.836	0.669	0.663
+1	1.789	-0.583	1.641	0.897	0.799	0.515	-0.100	-1.483	0.000	1.101	0.031	-0.326	2.215	1.195	0.001	0.126	2.548	0.768	-0.148	0.601	0.000	0.916	0.921	1.207	1.069	0.483	0.934	0.795
+1	1.332	-0.571	0.986	0.580	1.508	0.582	0.634	-0.746	1.087	1.084	-0.964	-0.489	0.000	0.785	0.274	0.343	2.548	0.779	0.721	1.489	0.000	1.733	1.145	0.990	1.270	0.715	0.897	0.915
+0	1.123	0.629	-1.708	0.597	-0.882	0.752	0.195	1.522	2.173	1.671	1.515	-0.003	0.000	0.778	0.514	0.139	1.274	0.801	1.260	1.600	0.000	1.495	0.976	0.988	0.676	0.921	1.010	0.943
+0	1.816	-0.515	0.171	0.980	-0.454	0.870	0.202	-1.399	2.173	1.130	1.066	-1.593	0.000	0.844	0.735	1.275	2.548	1.125	-1.133	0.348	0.000	0.837	0.693	0.988	1.112	0.784	1.009	0.974
+1	0.364	0.694	0.445	1.862	0.159	0.963	-1.356	1.260	1.087	0.887	-0.540	-1.533	2.215	0.658	-2.544	-1.236	0.000	0.516	-0.807	0.039	0.000	0.891	1.004	0.991	1.092	0.976	1.000	0.953
+1	0.790	-1.175	0.475	1.846	0.094	0.999	-1.090	0.257	0.000	1.422	0.854	1.112	2.215	1.302	1.004	-1.702	1.274	2.557	-0.787	-1.048	0.000	0.890	1.429	0.993	2.807	0.840	2.248	1.821
+1	0.765	-0.500	-0.603	1.843	-0.560	1.068	0.007	0.746	2.173	1.154	-0.017	1.329	0.000	1.165	1.791	-1.585	0.000	1.116	0.441	-0.886	0.000	0.774	0.982	0.989	1.102	0.633	1.178	1.021
+1	1.407	1.293	-1.418	0.502	-1.527	2.005	-2.122	0.622	0.000	1.699	1.508	-0.649	2.215	1.665	0.748	-0.755	0.000	2.555	0.811	1.423	1.551	7.531	5.520	0.985	1.115	1.881	4.487	3.379
+1	0.772	-0.186	-1.372	0.823	-0.140	0.781	0.763	0.046	2.173	1.128	0.516	1.380	0.000	0.797	-0.640	-0.134	2.548	2.019	-0.972	-1.670	0.000	2.022	1.466	0.989	0.856	0.808	1.230	0.991
+1	0.546	-0.954	0.715	1.335	-1.689	0.783	-0.443	-1.735	2.173	1.081	0.185	-0.435	0.000	1.433	-0.662	-0.389	0.000	0.969	0.924	1.099	0.000	0.910	0.879	0.988	0.683	0.753	0.878	0.865
+1	0.596	0.276	-1.054	1.358	1.355	1.444	1.813	-0.208	0.000	1.175	-0.949	-1.573	0.000	0.855	-1.228	-0.925	2.548	1.837	-0.400	0.913	0.000	0.637	0.901	1.028	0.553	0.790	0.679	0.677
+0	0.458	2.292	1.530	0.291	1.283	0.749	-0.930	-0.198	0.000	0.300	-1.560	0.990	0.000	0.811	-0.176	0.995	2.548	1.085	-0.178	-1.213	3.102	0.891	0.648	0.999	0.732	0.655	0.619	0.620
+0	0.638	-0.575	-1.048	0.125	0.178	0.846	-0.753	-0.339	1.087	0.799	-0.727	1.182	0.000	0.888	0.283	0.717	0.000	1.051	-1.046	-1.557	3.102	0.889	0.871	0.989	0.884	0.923	0.836	0.779
+1	0.434	-1.119	-0.313	2.427	0.461	0.497	0.261	-1.177	2.173	0.618	-0.737	-0.688	0.000	1.150	-1.237	-1.652	2.548	0.757	-0.054	1.700	0.000	0.809	0.741	0.982	1.450	0.936	1.086	0.910
+1	0.431	-1.144	-1.030	0.778	-0.655	0.490	0.047	-1.546	0.000	1.583	-0.014	0.891	2.215	0.516	0.956	0.567	2.548	0.935	-1.123	-0.082	0.000	0.707	0.995	0.995	0.700	0.602	0.770	0.685
+1	1.894	0.222	1.224	1.578	1.715	0.966	2.890	-0.013	0.000	0.922	-0.703	-0.844	0.000	0.691	2.056	1.039	0.000	0.900	-0.733	-1.240	3.102	1.292	1.992	1.026	0.881	0.684	1.759	1.755
+0	0.985	-0.316	0.141	1.067	-0.946	0.819	-1.177	1.307	2.173	1.080	-0.429	0.557	1.107	1.726	1.435	-1.075	0.000	1.100	1.547	-0.647	0.000	0.873	1.696	1.179	1.146	1.015	1.538	1.270
+0	0.998	-0.187	-0.236	0.882	0.755	0.468	0.950	-0.439	2.173	0.579	-0.550	-0.624	0.000	1.847	1.196	1.384	1.274	0.846	1.273	-1.072	0.000	1.194	0.797	1.013	1.319	1.174	0.963	0.898
+0	0.515	0.246	-0.593	1.082	1.591	0.912	-0.623	-0.957	2.173	0.858	0.418	0.844	0.000	0.948	2.519	1.599	0.000	1.158	1.385	-0.095	3.102	0.973	1.033	0.988	0.998	1.716	1.054	0.901
+0	0.919	-1.001	1.506	1.389	0.653	0.507	-0.616	-0.689	2.173	0.808	0.536	-0.467	2.215	0.496	2.187	-0.859	0.000	0.822	0.807	1.163	0.000	0.876	0.861	1.088	0.947	0.614	0.911	1.087
+0	0.794	0.051	1.477	1.504	-1.695	0.716	0.315	0.264	1.087	0.879	-0.135	-1.094	2.215	1.433	-0.741	0.201	0.000	1.566	0.534	-0.989	0.000	0.627	0.882	0.974	0.807	1.130	0.929	0.925
+1	0.455	-0.946	-1.175	1.453	-0.580	0.763	-0.856	0.840	0.000	0.829	1.223	1.174	2.215	0.714	0.638	-0.466	0.000	1.182	0.223	-1.333	0.000	0.977	0.938	0.986	0.713	0.714	0.796	0.843
+1	0.662	-0.296	-1.287	1.212	-0.707	0.641	1.457	0.222	0.000	0.600	0.525	-1.700	2.215	0.784	-0.835	-0.961	2.548	0.865	1.131	1.162	0.000	0.854	0.877	0.978	0.740	0.734	0.888	0.811
+0	0.390	0.698	-1.629	1.888	0.298	0.990	1.614	-1.572	0.000	1.666	0.170	0.719	2.215	1.590	1.064	-0.886	1.274	0.952	0.305	-1.216	0.000	1.048	0.897	1.173	0.891	1.936	1.273	1.102
+0	1.014	0.117	1.384	0.686	-1.047	0.609	-1.245	-0.850	0.000	1.076	-1.158	0.814	1.107	1.598	-0.389	-0.111	0.000	0.907	1.688	-1.673	0.000	1.333	0.866	0.989	0.975	0.442	0.797	0.788
+0	1.530	-1.408	-0.207	0.440	-1.357	0.902	-0.647	1.325	1.087	1.320	-0.819	0.246	1.107	0.503	1.407	-1.683	0.000	1.189	-0.972	-0.925	0.000	0.386	1.273	0.988	0.829	1.335	1.173	1.149
+1	1.689	-0.590	0.915	2.076	1.202	0.644	-0.478	-0.238	0.000	0.809	-1.660	-1.184	0.000	1.227	-0.224	-0.808	2.548	1.655	1.047	-0.623	0.000	0.621	1.192	0.988	1.309	0.866	0.924	1.012
+0	1.102	0.402	-1.622	1.262	1.022	0.576	0.271	-0.269	0.000	0.591	0.495	-1.278	0.000	1.271	0.209	0.575	2.548	0.941	0.964	-0.685	3.102	0.989	0.963	1.124	0.857	0.858	0.716	0.718
+0	2.491	0.825	0.581	1.593	0.205	0.782	-0.815	1.499	0.000	1.179	-0.999	-1.509	0.000	0.926	0.920	-0.522	2.548	2.068	-1.021	-1.050	3.102	0.874	0.943	0.980	0.945	1.525	1.570	1.652
+0	0.666	0.254	1.601	1.303	-0.250	1.236	-1.929	0.793	0.000	1.074	0.447	-0.871	0.000	0.991	1.059	-0.342	0.000	1.703	-0.393	-1.419	3.102	0.921	0.945	1.285	0.931	0.462	0.770	0.729
+0	0.937	-1.126	1.424	1.395	1.743	0.760	0.428	-0.238	2.173	0.846	0.494	1.320	2.215	0.872	-1.826	-0.507	0.000	0.612	1.860	1.403	0.000	3.402	2.109	0.985	1.298	1.165	1.404	1.240
+1	0.881	-1.086	-0.870	0.513	0.266	2.049	-1.870	1.160	0.000	2.259	-0.428	-0.935	2.215	1.321	-0.655	-0.449	2.548	1.350	-1.766	-0.108	0.000	0.911	1.852	0.987	1.167	0.820	1.903	1.443
+0	0.410	0.835	-0.819	1.257	1.112	0.871	-1.737	-0.401	0.000	0.927	0.158	1.253	0.000	1.183	0.405	-1.570	0.000	0.807	-0.704	-0.438	3.102	0.932	0.962	0.987	0.653	0.315	0.616	0.648
+1	0.634	0.196	-1.679	1.379	-0.967	2.260	-0.273	1.114	0.000	1.458	1.070	-0.278	1.107	1.195	0.110	-0.688	2.548	0.907	0.298	-1.359	0.000	0.949	1.129	0.984	0.675	0.877	0.938	0.824
+1	0.632	-1.254	1.201	0.496	-0.106	0.235	2.731	-0.955	0.000	0.615	-0.805	0.600	0.000	0.633	-0.934	1.641	0.000	1.407	-0.483	-0.962	1.551	0.778	0.797	0.989	0.578	0.722	0.576	0.539
+0	0.714	1.122	1.566	2.399	-1.431	1.665	0.299	0.323	0.000	1.489	1.087	-0.861	2.215	1.174	0.140	1.083	2.548	0.404	-0.968	1.105	0.000	0.867	0.969	0.981	1.039	1.552	1.157	1.173
+1	0.477	-0.321	-0.471	1.966	1.034	2.282	1.359	-0.874	0.000	1.672	-0.258	1.109	0.000	1.537	0.604	0.231	2.548	1.534	-0.640	0.827	0.000	0.746	1.337	1.311	0.653	0.721	0.795	0.742
+1	1.351	0.460	0.031	1.194	-1.185	0.670	-1.157	-1.637	2.173	0.599	-0.823	0.680	0.000	0.478	0.373	1.716	0.000	0.809	-0.919	0.010	1.551	0.859	0.839	1.564	0.994	0.777	0.971	0.826
+1	0.520	-1.442	-0.348	0.840	1.654	1.273	-0.760	1.317	0.000	0.861	2.579	-0.791	0.000	1.779	0.257	-0.703	0.000	2.154	1.928	0.457	0.000	1.629	3.194	0.992	0.730	1.107	2.447	2.747
+0	0.700	-0.308	0.920	0.438	-0.879	0.516	1.409	1.101	0.000	0.960	0.701	-0.049	2.215	1.442	-0.416	-1.439	2.548	0.628	1.009	-0.364	0.000	0.848	0.817	0.987	0.759	1.421	0.937	0.920
+1	0.720	1.061	-0.546	0.798	-1.521	1.066	0.173	0.271	1.087	1.453	0.114	1.336	1.107	0.702	0.616	-0.367	0.000	0.543	-0.386	-1.301	0.000	0.653	0.948	0.989	1.031	1.500	0.965	0.790
+1	0.735	-0.416	0.588	1.308	-0.382	1.042	0.344	1.609	0.000	0.926	0.163	-0.520	1.107	1.050	-0.427	1.159	2.548	0.834	0.613	0.948	0.000	0.848	1.189	1.042	0.844	1.099	0.829	0.843
+1	0.777	-0.396	1.540	1.608	0.638	0.955	0.040	0.918	2.173	1.315	1.116	-0.823	0.000	0.781	-0.762	0.564	2.548	0.945	-0.573	1.379	0.000	0.679	0.706	1.124	0.608	0.593	0.515	0.493
+1	0.934	0.319	-0.257	0.970	-0.980	0.726	0.774	0.731	0.000	0.896	0.038	-1.465	1.107	0.773	-0.055	-0.831	2.548	1.439	-0.229	0.698	0.000	0.964	1.031	0.995	0.845	0.480	0.810	0.762
+0	0.461	0.771	0.019	2.055	-1.288	1.043	0.147	0.261	2.173	0.833	-0.156	1.425	0.000	0.832	0.805	-0.491	2.548	0.589	1.252	1.414	0.000	0.850	0.906	1.245	1.364	0.850	0.908	0.863
+1	0.858	-0.116	-0.937	0.966	1.167	0.825	-0.108	1.111	1.087	0.733	1.163	-0.634	0.000	0.894	0.771	0.020	0.000	0.846	-1.124	-1.195	3.102	0.724	1.194	1.195	0.813	0.969	0.985	0.856
+0	0.720	-0.335	-0.307	1.445	0.540	1.108	-0.034	-1.691	1.087	0.883	-1.356	-0.678	2.215	0.440	1.093	0.253	0.000	0.389	-1.582	-1.097	0.000	1.113	1.034	0.988	1.256	1.572	1.062	0.904
+1	0.750	-0.811	-0.542	0.985	0.408	0.471	0.477	0.355	0.000	1.347	-0.875	-1.556	2.215	0.564	1.082	-0.724	0.000	0.793	-0.958	-0.020	3.102	0.836	0.825	0.986	1.066	0.924	0.927	0.883
+0	0.392	-0.468	-0.216	0.680	1.565	1.086	-0.765	-0.581	1.087	1.264	-1.035	1.189	2.215	0.986	-0.338	0.747	0.000	0.884	-1.328	-0.965	0.000	1.228	0.988	0.982	1.135	1.741	1.108	0.956
+1	0.434	-1.269	0.643	0.713	0.608	0.597	0.832	1.627	0.000	0.708	-0.422	0.079	2.215	1.533	-0.823	-1.127	2.548	0.408	-1.357	-0.828	0.000	1.331	1.087	0.999	1.075	1.015	0.875	0.809
+0	0.828	-1.803	0.342	0.847	-0.162	1.585	-1.128	-0.272	2.173	1.974	0.039	-1.717	0.000	0.900	0.764	-1.741	0.000	1.349	-0.079	1.035	3.102	0.984	0.815	0.985	0.780	1.661	1.403	1.184
+1	1.089	-0.350	-0.747	1.472	0.792	1.087	-0.069	-1.192	0.000	0.512	-0.841	-1.284	0.000	2.162	-0.821	0.545	2.548	1.360	2.243	-0.183	0.000	0.977	0.628	1.725	1.168	0.635	0.823	0.822
+1	0.444	0.451	-1.332	1.176	-0.247	0.898	0.194	0.007	0.000	1.958	0.576	-1.618	2.215	0.584	1.203	0.268	0.000	0.939	1.033	1.264	3.102	0.829	0.886	0.985	1.265	0.751	1.032	0.948
+0	0.629	0.114	1.177	0.917	-1.204	0.845	0.828	-0.088	0.000	0.962	-1.302	0.823	2.215	0.732	0.358	-1.334	2.548	0.538	0.582	1.561	0.000	1.028	0.834	0.988	0.904	1.205	1.039	0.885
+1	1.754	-1.259	-0.573	0.959	-1.483	0.358	0.448	-1.452	0.000	0.711	0.313	0.499	2.215	1.482	-0.390	1.474	2.548	1.879	-1.540	0.668	0.000	0.843	0.825	1.313	1.315	0.939	1.048	0.871
+1	0.549	0.706	-1.437	0.894	0.891	0.680	-0.762	-1.568	0.000	0.981	0.499	-0.425	2.215	1.332	0.678	0.485	1.274	0.803	0.022	-0.893	0.000	0.793	1.043	0.987	0.761	0.899	0.915	0.794
+0	0.475	0.542	-0.987	1.569	0.069	0.551	1.543	-1.488	0.000	0.608	0.301	1.734	2.215	0.277	0.499	-0.522	0.000	1.375	1.212	0.696	3.102	0.652	0.756	0.987	0.828	0.830	0.715	0.679
+1	0.723	0.049	-1.153	1.300	0.083	0.723	-0.749	0.630	0.000	1.126	0.412	-0.384	0.000	1.272	1.256	1.358	2.548	3.108	0.777	-1.486	3.102	0.733	1.096	1.206	1.269	0.899	1.015	0.903
+1	1.062	0.296	0.725	0.285	-0.531	0.819	1.277	-0.667	0.000	0.687	0.829	-0.092	0.000	1.158	0.447	1.047	2.548	1.444	-0.186	-1.491	3.102	0.863	1.171	0.986	0.769	0.828	0.919	0.840
+0	0.572	-0.349	1.396	2.023	0.795	0.577	0.457	-0.533	0.000	1.351	0.701	-1.091	0.000	0.724	-1.012	-0.182	2.548	0.923	-0.012	0.789	3.102	0.936	1.025	0.985	1.002	0.600	0.828	0.909
+1	0.563	0.387	0.412	0.553	1.050	0.723	-0.992	-0.447	0.000	0.748	0.948	0.546	2.215	1.761	-0.559	-1.183	0.000	1.114	-0.251	1.192	3.102	0.936	0.912	0.976	0.578	0.722	0.829	0.892
+1	1.632	1.577	-0.697	0.708	-1.263	0.863	0.012	1.197	2.173	0.498	0.990	-0.806	0.000	0.627	2.387	-1.283	0.000	0.607	1.290	-0.174	3.102	0.916	1.328	0.986	0.557	0.971	0.935	0.836
+1	0.562	-0.360	0.399	0.803	-1.334	1.443	-0.116	1.628	2.173	0.750	0.987	0.135	1.107	0.795	0.298	-0.556	0.000	1.150	-0.113	-0.093	0.000	0.493	1.332	0.985	1.001	1.750	1.013	0.886
+1	0.987	0.706	-0.492	0.861	0.607	0.593	0.088	-0.184	0.000	0.802	0.894	1.608	2.215	0.782	-0.471	1.500	2.548	0.521	0.772	-0.960	0.000	0.658	0.893	1.068	0.877	0.664	0.709	0.661
+1	1.052	0.883	-0.581	1.566	0.860	0.931	1.515	-0.873	0.000	0.493	0.145	-0.672	0.000	1.133	0.935	1.581	2.548	1.630	0.695	0.923	3.102	1.105	1.087	1.713	0.948	0.590	0.872	0.883
+1	2.130	-0.516	-0.291	0.776	-1.230	0.689	-0.257	0.800	2.173	0.730	-0.274	-1.437	0.000	0.615	0.241	1.083	0.000	0.834	0.757	1.613	3.102	0.836	0.806	1.333	1.061	0.730	0.889	0.783
+1	0.742	0.797	1.628	0.311	-0.418	0.620	0.685	-1.457	0.000	0.683	1.774	-1.082	0.000	1.700	1.104	0.225	2.548	0.382	-2.184	-1.307	0.000	0.945	1.228	0.984	0.864	0.931	0.988	0.838
+0	0.311	-1.249	-0.927	1.272	-1.262	0.642	-1.228	-0.136	0.000	1.220	-0.804	-1.558	2.215	0.950	-0.828	0.495	1.274	2.149	-1.672	0.634	0.000	1.346	0.887	0.981	0.856	1.101	1.001	1.106
+0	0.660	-1.834	-0.667	0.601	1.236	0.932	-0.933	-0.135	2.173	1.373	-0.122	1.429	0.000	0.654	-0.034	-0.847	2.548	0.711	0.911	0.703	0.000	1.144	0.942	0.984	0.822	0.739	0.992	0.895
+0	3.609	-0.590	0.851	0.615	0.455	1.280	0.003	-0.866	1.087	1.334	0.708	-1.131	0.000	0.669	0.480	0.092	0.000	0.975	0.983	-1.429	3.102	1.301	1.089	0.987	1.476	0.934	1.469	1.352
+1	0.905	-0.403	1.567	2.651	0.953	1.194	-0.241	-0.567	1.087	0.308	-0.384	-0.007	0.000	0.608	-0.175	-1.163	2.548	0.379	0.941	1.662	0.000	0.580	0.721	1.126	0.895	0.544	1.097	0.836
+1	0.983	0.255	1.093	0.905	-0.874	0.863	0.060	-0.368	0.000	0.824	-0.747	-0.633	0.000	0.614	0.961	1.052	0.000	0.792	-0.260	1.632	3.102	0.874	0.883	1.280	0.663	0.406	0.592	0.645
+1	1.160	-1.027	0.274	0.460	0.322	2.085	-1.623	-0.840	0.000	1.634	-1.046	1.182	2.215	0.492	-0.367	1.174	0.000	0.824	-0.998	1.617	0.000	0.943	0.884	1.001	1.209	1.313	1.034	0.866
+0	0.299	0.028	-1.372	1.930	-0.661	0.840	-0.979	0.664	1.087	0.535	-2.041	1.434	0.000	1.087	-1.797	0.344	0.000	0.485	-0.560	-1.105	3.102	0.951	0.890	0.980	0.483	0.684	0.730	0.706
+0	0.293	1.737	-1.418	2.074	0.794	0.679	1.024	-1.457	0.000	1.034	1.094	-0.168	1.107	0.506	1.680	-0.661	0.000	0.523	-0.042	-1.274	3.102	0.820	0.944	0.987	0.842	0.694	0.761	0.750
+0	0.457	-0.393	1.560	0.738	-0.007	0.475	-0.230	0.246	0.000	0.776	-1.264	-0.606	2.215	0.865	-0.731	-1.576	2.548	1.153	0.343	1.436	0.000	1.060	0.883	0.988	0.972	0.703	0.758	0.720
+0	0.935	-0.582	0.240	2.401	0.818	1.231	-0.618	-1.289	0.000	0.799	0.544	-0.228	2.215	0.525	-1.494	-0.969	0.000	0.609	-1.123	1.168	3.102	0.871	0.767	1.035	1.154	0.919	0.868	1.006
+1	0.902	-0.745	-1.215	1.174	-0.501	1.215	0.167	1.162	0.000	0.896	1.217	-0.976	0.000	0.585	-0.429	1.036	0.000	1.431	-0.416	0.151	3.102	0.524	0.952	0.990	0.707	0.271	0.592	0.826
+1	0.653	0.337	-0.320	1.118	-0.934	1.050	0.745	0.529	1.087	1.075	1.742	-1.538	0.000	0.585	1.090	0.973	0.000	1.091	-0.187	1.160	1.551	1.006	1.108	0.978	1.121	0.838	0.947	0.908
+0	1.157	1.401	0.340	0.395	-1.218	0.945	1.928	-0.876	0.000	1.384	0.320	1.002	1.107	1.900	1.177	-0.462	2.548	1.122	1.316	1.720	0.000	1.167	1.096	0.989	0.937	1.879	1.307	1.041
+0	0.960	0.355	-0.152	0.872	-0.338	0.391	0.348	0.956	1.087	0.469	2.664	1.409	0.000	0.756	-1.561	1.500	0.000	0.525	1.436	1.728	3.102	1.032	0.946	0.996	0.929	0.470	0.698	0.898
+1	1.038	0.274	0.825	1.198	0.963	1.078	-0.496	-1.014	2.173	0.739	-0.727	-0.151	2.215	1.035	-0.799	0.398	0.000	1.333	-0.872	-1.498	0.000	0.849	1.033	0.985	0.886	0.936	0.975	0.823
+0	0.490	0.277	0.318	1.303	0.694	1.333	-1.620	-0.563	0.000	1.459	-1.326	1.140	0.000	0.779	-0.673	-1.324	2.548	0.860	-1.247	0.043	0.000	0.857	0.932	0.992	0.792	0.278	0.841	1.498
+0	1.648	-0.688	-1.386	2.790	0.995	1.087	1.359	-0.687	0.000	1.050	-0.223	-0.261	2.215	0.613	-0.889	1.335	0.000	1.204	0.827	0.309	3.102	0.464	0.973	2.493	1.737	0.827	1.319	1.062
+0	1.510	-0.662	1.668	0.860	0.280	0.705	0.974	-1.647	1.087	0.662	-0.393	-0.225	0.000	0.610	-0.996	0.532	2.548	0.464	1.305	0.102	0.000	0.859	1.057	1.498	0.799	1.260	0.946	0.863
+1	0.850	-1.185	-0.117	0.943	-0.449	1.142	0.875	-0.030	0.000	2.223	-0.461	1.627	2.215	0.767	-1.761	-1.692	0.000	1.012	-0.727	0.639	3.102	3.649	2.062	0.985	1.478	1.087	1.659	1.358
+0	0.933	1.259	0.130	0.326	-0.890	0.306	1.136	1.142	0.000	0.964	0.705	-1.373	2.215	0.546	-0.196	-0.001	0.000	0.578	-1.169	1.004	3.102	0.830	0.836	0.988	0.837	1.031	0.749	0.655
+0	0.471	0.697	1.570	1.109	0.201	1.248	0.348	-1.448	0.000	2.103	0.773	0.686	2.215	1.451	-0.087	-0.453	2.548	1.197	-0.045	-1.026	0.000	0.793	1.094	0.987	0.851	1.804	1.378	1.089
+1	2.446	-0.701	-1.568	0.059	0.822	1.401	-0.600	-0.044	2.173	0.324	-0.001	1.344	2.215	0.913	-0.818	1.049	0.000	0.442	-1.088	-0.005	0.000	0.611	1.062	0.979	0.562	0.988	0.998	0.806
+0	0.619	2.029	0.933	0.528	-0.903	0.974	0.760	-0.311	2.173	0.825	0.658	-1.466	1.107	0.894	1.594	0.370	0.000	0.882	-0.258	1.661	0.000	1.498	1.088	0.987	0.867	1.139	0.900	0.779
+1	0.674	-0.131	-0.362	0.518	-1.574	0.876	0.442	0.145	1.087	0.497	-1.526	-1.704	0.000	0.680	2.514	-1.374	0.000	0.792	-0.479	0.773	1.551	0.573	1.198	0.984	0.800	0.667	0.987	0.832
+1	1.447	1.145	-0.937	0.307	-1.458	0.478	1.264	0.816	1.087	0.558	1.015	-0.101	2.215	0.937	-0.190	1.177	0.000	0.699	0.954	-1.512	0.000	0.877	0.838	0.990	0.873	0.566	0.646	0.713
+1	0.976	0.308	-0.844	0.436	0.610	1.253	0.149	-1.585	2.173	1.415	0.568	0.096	2.215	0.953	-0.855	0.441	0.000	0.867	-0.650	1.643	0.000	0.890	1.234	0.988	0.796	2.002	1.179	0.977
+0	0.697	0.401	-0.718	0.920	0.735	0.958	-0.172	0.168	2.173	0.872	-0.097	-1.335	0.000	0.513	-1.192	-1.710	1.274	0.426	-1.637	1.368	0.000	0.997	1.227	1.072	0.800	1.013	0.786	0.749
+1	1.305	-2.157	1.740	0.661	-0.912	0.705	-0.516	0.759	2.173	0.989	-0.716	-0.300	2.215	0.627	-1.052	-1.736	0.000	0.467	-2.467	0.568	0.000	0.807	0.964	0.988	1.427	1.012	1.165	0.926
+0	1.847	1.663	-0.618	0.280	1.258	1.462	-0.054	1.371	0.000	0.900	0.309	-0.544	0.000	0.331	-2.149	-0.341	0.000	1.091	-0.833	0.710	3.102	1.496	0.931	0.989	1.549	0.115	1.140	1.150
+0	0.410	-0.323	1.069	2.160	0.010	0.892	0.942	-1.640	2.173	0.946	0.938	1.314	0.000	1.213	-1.099	-0.794	2.548	0.650	0.053	0.056	0.000	1.041	0.916	1.063	0.985	1.910	1.246	1.107
+1	0.576	1.092	-0.088	0.777	-1.579	0.757	0.271	0.109	0.000	0.819	0.827	-1.554	2.215	1.313	2.341	-1.568	0.000	2.827	0.239	-0.338	0.000	0.876	0.759	0.986	0.692	0.457	0.796	0.791
+1	0.537	0.925	-1.406	0.306	-0.050	0.906	1.051	0.037	0.000	1.469	-0.177	-1.320	2.215	1.872	0.723	1.158	0.000	1.313	0.227	-0.501	3.102	0.953	0.727	0.978	0.755	0.892	0.932	0.781
+0	0.716	-0.065	-0.484	1.313	-1.563	0.596	-0.242	0.678	2.173	0.426	-1.909	0.616	0.000	0.885	-0.406	-1.343	2.548	0.501	-1.327	-0.340	0.000	0.470	0.728	1.109	0.919	0.881	0.665	0.692
+1	0.624	-0.389	0.128	1.636	-1.110	1.025	0.573	-0.843	2.173	0.646	-0.697	1.064	0.000	0.632	-1.442	0.961	0.000	0.863	-0.106	1.717	0.000	0.825	0.917	1.257	0.983	0.713	0.890	0.824
+0	0.484	2.101	1.714	1.131	-0.823	0.750	0.583	-1.304	1.087	0.894	0.421	0.559	2.215	0.921	-0.063	0.282	0.000	0.463	-0.474	-1.387	0.000	0.742	0.886	0.995	0.993	1.201	0.806	0.754
+0	0.570	0.339	-1.478	0.528	0.439	0.978	1.479	-1.411	2.173	0.763	1.541	-0.734	0.000	1.375	0.840	0.903	0.000	0.965	1.599	0.364	0.000	0.887	1.061	0.992	1.322	1.453	1.013	0.969
+0	0.940	1.303	1.636	0.851	-1.732	0.803	-0.030	-0.177	0.000	0.480	-0.125	-0.954	0.000	0.944	0.709	0.296	2.548	1.342	-0.418	1.197	3.102	0.853	0.989	0.979	0.873	0.858	0.719	0.786
+1	0.599	0.544	-0.238	0.816	1.043	0.857	0.660	1.128	2.173	0.864	-0.624	-0.843	0.000	1.159	0.367	0.174	0.000	1.520	-0.543	-1.508	0.000	0.842	0.828	0.984	0.759	0.895	0.918	0.791
+1	1.651	1.897	-0.914	0.423	0.315	0.453	0.619	-1.607	2.173	0.532	-0.424	0.209	1.107	0.369	2.479	0.034	0.000	0.701	0.217	0.984	0.000	0.976	0.951	1.035	0.879	0.825	0.915	0.798
+1	0.926	-0.574	-0.763	0.285	1.094	0.672	2.314	1.545	0.000	1.124	0.415	0.809	0.000	1.387	0.270	-0.949	2.548	1.547	-0.631	-0.200	3.102	0.719	0.920	0.986	0.889	0.933	0.797	0.777
+0	0.677	1.698	-0.890	0.641	-0.449	0.607	1.754	1.720	0.000	0.776	0.372	0.782	2.215	0.511	1.491	-0.480	0.000	0.547	-0.341	0.853	3.102	0.919	1.026	0.997	0.696	0.242	0.694	0.687
+0	1.266	0.602	0.958	0.487	1.256	0.709	0.843	-1.196	0.000	0.893	1.303	-0.594	1.107	1.090	1.320	0.354	0.000	0.797	1.846	1.139	0.000	0.780	0.896	0.986	0.661	0.709	0.790	0.806
+1	0.628	-0.616	-0.329	0.764	-1.150	0.477	-0.715	1.187	2.173	1.250	0.607	1.026	2.215	0.983	-0.023	-0.583	0.000	0.377	1.344	-1.015	0.000	0.744	0.954	0.987	0.837	0.841	0.795	0.694
+1	1.035	-0.828	-1.358	1.870	-1.060	1.075	0.130	0.448	2.173	0.660	0.697	0.641	0.000	0.425	1.006	-1.035	0.000	0.751	1.055	1.364	3.102	0.826	0.822	0.988	0.967	0.901	1.077	0.906
+1	0.830	0.265	-0.150	0.660	1.105	0.592	-0.557	0.908	2.173	0.670	-1.419	-0.671	0.000	1.323	-0.409	1.644	2.548	0.850	-0.033	-0.615	0.000	0.760	0.967	0.984	0.895	0.681	0.747	0.770
+1	1.395	1.100	1.167	1.088	0.218	0.400	-0.132	0.024	2.173	0.743	0.530	-1.361	2.215	0.341	-0.691	-0.238	0.000	0.396	-1.426	-0.933	0.000	0.363	0.472	1.287	0.922	0.810	0.792	0.656
+1	1.070	1.875	-1.298	1.215	-0.106	0.767	0.795	0.514	1.087	0.401	2.780	1.276	0.000	0.686	1.127	1.721	2.548	0.391	-0.259	-1.167	0.000	1.278	1.113	1.389	0.852	0.824	0.838	0.785
+0	1.114	-0.071	1.719	0.399	-1.383	0.849	0.254	0.481	0.000	0.958	-0.579	0.742	0.000	1.190	-0.140	-0.862	2.548	0.479	1.390	0.856	0.000	0.952	0.988	0.985	0.764	0.419	0.835	0.827
+0	0.714	0.376	-0.568	1.578	-1.165	0.648	0.141	0.639	2.173	0.472	0.569	1.449	1.107	0.783	1.483	0.361	0.000	0.540	-0.790	0.032	0.000	0.883	0.811	0.982	0.775	0.572	0.760	0.745
+0	0.401	-1.731	0.765	0.974	1.648	0.652	-1.024	0.191	0.000	0.544	-0.366	-1.246	2.215	0.627	0.140	1.008	2.548	0.810	0.409	0.429	0.000	0.950	0.934	0.977	0.621	0.580	0.677	0.650
+1	0.391	1.679	-1.298	0.605	-0.832	0.549	1.338	0.522	2.173	1.244	0.884	1.070	0.000	1.002	0.846	-1.345	2.548	0.783	-2.464	-0.237	0.000	4.515	2.854	0.981	0.877	0.939	1.942	1.489
+1	0.513	-0.220	-0.444	1.699	0.479	1.109	0.181	-0.999	2.173	0.883	-0.335	-1.716	2.215	1.075	-0.380	1.352	0.000	0.857	0.048	0.147	0.000	0.937	0.758	0.986	1.206	0.958	0.949	0.876
+0	1.367	-0.388	0.798	1.158	1.078	0.811	-1.024	-1.628	0.000	1.504	0.097	-0.999	2.215	1.652	-0.860	0.054	2.548	0.573	-0.142	-1.401	0.000	0.869	0.833	1.006	1.412	1.641	1.214	1.041
+1	1.545	-0.533	-1.517	1.177	1.289	2.331	-0.370	-0.073	0.000	1.295	-0.358	-0.891	2.215	0.476	0.756	0.985	0.000	1.945	-0.016	-1.651	3.102	1.962	1.692	1.073	0.656	0.941	1.312	1.242
+0	0.858	0.978	-1.258	0.286	0.161	0.729	1.230	1.087	2.173	0.561	2.670	-0.109	0.000	0.407	2.346	0.938	0.000	1.078	0.729	-0.658	3.102	0.597	0.921	0.982	0.579	0.954	0.733	0.769
+1	1.454	-1.384	0.870	0.067	0.394	1.033	-0.673	0.318	0.000	1.166	-0.763	-1.533	2.215	2.848	-0.045	-0.856	2.548	0.697	-0.140	1.134	0.000	0.931	1.293	0.977	1.541	1.326	1.201	1.078
+1	0.559	-0.913	0.486	1.104	-0.321	1.073	-0.348	1.345	0.000	0.901	-0.827	-0.842	0.000	0.739	0.047	-0.415	2.548	0.433	-1.132	1.268	0.000	0.797	0.695	0.985	0.868	0.346	0.674	0.623
+1	1.333	0.780	-0.964	0.916	1.202	1.822	-0.071	0.742	2.173	1.486	-0.399	-0.824	0.000	0.740	0.568	-0.134	0.000	0.971	-0.070	-1.589	3.102	1.278	0.929	1.421	1.608	1.214	1.215	1.137
+1	2.417	0.631	-0.317	0.323	0.581	0.841	1.524	-1.738	0.000	0.543	1.176	-0.325	0.000	0.827	0.700	0.866	0.000	0.834	-0.262	-1.702	3.102	0.932	0.820	0.988	0.646	0.287	0.595	0.589
+0	0.955	-1.242	0.938	1.104	0.474	0.798	-0.743	1.535	0.000	1.356	-1.357	-1.080	2.215	1.320	-1.396	-0.132	2.548	0.728	-0.529	-0.633	0.000	0.832	0.841	0.988	0.923	1.077	0.988	0.816
+1	1.305	-1.918	0.391	1.161	0.063	0.724	2.593	1.481	0.000	0.592	-1.207	-0.329	0.000	0.886	-0.836	-1.168	2.548	1.067	-1.481	-1.440	0.000	0.916	0.688	0.991	0.969	0.550	0.665	0.638
+0	1.201	0.071	-1.123	2.242	-1.533	0.702	-0.256	0.688	0.000	0.967	0.491	1.040	2.215	1.271	-0.558	0.095	0.000	1.504	0.676	-0.383	3.102	0.917	1.006	0.985	1.017	1.057	0.928	1.057
+0	0.994	-1.607	1.596	0.774	-1.391	0.625	-0.134	-0.862	2.173	0.746	-0.765	-0.316	2.215	1.131	-0.320	0.869	0.000	0.607	0.826	0.301	0.000	0.798	0.967	0.999	0.880	0.581	0.712	0.774
+1	0.482	-0.467	0.729	1.419	1.458	0.824	0.376	-0.242	0.000	1.368	0.023	1.459	2.215	0.826	0.669	-1.079	2.548	0.936	2.215	-0.309	0.000	1.883	1.216	0.997	1.065	0.946	1.224	1.526
+1	0.383	1.588	1.611	0.748	1.194	0.866	-0.279	-0.636	0.000	0.707	0.536	0.801	2.215	1.647	-1.155	0.367	0.000	1.292	0.303	-1.681	3.102	2.016	1.581	0.986	0.584	0.684	1.107	0.958
+0	0.629	0.203	0.736	0.671	-0.271	1.350	-0.486	0.761	2.173	0.496	-0.805	-1.718	0.000	2.393	0.044	-1.046	1.274	0.651	-0.116	-0.541	0.000	0.697	1.006	0.987	1.069	2.317	1.152	0.902
+0	0.905	-0.564	-0.570	0.263	1.096	1.219	-1.397	-1.414	1.087	1.164	-0.533	-0.208	0.000	1.459	1.965	0.784	0.000	2.220	-1.421	0.452	0.000	0.918	1.360	0.993	0.904	0.389	2.118	1.707
+1	1.676	1.804	1.171	0.529	1.175	1.664	0.354	-0.530	0.000	1.004	0.691	-1.280	2.215	0.838	0.373	0.626	2.548	1.094	1.774	0.501	0.000	0.806	1.100	0.991	0.769	0.976	0.807	0.740
+1	1.364	-1.936	0.020	1.327	0.428	1.021	-1.665	-0.907	2.173	0.818	-2.701	1.303	0.000	0.716	-0.590	-1.629	2.548	0.895	-2.280	-1.602	0.000	1.211	0.849	0.989	1.320	0.864	1.065	0.949
+0	0.629	-0.626	0.609	1.828	1.280	0.644	-0.856	-0.873	2.173	0.555	1.066	-0.640	0.000	0.477	-1.364	-1.021	2.548	1.017	0.036	0.380	0.000	0.947	0.941	0.994	1.128	0.241	0.793	0.815
+1	1.152	-0.843	0.926	1.802	0.800	2.493	-1.449	-1.127	0.000	1.737	0.833	0.488	0.000	1.026	0.929	-0.990	2.548	1.408	0.689	1.142	3.102	1.171	0.956	0.993	2.009	0.867	1.499	1.474
+0	2.204	0.081	0.008	1.021	-0.679	2.676	0.090	1.163	0.000	2.210	-1.686	-1.195	0.000	1.805	0.891	-0.148	2.548	0.450	-0.502	-1.295	3.102	6.959	3.492	1.205	0.908	0.845	2.690	2.183
+1	0.957	0.954	1.702	0.043	-0.503	1.113	0.033	-0.308	0.000	0.757	-0.363	-1.129	2.215	1.635	0.068	1.048	1.274	0.415	-2.098	0.061	0.000	1.010	0.979	0.992	0.704	1.125	0.761	0.715
+0	1.222	0.418	1.059	1.303	1.442	0.282	-1.499	-1.286	0.000	1.567	0.016	-0.164	2.215	0.451	2.229	-1.229	0.000	0.660	-0.513	-0.296	3.102	2.284	1.340	0.985	1.531	0.314	1.032	1.094
+1	0.603	1.675	-0.973	0.703	-1.709	1.023	0.652	1.296	2.173	1.078	0.363	-0.263	0.000	0.734	-0.457	-0.745	1.274	0.561	1.434	-0.042	0.000	0.888	0.771	0.984	0.847	1.234	0.874	0.777
+0	0.897	0.949	-0.848	1.115	-0.085	0.522	-1.267	-1.418	0.000	0.684	-0.599	1.474	0.000	1.176	0.922	0.641	2.548	0.470	0.103	0.148	3.102	0.775	0.697	0.984	0.839	0.358	0.847	1.008
+1	0.987	1.013	-1.504	0.468	-0.259	1.160	0.476	-0.971	2.173	1.266	0.919	0.780	0.000	0.634	1.695	0.233	0.000	0.487	-0.082	0.719	3.102	0.921	0.641	0.991	0.730	0.828	0.952	0.807
+1	0.847	1.581	-1.397	1.629	1.529	1.053	0.816	-0.344	2.173	0.895	0.779	0.332	0.000	0.750	1.311	0.419	2.548	1.604	0.844	1.367	0.000	1.265	0.798	0.989	1.328	0.783	0.930	0.879
+1	0.805	1.416	-1.327	0.397	0.589	0.488	0.982	0.843	0.000	0.664	-0.999	0.129	0.000	0.624	0.613	-0.558	0.000	1.431	-0.667	-1.561	3.102	0.959	1.103	0.989	0.590	0.632	0.926	0.798
+0	1.220	-0.313	-0.489	1.759	0.201	1.698	-0.220	0.241	2.173	1.294	1.390	-1.682	0.000	1.447	-1.623	-1.296	0.000	1.710	0.872	-1.356	3.102	1.198	0.981	1.184	0.859	2.165	1.807	1.661
+0	0.772	-0.611	-0.549	0.465	-1.528	1.103	-0.140	0.001	2.173	0.854	-0.406	1.655	0.000	0.733	-1.250	1.072	0.000	0.883	0.627	-1.132	3.102	0.856	0.927	0.987	1.094	1.013	0.938	0.870
+1	1.910	0.771	0.828	0.231	1.267	1.398	1.455	-0.295	2.173	0.837	-2.564	0.770	0.000	0.540	2.189	1.287	0.000	1.345	1.311	-1.151	0.000	0.861	0.869	0.984	1.359	1.562	1.105	0.963
+1	0.295	0.832	1.399	1.222	-0.517	2.480	0.013	1.591	0.000	2.289	0.436	0.287	2.215	1.995	-0.367	-0.409	1.274	0.375	1.367	-1.716	0.000	1.356	2.171	0.990	1.467	1.664	1.855	1.705
+1	1.228	0.339	-0.575	0.417	1.474	0.480	-1.416	-1.498	2.173	0.614	-0.933	-0.961	0.000	1.189	1.690	1.003	0.000	1.690	-1.065	0.106	3.102	0.963	1.147	0.987	1.086	0.948	0.930	0.866
+0	2.877	-1.014	1.440	0.782	0.483	1.134	-0.735	-0.196	2.173	1.123	0.084	-0.596	0.000	1.796	-0.356	1.044	2.548	1.406	1.582	-0.991	0.000	0.939	1.178	1.576	0.996	1.629	1.216	1.280
+1	2.178	0.259	1.107	0.256	1.222	0.979	-0.440	-0.538	1.087	0.496	-0.760	-0.049	0.000	1.471	1.683	-1.486	0.000	0.646	0.695	-1.577	3.102	1.093	1.070	0.984	0.608	0.889	0.962	0.866
+1	0.604	0.592	1.295	0.964	0.348	1.178	-0.016	0.832	2.173	1.626	-0.420	-0.760	0.000	0.748	0.461	-0.906	0.000	0.728	0.309	-1.269	1.551	0.852	0.604	0.989	0.678	0.949	1.021	0.878
+0	0.428	-1.352	-0.912	1.713	0.797	1.894	-1.452	0.191	2.173	2.378	2.113	-1.190	0.000	0.860	2.174	0.949	0.000	1.693	0.759	1.426	3.102	0.885	1.527	1.186	1.090	3.294	4.492	3.676
+0	0.473	0.485	0.154	1.433	-1.504	0.766	1.257	-1.302	2.173	0.414	0.119	0.238	0.000	0.805	0.242	-0.691	2.548	0.734	0.749	0.753	0.000	0.430	0.893	1.137	0.686	0.724	0.618	0.608
+1	0.763	-0.601	0.876	0.182	-1.678	0.818	0.599	0.481	2.173	0.658	-0.737	-0.553	0.000	0.857	-1.138	-1.435	0.000	1.540	-1.466	-0.447	0.000	0.870	0.566	0.989	0.728	0.658	0.821	0.726
+0	0.619	-0.273	-0.143	0.992	-1.267	0.566	0.876	-1.396	2.173	0.515	0.892	0.618	0.000	0.434	-0.902	0.862	2.548	0.490	-0.539	0.549	0.000	0.568	0.794	0.984	0.667	0.867	0.597	0.578
+0	0.793	0.970	0.324	0.570	0.816	0.761	-0.550	1.519	2.173	1.150	0.496	-0.447	0.000	0.925	0.724	1.008	1.274	1.135	-0.275	-0.843	0.000	0.829	1.068	0.978	1.603	0.892	1.041	1.059
+1	0.480	0.364	-0.067	1.906	-1.582	1.397	1.159	0.140	0.000	0.639	0.398	-1.102	0.000	1.597	-0.668	1.607	2.548	1.306	-0.797	0.288	3.102	0.856	1.259	1.297	1.022	1.032	1.049	0.939
+0	0.514	1.304	1.490	1.741	-0.220	0.648	0.155	0.535	0.000	0.562	-1.016	0.837	0.000	0.863	-0.780	-0.815	2.548	1.688	-0.130	-1.545	3.102	0.887	0.980	1.309	1.269	0.654	1.044	1.035
+0	1.225	0.333	0.656	0.893	0.859	1.037	-0.876	1.603	1.087	1.769	0.272	-0.227	2.215	1.000	0.579	-1.690	0.000	1.385	0.471	-0.860	0.000	0.884	1.207	0.995	1.097	2.336	1.282	1.145
+0	2.044	-1.472	-0.294	0.392	0.369	0.927	0.718	1.492	1.087	1.619	-0.736	0.047	2.215	1.884	-0.101	-1.540	0.000	0.548	-0.441	1.117	0.000	0.798	0.877	0.981	0.750	2.272	1.469	1.276
+0	1.037	-0.276	0.735	3.526	1.156	2.498	0.401	-0.590	1.087	0.714	-1.203	1.393	2.215	0.681	0.629	1.534	0.000	0.719	-0.355	-0.706	0.000	0.831	0.857	0.988	2.864	2.633	1.988	1.466
+1	0.651	-1.218	-0.791	0.770	-1.449	0.610	-0.535	0.960	2.173	0.380	-1.072	-0.031	2.215	0.415	2.123	-1.100	0.000	0.776	0.217	0.420	0.000	0.986	1.008	1.001	0.853	0.588	0.799	0.776
+0	1.586	-0.409	0.085	3.258	0.405	1.647	-0.674	-1.519	0.000	0.640	-1.027	-1.681	0.000	1.452	-0.444	-0.957	2.548	0.927	-0.017	1.215	3.102	0.519	0.866	0.992	0.881	0.847	1.018	1.278
+0	0.712	0.092	-0.466	0.688	1.236	0.921	-1.217	-1.022	2.173	2.236	-1.167	0.868	2.215	0.851	-1.892	-0.753	0.000	0.475	-1.216	-0.383	0.000	0.668	0.758	0.988	1.180	2.093	1.157	0.934
+0	0.419	0.471	0.974	2.805	0.235	1.473	-0.198	1.255	1.087	0.931	1.083	-0.712	0.000	1.569	1.358	-1.179	2.548	2.506	0.199	-0.842	0.000	0.929	0.991	0.992	1.732	2.367	1.549	1.430
+1	0.667	1.003	1.504	0.368	1.061	0.885	-0.318	-0.353	0.000	1.438	-1.939	0.710	0.000	1.851	0.277	-1.460	2.548	1.403	0.517	-0.157	0.000	0.883	1.019	1.000	0.790	0.859	0.938	0.841
+1	1.877	-0.492	0.372	0.441	0.955	1.034	-1.220	-0.846	1.087	0.952	-0.320	1.125	0.000	0.542	0.308	-1.261	2.548	1.018	-1.415	-1.547	0.000	1.280	0.932	0.991	1.273	0.878	0.921	0.906
+0	1.052	0.901	1.176	1.280	1.517	0.562	-1.150	-0.079	2.173	1.228	-0.308	-0.354	0.000	0.790	-1.492	-0.963	0.000	0.942	-0.672	-1.588	3.102	1.116	0.902	0.988	1.993	0.765	1.375	1.325
+1	0.518	-0.254	1.642	0.865	0.725	0.980	0.734	0.023	0.000	1.448	0.780	-1.736	2.215	0.955	0.513	-0.519	0.000	0.365	-0.444	-0.243	3.102	0.833	0.555	0.984	0.827	0.795	0.890	0.786
+0	0.870	0.815	-0.506	0.663	-0.518	0.935	0.289	-1.675	2.173	1.188	0.005	0.635	0.000	0.580	0.066	-1.455	2.548	0.580	-0.634	-0.199	0.000	0.852	0.788	0.979	1.283	0.208	0.856	0.950
+0	0.628	1.382	0.135	0.683	0.571	1.097	0.564	-0.950	2.173	0.617	-0.326	0.371	0.000	1.093	0.918	1.667	2.548	0.460	1.221	0.708	0.000	0.743	0.861	0.975	1.067	1.007	0.843	0.762
+0	4.357	0.816	-1.609	1.845	-1.288	3.292	0.726	0.324	2.173	1.528	0.583	-0.801	2.215	0.605	0.572	1.406	0.000	0.794	-0.791	0.122	0.000	0.967	1.132	1.124	3.602	2.811	2.460	1.861
+0	0.677	-1.265	1.559	0.866	-0.618	0.823	0.260	0.185	0.000	1.133	0.337	1.589	2.215	0.563	-0.830	0.510	0.000	0.777	0.117	-0.941	3.102	0.839	0.763	0.986	1.182	0.649	0.796	0.851
+0	2.466	-1.838	-1.648	1.717	1.533	1.676	-1.553	-0.109	2.173	0.670	-0.666	0.284	0.000	0.334	-2.480	0.316	0.000	0.366	-0.804	-1.298	3.102	0.875	0.894	0.997	0.548	0.770	1.302	1.079
+1	1.403	0.129	-1.307	0.688	0.306	0.579	0.753	0.814	1.087	0.474	0.694	-1.400	0.000	0.520	1.995	0.185	0.000	0.929	-0.504	1.270	3.102	0.972	0.998	1.353	0.948	0.650	0.688	0.724
+1	0.351	1.188	-0.360	0.254	-0.346	1.129	0.545	1.691	0.000	0.652	-0.039	-0.258	2.215	1.089	0.655	0.472	2.548	0.554	-0.493	1.366	0.000	0.808	1.045	0.992	0.570	0.649	0.809	0.744
+0	1.875	-0.013	-0.128	0.236	1.163	0.902	0.426	0.590	2.173	1.251	-1.210	-0.616	0.000	1.035	1.534	0.912	0.000	1.944	1.789	-1.691	0.000	0.974	1.113	0.990	0.925	1.120	0.956	0.912
+0	0.298	0.750	-0.507	1.555	1.463	0.804	1.200	-0.665	0.000	0.439	-0.829	-0.252	1.107	0.770	-1.090	0.947	2.548	1.165	-0.166	-0.763	0.000	1.140	0.997	0.988	1.330	0.555	1.005	1.012
+0	0.647	0.342	0.245	4.340	-0.157	2.229	0.068	1.170	2.173	2.133	-0.201	-1.441	0.000	1.467	0.697	-0.532	1.274	1.457	0.583	-1.640	0.000	0.875	1.417	0.976	2.512	2.390	1.794	1.665
+1	1.731	-0.803	-1.013	1.492	-0.020	1.646	-0.541	1.121	2.173	0.459	-1.251	-1.495	2.215	0.605	-1.711	-0.232	0.000	0.658	0.634	-0.068	0.000	1.214	0.886	1.738	1.833	1.024	1.192	1.034
+0	0.515	1.416	-1.089	1.697	1.426	1.414	0.941	0.027	0.000	1.480	0.133	-1.595	2.215	1.110	0.752	0.760	2.548	1.062	0.697	-0.492	0.000	0.851	0.955	0.994	1.105	1.255	1.175	1.095
+0	1.261	0.858	1.465	0.757	0.305	2.310	0.679	1.080	2.173	1.544	2.518	-0.464	0.000	2.326	0.270	-0.841	0.000	2.163	0.839	-0.500	3.102	0.715	0.825	1.170	0.980	2.371	1.527	1.221
+1	1.445	1.509	1.471	0.414	-1.285	0.767	0.864	-0.677	2.173	0.524	1.388	0.171	0.000	0.826	0.190	0.121	2.548	0.572	1.691	-1.603	0.000	0.870	0.935	0.994	0.968	0.735	0.783	0.777
+1	0.919	-0.264	-1.245	0.681	-1.722	1.022	1.010	0.097	2.173	0.685	0.403	-1.351	0.000	1.357	-0.429	1.262	1.274	0.687	1.021	-0.563	0.000	0.953	0.796	0.991	0.873	1.749	1.056	0.917
+1	0.293	-2.258	-1.427	1.191	1.202	0.394	-2.030	1.438	0.000	0.723	0.596	-0.024	2.215	0.525	-1.678	-0.290	0.000	0.788	-0.824	-1.029	3.102	0.821	0.626	0.976	1.080	0.810	0.842	0.771
+0	3.286	0.386	1.688	1.619	-1.620	1.392	-0.009	0.280	0.000	1.179	-0.776	-0.110	2.215	1.256	0.248	-1.114	2.548	0.777	0.825	-0.156	0.000	1.026	1.065	0.964	0.909	1.249	1.384	1.395
+1	1.075	0.603	0.561	0.656	-0.685	0.985	0.175	0.979	2.173	1.154	0.584	-0.886	0.000	1.084	-0.354	-1.004	2.548	0.865	1.224	1.269	0.000	1.346	1.073	1.048	0.873	1.310	1.003	0.865
+1	1.098	-0.091	1.466	1.558	0.915	0.649	1.314	-1.182	2.173	0.791	0.073	0.351	0.000	0.517	0.940	1.195	0.000	1.150	1.187	-0.692	3.102	0.866	0.822	0.980	1.311	0.394	1.119	0.890
+1	0.481	-1.042	0.148	1.135	-1.249	1.202	-0.344	0.308	1.087	0.779	-1.431	1.581	0.000	0.860	-0.860	-1.125	0.000	0.785	0.303	1.199	3.102	0.878	0.853	0.988	1.072	0.827	0.936	0.815
+0	1.348	0.497	0.318	0.806	0.976	1.393	-0.152	0.632	2.173	2.130	0.515	-1.054	0.000	0.908	0.062	-0.780	0.000	1.185	0.687	1.668	1.551	0.720	0.898	0.985	0.683	1.292	1.320	1.131
+0	2.677	-0.420	-1.685	1.828	1.433	2.040	-0.718	-0.039	0.000	0.400	-0.873	0.472	0.000	0.444	0.340	-0.830	2.548	0.431	0.768	-1.417	3.102	0.869	0.917	0.996	0.707	0.193	0.728	1.154
+1	1.300	0.586	-0.122	1.306	0.609	0.727	-0.556	-1.652	2.173	0.636	0.720	1.393	2.215	0.328	1.280	-0.390	0.000	0.386	0.752	-0.905	0.000	0.202	0.751	1.106	0.864	0.799	0.928	0.717
+0	0.637	-0.176	1.737	1.322	-0.414	0.702	-0.964	-0.680	0.000	1.054	-0.461	0.889	2.215	0.861	-0.267	0.225	0.000	1.910	-1.888	1.027	0.000	0.919	0.899	1.186	0.993	1.109	0.862	0.775
+1	0.723	-0.104	1.572	0.428	-0.840	0.655	0.544	1.401	2.173	1.522	-0.154	-0.452	2.215	0.996	0.190	0.273	0.000	1.906	-0.176	0.966	0.000	0.945	0.894	0.990	0.981	1.555	0.988	0.893
+0	2.016	-0.570	1.612	0.798	0.441	0.334	0.191	-0.909	0.000	0.939	0.146	0.021	2.215	0.553	-0.444	1.156	2.548	0.781	-1.545	-0.520	0.000	0.922	0.956	1.528	0.722	0.699	0.778	0.901
+0	1.352	-0.707	1.284	0.665	0.580	0.694	-1.040	-0.899	2.173	0.692	-2.048	0.029	0.000	0.545	-2.042	1.259	0.000	0.661	-0.808	-1.251	3.102	0.845	0.991	0.979	0.662	0.225	0.685	0.769
+1	1.057	-1.561	-0.411	0.952	-0.681	1.236	-1.107	1.045	2.173	1.288	-2.521	-0.521	0.000	1.361	-1.239	1.546	0.000	0.373	-1.540	0.028	0.000	0.794	0.782	0.987	0.889	0.832	0.972	0.828
+0	1.118	-0.017	-1.227	1.077	1.256	0.714	0.624	-0.811	0.000	0.800	0.704	0.387	1.107	0.604	0.234	0.986	0.000	1.306	-0.456	0.094	3.102	0.828	0.984	1.195	0.987	0.672	0.774	0.748
+1	0.602	2.201	0.212	0.119	0.182	0.474	2.130	1.270	0.000	0.370	2.088	-0.573	0.000	0.780	-0.725	-1.033	0.000	1.642	0.598	0.303	3.102	0.886	0.988	0.985	0.644	0.756	0.651	0.599
+0	1.677	-0.844	1.581	0.585	0.887	1.012	-2.315	0.752	0.000	1.077	0.748	-0.195	0.000	0.718	0.832	-1.337	1.274	1.181	-0.557	-1.006	3.102	1.018	1.247	0.988	0.908	0.651	1.311	1.120
+1	1.695	0.259	1.224	1.344	1.067	0.718	-1.752	-0.215	0.000	0.473	0.991	-0.993	0.000	0.891	1.285	-1.500	2.548	0.908	-0.131	0.288	0.000	0.945	0.824	0.979	1.009	0.951	0.934	0.833
+0	0.793	0.628	0.432	1.707	0.302	0.919	1.045	-0.784	0.000	1.472	0.175	-1.284	2.215	1.569	0.155	0.971	2.548	0.435	0.735	1.625	0.000	0.801	0.907	0.992	0.831	1.446	1.082	1.051
+1	0.537	-0.664	-0.244	1.104	1.272	1.154	0.394	1.633	0.000	1.527	0.963	0.559	2.215	1.744	0.650	-0.912	0.000	1.097	0.730	-0.368	3.102	1.953	1.319	1.045	1.309	0.869	1.196	1.126
+1	0.585	-1.469	1.005	0.749	-1.060	1.224	-0.717	-0.323	2.173	1.012	-0.201	1.268	0.000	0.359	-0.567	0.476	0.000	1.117	-1.124	1.557	3.102	0.636	1.281	0.986	0.616	1.289	0.890	0.881
+1	0.354	-1.517	0.667	2.534	-1.298	1.020	-0.375	1.254	0.000	1.119	-0.060	-1.538	2.215	1.059	-0.395	-0.140	0.000	2.609	0.199	-0.778	1.551	0.957	0.975	1.286	1.666	1.003	1.224	1.135
+1	0.691	-1.619	-1.380	0.361	1.727	1.493	-1.093	-0.289	0.000	1.447	-0.640	1.341	0.000	1.453	-0.617	-1.456	1.274	1.061	-1.481	-0.091	0.000	0.744	0.649	0.987	0.596	0.727	0.856	0.797
+0	1.336	1.293	-1.359	0.357	0.067	1.110	-0.058	-0.515	0.000	0.976	1.498	1.207	0.000	1.133	0.437	1.053	2.548	0.543	1.374	0.171	0.000	0.764	0.761	0.984	0.827	0.553	0.607	0.612
+0	0.417	-1.111	1.661	2.209	-0.683	1.931	-0.642	0.959	1.087	1.514	-2.032	-0.686	0.000	1.521	-0.539	1.344	0.000	0.978	-0.866	0.363	1.551	2.813	1.850	1.140	1.854	0.799	1.600	1.556
+0	1.058	0.390	-0.591	0.134	1.149	0.346	-1.550	0.186	0.000	1.108	-0.999	0.843	1.107	1.124	0.415	-1.514	0.000	1.067	-0.426	-1.000	3.102	1.744	1.050	0.985	1.006	1.010	0.883	0.789
+1	1.655	0.253	1.216	0.270	1.703	0.500	-0.006	-1.418	2.173	0.690	-0.350	0.170	2.215	1.045	-0.924	-0.774	0.000	0.996	-0.745	-0.123	0.000	0.839	0.820	0.993	0.921	0.869	0.725	0.708
+0	1.603	-0.850	0.564	0.829	0.093	1.270	-1.113	-1.155	2.173	0.853	-1.021	1.248	2.215	0.617	-1.270	1.733	0.000	0.935	-0.092	0.136	0.000	1.011	1.074	0.977	0.823	1.269	1.054	0.878
+0	1.568	-0.792	1.005	0.545	0.896	0.895	-1.698	-0.988	0.000	0.608	-1.634	1.705	0.000	0.826	0.208	0.618	1.274	2.063	-1.743	-0.520	0.000	0.939	0.986	0.990	0.600	0.435	1.033	1.087
+0	0.489	-1.335	-1.102	1.738	1.028	0.628	-0.992	-0.627	0.000	0.652	-0.064	-0.215	0.000	1.072	0.173	-1.251	2.548	1.042	0.057	0.841	3.102	0.823	0.895	1.200	1.164	0.770	0.837	0.846
+1	1.876	0.870	1.234	0.556	-1.262	1.764	0.855	-0.467	2.173	1.079	1.351	0.852	0.000	0.773	0.383	0.874	0.000	1.292	0.829	-1.228	3.102	0.707	0.969	1.102	1.601	1.017	1.112	1.028
+0	1.033	0.407	-0.374	0.705	-1.254	0.690	-0.231	1.502	2.173	0.433	-2.009	-0.057	0.000	0.861	1.151	0.334	0.000	0.960	-0.839	1.299	3.102	2.411	1.480	0.982	0.995	0.377	1.012	0.994
+0	1.092	0.653	-0.801	0.463	0.426	0.529	-1.055	0.040	0.000	0.663	0.999	1.255	1.107	0.749	-1.106	1.185	2.548	0.841	-0.745	-1.029	0.000	0.841	0.743	0.988	0.750	1.028	0.831	0.868
+1	0.799	-0.285	-0.011	0.531	1.392	1.063	0.854	0.494	2.173	1.187	-1.065	-0.851	0.000	0.429	-0.296	1.072	0.000	0.942	-1.985	1.172	0.000	0.873	0.693	0.992	0.819	0.689	1.131	0.913
+0	0.503	1.973	-0.377	1.515	-1.514	0.708	1.081	-0.313	2.173	1.110	-0.417	0.839	0.000	0.712	-1.153	1.165	0.000	0.675	-0.303	-0.930	1.551	0.709	0.761	1.032	0.986	0.698	0.963	1.291
+0	0.690	-0.574	-1.608	1.182	1.118	0.557	-2.243	0.144	0.000	0.969	0.216	-1.383	1.107	1.054	0.888	-0.709	2.548	0.566	1.663	-0.550	0.000	0.752	1.528	0.987	1.408	0.740	1.290	1.123
+1	0.890	1.501	0.786	0.779	-0.615	1.126	0.716	1.541	2.173	0.887	0.728	-0.673	2.215	1.216	0.332	-0.020	0.000	0.965	1.828	0.101	0.000	0.827	0.715	1.099	1.088	1.339	0.924	0.878
+0	0.566	0.883	0.655	1.600	0.034	1.155	2.028	-1.499	0.000	0.723	-0.871	0.763	0.000	1.286	-0.696	-0.676	2.548	1.134	-0.113	1.207	3.102	4.366	2.493	0.984	0.960	0.962	1.843	1.511
+0	1.146	1.086	-0.911	0.838	1.298	0.821	0.127	-0.145	0.000	1.352	0.474	-1.580	2.215	1.619	-0.081	0.675	2.548	1.382	-0.748	0.127	0.000	0.958	0.976	1.239	0.876	1.481	1.116	1.076
+0	1.739	-0.326	-1.661	0.420	-1.705	1.193	-0.031	-1.212	2.173	1.783	-0.442	0.522	0.000	1.064	-0.692	0.027	0.000	1.314	0.359	-0.037	3.102	0.968	0.897	0.986	0.907	1.196	1.175	1.112
+1	0.669	0.194	-0.703	0.657	-0.260	0.899	-2.511	0.311	0.000	1.482	0.773	0.974	2.215	3.459	0.037	-1.299	1.274	2.113	0.067	1.516	0.000	0.740	0.871	0.979	1.361	2.330	1.322	1.046
+1	1.355	-1.033	-1.173	0.552	-0.048	0.899	-0.482	-1.287	2.173	1.422	-1.227	0.390	1.107	1.937	-0.028	0.914	0.000	0.849	-0.230	-1.734	0.000	0.986	1.224	1.017	1.051	1.788	1.150	1.009
+1	0.511	-0.202	1.029	0.780	1.154	0.816	0.532	-0.731	0.000	0.757	0.517	0.749	2.215	1.302	0.289	-1.188	0.000	0.584	1.211	-0.350	0.000	0.876	0.943	0.995	0.963	0.256	0.808	0.891
+1	1.109	0.572	1.484	0.753	1.543	1.711	-0.145	-0.746	1.087	1.759	0.631	0.845	2.215	0.945	0.542	0.003	0.000	0.378	-1.150	-0.044	0.000	0.764	1.042	0.992	1.045	2.736	1.441	1.140
+0	0.712	-0.025	0.553	0.928	-0.711	1.304	0.045	-0.300	0.000	0.477	0.720	0.969	0.000	1.727	-0.474	1.328	1.274	1.282	2.222	1.684	0.000	0.819	0.765	1.023	0.961	0.657	0.799	0.744
+1	1.131	-0.302	1.079	0.901	0.236	0.904	-0.249	1.694	2.173	1.507	-0.702	-1.128	0.000	0.774	0.565	0.284	2.548	1.802	1.446	-0.192	0.000	3.720	2.108	0.986	0.930	1.101	1.484	1.238
+0	1.392	1.253	0.118	0.864	-1.358	0.922	-0.447	-1.243	1.087	1.969	1.031	0.774	2.215	1.333	-0.359	-0.681	0.000	1.099	-0.257	1.473	0.000	1.246	0.909	1.475	1.234	2.531	1.449	1.306
+0	1.374	2.291	-0.479	1.339	-0.243	0.687	2.345	1.310	0.000	0.467	1.081	0.772	0.000	0.656	1.155	-1.636	2.548	0.592	0.536	-1.269	3.102	0.981	0.821	1.010	0.877	0.217	0.638	0.758
+1	0.401	-1.516	0.909	2.738	0.519	0.887	0.566	-1.202	0.000	0.909	-0.176	1.682	0.000	2.149	-0.878	-0.514	2.548	0.929	-0.563	-1.555	3.102	1.228	0.803	0.980	1.382	0.884	1.025	1.172
+1	0.430	-1.589	1.417	2.158	1.226	1.180	-0.829	-0.781	2.173	0.798	1.400	-0.111	0.000	0.939	-0.878	1.076	2.548	0.576	1.335	-0.826	0.000	0.861	0.970	0.982	1.489	1.308	1.015	0.992
+1	1.943	-0.391	-0.840	0.621	-1.613	2.026	1.734	1.025	0.000	0.930	0.573	-0.912	0.000	1.326	0.847	-0.220	1.274	1.181	0.079	0.709	3.102	1.164	1.007	0.987	1.094	0.821	0.857	0.786
+1	0.499	0.436	0.887	0.859	1.509	0.733	-0.559	1.111	1.087	1.011	-0.796	0.279	2.215	1.472	-0.510	-0.982	0.000	1.952	0.379	-0.733	0.000	1.076	1.358	0.991	0.589	0.879	1.068	0.922
+0	0.998	-0.407	-1.711	0.139	0.652	0.810	-0.331	-0.721	0.000	0.471	-0.533	0.442	0.000	0.531	-1.405	0.120	2.548	0.707	0.098	-1.176	1.551	1.145	0.809	0.988	0.529	0.612	0.562	0.609
+1	1.482	0.872	0.638	1.288	0.362	0.856	0.900	-0.511	1.087	1.072	1.061	-1.432	2.215	1.770	-2.292	-1.547	0.000	1.131	1.374	0.783	0.000	6.316	4.381	1.002	1.317	1.048	2.903	2.351
+1	2.084	-0.422	1.289	1.125	0.735	1.104	-0.518	-0.326	2.173	0.413	-0.719	-0.699	0.000	0.857	0.108	-1.631	0.000	0.527	0.641	-1.362	3.102	0.791	0.952	1.016	0.776	0.856	0.987	0.836
+0	0.464	0.674	0.025	0.430	-1.703	0.982	-1.311	-0.808	2.173	1.875	1.060	0.821	2.215	0.954	-0.480	-1.677	0.000	0.567	0.702	-0.939	0.000	0.781	1.076	0.989	1.256	3.632	1.652	1.252
+1	0.457	-1.944	-1.010	1.409	0.931	1.098	-0.742	-0.415	0.000	1.537	-0.834	0.945	2.215	1.752	-0.287	-1.269	2.548	0.692	-1.537	-0.223	0.000	0.801	1.192	1.094	1.006	1.659	1.175	1.122
+0	3.260	-0.943	1.737	0.920	1.309	0.946	-0.139	-0.271	2.173	0.994	-0.952	-0.311	0.000	0.563	-0.136	-0.881	0.000	1.236	-0.507	0.906	1.551	0.747	0.869	0.985	1.769	1.034	1.179	1.042
+0	0.615	-0.778	0.246	1.861	1.619	0.560	-0.943	-0.204	2.173	0.550	-0.759	-1.342	2.215	0.578	0.076	-0.973	0.000	0.939	0.035	0.680	0.000	0.810	0.747	1.401	0.772	0.702	0.719	0.662
+1	2.370	-0.064	-0.237	1.737	0.154	2.319	-1.838	-1.673	0.000	1.053	-1.305	-0.075	0.000	0.925	0.149	0.318	1.274	0.851	-0.922	0.981	3.102	0.919	0.940	0.989	0.612	0.598	1.219	1.626
+1	1.486	0.311	-1.262	1.354	-0.847	0.886	-0.158	1.213	2.173	1.160	-0.218	0.239	0.000	1.166	0.494	0.278	2.548	0.575	1.454	-1.701	0.000	0.429	1.129	0.983	1.111	1.049	1.006	0.920
+1	1.294	1.587	-0.864	0.487	-0.312	0.828	1.051	-0.031	1.087	2.443	1.216	1.609	2.215	1.167	0.813	0.921	0.000	1.751	-0.415	0.119	0.000	1.015	1.091	0.974	1.357	2.093	1.178	1.059
+1	0.984	0.465	-1.661	0.379	-0.554	0.977	0.237	0.365	0.000	0.510	0.143	1.101	0.000	1.099	-0.662	-1.593	2.548	1.104	-0.197	-0.648	3.102	0.925	0.922	0.986	0.642	0.667	0.806	0.722
+1	0.930	-0.009	0.047	0.667	1.367	1.065	-0.231	0.815	0.000	1.199	-1.114	-0.877	2.215	0.940	0.824	-1.583	0.000	1.052	-0.407	-0.076	1.551	1.843	1.257	1.013	1.047	0.751	1.158	0.941
+0	0.767	-0.011	-0.637	0.341	-1.437	1.438	-0.425	-0.450	2.173	1.073	-0.718	1.341	2.215	0.633	-1.394	0.486	0.000	0.603	-1.945	-1.626	0.000	0.703	0.790	0.984	1.111	1.848	1.129	1.072
+1	1.779	0.017	0.432	0.402	1.022	0.959	1.480	1.595	2.173	1.252	1.365	0.006	0.000	1.188	-0.174	-1.107	0.000	1.181	0.518	-0.258	0.000	1.057	0.910	0.991	1.616	0.779	1.158	1.053
+0	0.881	0.630	1.029	1.990	0.508	1.102	0.742	-1.298	2.173	1.565	1.085	0.686	2.215	2.691	1.391	-0.904	0.000	0.499	1.388	-1.199	0.000	0.347	0.861	0.997	0.881	1.920	1.233	1.310
+0	1.754	-0.266	0.389	0.347	-0.030	0.462	-1.408	-0.957	2.173	0.515	-2.341	-1.700	0.000	0.588	-0.797	1.355	2.548	0.608	0.329	-1.389	0.000	1.406	0.909	0.988	0.760	0.593	0.768	0.847
+0	1.087	0.311	-1.447	0.173	0.567	0.854	0.362	0.584	0.000	1.416	-0.716	-1.211	2.215	0.648	-0.358	-0.692	1.274	0.867	-0.513	0.206	0.000	0.803	0.813	0.984	1.110	0.491	0.921	0.873
+0	0.279	1.114	-1.190	3.004	-0.738	1.233	0.896	1.092	2.173	0.454	-0.374	0.117	2.215	0.357	0.119	1.270	0.000	0.458	1.343	0.316	0.000	0.495	0.540	0.988	1.715	1.139	1.618	1.183
+1	1.773	-0.694	-1.518	2.306	-1.200	3.104	0.749	0.362	0.000	1.871	0.230	-1.686	2.215	0.805	-0.179	-0.871	1.274	0.910	0.607	-0.246	0.000	1.338	1.598	0.984	1.050	0.919	1.678	1.807
+0	0.553	0.683	0.827	0.973	-0.706	1.488	0.149	1.140	2.173	1.788	0.447	-0.478	0.000	0.596	1.043	1.607	0.000	0.373	-0.868	-1.308	1.551	1.607	1.026	0.998	1.134	0.808	1.142	0.936
+1	0.397	1.101	-1.139	1.688	0.146	0.972	0.541	1.518	0.000	1.549	-0.873	-1.012	0.000	2.282	-0.151	0.314	2.548	1.174	0.033	-1.368	0.000	0.937	0.776	1.039	1.143	0.959	0.986	1.013
+1	0.840	1.906	-0.959	0.869	0.576	0.642	0.554	-1.351	0.000	0.756	0.923	-0.823	2.215	1.251	1.130	0.545	2.548	1.513	0.410	1.073	0.000	1.231	0.985	1.163	0.812	0.987	0.816	0.822
+1	0.477	1.665	0.814	0.763	-0.382	0.828	-0.008	0.280	2.173	1.213	-0.001	1.560	0.000	1.136	0.311	-1.289	0.000	0.797	1.091	-0.616	3.102	1.026	0.964	0.992	0.772	0.869	0.916	0.803
+0	2.655	0.020	0.273	1.464	0.482	1.709	-0.107	-1.456	2.173	0.825	0.141	-0.386	0.000	1.342	-0.592	1.635	1.274	0.859	-0.175	-0.874	0.000	0.829	0.946	1.003	2.179	0.836	1.505	1.176
+0	0.771	-1.992	-0.720	0.732	-1.464	0.869	-1.290	0.388	2.173	0.926	-1.072	-1.489	2.215	0.640	-1.232	0.840	0.000	0.528	-2.440	-0.446	0.000	0.811	0.868	0.993	0.995	1.317	0.809	0.714
+0	1.357	1.302	0.076	0.283	-1.060	0.783	1.559	-0.994	0.000	0.947	1.212	1.617	0.000	1.127	0.311	0.442	2.548	0.582	-0.052	1.186	1.551	1.330	0.995	0.985	0.846	0.404	0.858	0.815
+0	0.442	-0.381	-0.424	1.244	0.591	0.731	0.605	-0.713	2.173	0.629	2.762	1.040	0.000	0.476	2.693	-0.617	0.000	0.399	0.442	1.486	3.102	0.839	0.755	0.988	0.869	0.524	0.877	0.918
+0	0.884	0.422	0.055	0.818	0.624	0.950	-0.763	1.624	0.000	0.818	-0.609	-1.166	0.000	1.057	-0.528	1.070	2.548	1.691	-0.124	-0.335	3.102	1.104	0.933	0.985	0.913	1.000	0.863	1.056
+0	1.276	0.156	1.714	1.053	-1.189	0.672	-0.464	-0.030	2.173	0.469	-2.483	0.442	0.000	0.564	2.580	-0.253	0.000	0.444	-0.628	1.080	1.551	5.832	2.983	0.985	1.162	0.494	1.809	1.513
+0	1.106	-0.556	0.406	0.573	-1.400	0.769	-0.518	1.457	2.173	0.743	-0.352	-0.010	0.000	1.469	-0.550	-0.930	2.548	0.540	1.236	-0.571	0.000	0.962	0.970	1.101	0.805	1.107	0.873	0.773
+0	0.539	-0.964	-0.464	1.371	-1.606	0.667	-0.160	0.655	0.000	0.952	0.352	-0.740	2.215	0.952	0.007	1.123	0.000	1.061	-0.505	1.389	3.102	1.063	0.991	1.019	0.633	0.967	0.732	0.799
+1	0.533	-0.989	-1.608	0.462	-1.723	1.204	-0.598	-0.098	2.173	1.343	-0.460	1.632	2.215	0.577	0.221	-0.492	0.000	0.628	-0.073	0.472	0.000	0.518	0.880	0.988	1.179	1.874	1.041	0.813
+1	1.024	1.075	-0.795	0.286	-1.436	1.365	0.857	-0.309	2.173	0.804	1.532	1.435	0.000	1.511	0.722	1.494	0.000	1.778	0.903	0.753	1.551	0.686	0.810	0.999	0.900	1.360	1.133	0.978
+1	2.085	-0.269	-1.423	0.789	1.298	0.281	1.652	0.187	0.000	0.658	-0.760	-0.042	2.215	0.663	0.024	0.120	0.000	0.552	-0.299	-0.428	3.102	0.713	0.811	1.130	0.705	0.218	0.675	0.743
+1	0.980	-0.443	0.813	0.785	-1.253	0.719	0.448	-1.458	0.000	1.087	0.595	0.635	1.107	1.428	0.029	-0.995	0.000	1.083	1.562	-0.092	0.000	0.834	0.891	1.165	0.967	0.661	0.880	0.817
+1	0.903	-0.733	-0.980	0.634	-0.639	0.780	0.266	-0.287	2.173	1.264	-0.936	1.004	0.000	1.002	-0.056	-1.344	2.548	1.183	-0.098	1.169	0.000	0.733	1.002	0.985	0.711	0.916	0.966	0.875
+0	0.734	-0.304	-1.175	2.851	1.674	0.904	-0.634	0.412	2.173	1.363	-1.050	-0.282	0.000	1.476	-1.603	0.103	0.000	2.231	-0.718	1.708	3.102	0.813	0.896	1.088	0.686	1.392	1.033	1.078
+1	1.680	0.591	-0.243	0.111	-0.478	0.326	-0.079	-1.555	2.173	0.711	0.714	0.922	2.215	0.355	0.858	1.682	0.000	0.727	1.620	1.360	0.000	0.334	0.526	1.001	0.862	0.633	0.660	0.619
+1	1.163	0.225	-0.202	0.501	-0.979	1.609	-0.938	1.424	0.000	1.224	-0.118	-1.274	0.000	2.034	1.241	-0.254	0.000	1.765	0.536	0.237	3.102	0.894	0.838	0.988	0.693	0.579	0.762	0.726
+0	1.223	1.232	1.471	0.489	1.728	0.703	-0.111	0.411	0.000	1.367	1.014	-1.294	1.107	1.524	-0.414	-0.164	2.548	1.292	0.833	0.316	0.000	0.861	0.752	0.994	0.836	1.814	1.089	0.950
+0	0.816	1.637	-1.557	1.036	-0.342	0.913	1.333	0.949	2.173	0.812	0.756	-0.628	2.215	1.333	0.470	1.495	0.000	1.204	-2.222	-1.675	0.000	1.013	0.924	1.133	0.758	1.304	0.855	0.860
+0	0.851	-0.564	-0.691	0.692	1.345	1.219	1.014	0.318	0.000	1.422	-0.262	-1.635	2.215	0.531	1.802	0.008	0.000	0.508	0.515	-1.267	3.102	0.821	0.787	1.026	0.783	0.432	1.149	1.034
+0	0.800	-0.599	0.204	0.552	-0.484	0.974	0.413	0.961	2.173	1.269	-0.984	-1.039	2.215	0.380	-1.213	1.371	0.000	0.551	0.332	-0.659	0.000	0.694	0.852	0.984	1.057	2.037	1.096	0.846
+0	0.744	-0.071	-0.255	0.638	0.512	1.125	0.407	0.844	2.173	0.860	-0.481	-0.677	0.000	1.102	0.181	-1.194	0.000	1.011	-1.081	-1.713	3.102	0.854	0.862	0.982	1.111	1.372	1.042	0.920
+1	0.400	1.049	-0.625	0.880	-0.407	1.040	2.150	-1.359	0.000	0.747	-0.144	0.847	2.215	0.560	-1.829	0.698	0.000	1.663	-0.668	0.267	0.000	0.845	0.964	0.996	0.820	0.789	0.668	0.668
+0	1.659	-0.705	-1.057	1.803	-1.436	1.008	0.693	0.005	0.000	0.895	-0.007	0.681	1.107	1.085	0.125	1.476	2.548	1.214	1.068	0.486	0.000	0.867	0.919	0.986	1.069	0.692	1.026	1.313
+0	0.829	-0.153	0.861	0.615	-0.548	0.589	1.077	-0.041	2.173	1.056	0.763	-1.737	0.000	0.639	0.970	0.725	0.000	0.955	1.227	-0.799	3.102	1.020	1.024	0.985	0.750	0.525	0.685	0.671
+1	0.920	-0.806	-0.840	1.048	0.278	0.973	-0.077	-1.364	2.173	1.029	0.309	0.133	0.000	1.444	1.484	1.618	1.274	1.419	-0.482	0.417	0.000	0.831	1.430	1.151	1.829	1.560	1.343	1.224
+1	0.686	0.249	-0.905	0.343	-1.731	0.724	-2.823	-0.901	0.000	0.982	0.303	1.312	1.107	1.016	0.245	0.610	0.000	1.303	-0.557	-0.360	3.102	1.384	1.030	0.984	0.862	1.144	0.866	0.779
+0	1.603	0.444	0.508	0.586	0.401	0.610	0.467	-1.735	2.173	0.914	0.626	-1.019	0.000	0.812	0.422	-0.408	2.548	0.902	1.679	1.490	0.000	1.265	0.929	0.990	1.004	0.816	0.753	0.851
+1	0.623	0.780	-0.203	0.056	0.015	0.899	0.793	1.326	1.087	0.803	1.478	-1.499	2.215	1.561	1.492	-0.120	0.000	0.904	0.795	0.137	0.000	0.548	1.009	0.850	0.924	0.838	0.914	0.860
+0	1.654	-2.032	-1.160	0.859	-1.583	0.689	-1.965	0.891	0.000	0.646	-1.014	-0.288	2.215	0.630	-0.815	0.402	0.000	0.638	0.316	0.655	3.102	0.845	0.879	0.993	1.067	0.625	1.041	0.958
+1	0.828	-1.269	-1.203	0.744	-0.213	0.626	-1.017	-0.404	0.000	1.281	-0.931	1.733	2.215	0.699	-0.351	1.287	0.000	1.251	-1.171	0.197	0.000	0.976	1.186	0.987	0.646	0.655	0.733	0.671
+1	0.677	0.111	1.090	1.580	1.591	1.560	0.654	-0.341	2.173	0.794	-0.266	0.702	0.000	0.823	0.651	-1.239	2.548	0.730	1.467	-1.530	0.000	1.492	1.023	0.983	1.909	1.022	1.265	1.127
+1	0.736	0.882	-1.060	0.589	0.168	1.663	0.781	1.022	2.173	2.025	1.648	-1.292	0.000	1.240	0.924	-0.421	1.274	1.354	0.065	0.501	0.000	0.316	0.925	0.988	0.664	1.736	0.992	0.807
+1	1.040	-0.822	1.638	0.974	-0.674	0.393	0.830	0.011	2.173	0.770	-0.140	-0.402	0.000	0.294	-0.133	0.030	0.000	1.220	0.807	0.638	0.000	0.826	1.063	1.216	1.026	0.705	0.934	0.823
+1	0.711	0.602	0.048	1.145	0.966	0.934	0.263	-1.589	2.173	0.971	-0.496	-0.421	1.107	0.628	-0.865	0.845	0.000	0.661	-0.008	-0.565	0.000	0.893	0.705	0.988	0.998	1.339	0.908	0.872
+1	0.953	-1.651	-0.167	0.885	1.053	1.013	-1.239	0.133	0.000	1.884	-1.122	1.222	2.215	1.906	-0.860	-1.184	1.274	1.413	-0.668	-1.647	0.000	1.873	1.510	1.133	1.050	1.678	1.246	1.061
+1	0.986	-0.892	-1.380	0.917	1.134	0.950	-1.162	-0.469	0.000	0.569	-1.393	0.215	0.000	0.320	2.667	1.712	0.000	1.570	-0.375	1.457	3.102	0.925	1.128	1.011	0.598	0.824	0.913	0.833
+1	1.067	0.099	1.154	0.527	-0.789	1.085	0.623	-1.602	2.173	1.511	-0.230	0.022	2.215	0.269	-0.377	0.883	0.000	0.571	-0.540	-0.512	0.000	0.414	0.803	1.022	0.959	2.053	1.041	0.780
+0	0.825	-2.118	0.217	1.453	-0.493	0.819	0.313	-0.942	0.000	2.098	-0.725	1.096	2.215	0.484	1.336	1.458	0.000	0.482	0.100	1.163	0.000	0.913	0.536	0.990	1.679	0.957	1.095	1.143
+1	1.507	0.054	1.120	0.698	-1.340	0.912	0.384	0.015	1.087	0.720	0.247	-0.820	0.000	0.286	0.154	1.578	2.548	0.629	1.582	-0.576	0.000	0.828	0.893	1.136	0.514	0.632	0.699	0.709
+1	0.610	1.180	-0.993	0.816	0.301	0.932	0.758	1.539	0.000	0.726	-0.830	0.248	2.215	0.883	0.857	-1.305	0.000	1.338	1.009	-0.252	3.102	0.901	1.074	0.987	0.875	1.159	1.035	0.858
+1	1.247	-1.360	1.502	1.525	-1.332	0.618	1.063	0.755	0.000	0.582	-0.155	0.473	2.215	1.214	-0.422	-0.551	2.548	0.838	-1.171	-1.166	0.000	2.051	1.215	1.062	1.091	0.725	0.896	1.091
+0	0.373	-0.600	1.291	2.573	0.207	0.765	-0.209	1.667	0.000	0.668	0.724	-1.499	0.000	1.045	-0.338	-0.754	2.548	0.558	-0.469	0.029	3.102	0.868	0.939	1.124	0.519	0.383	0.636	0.838
+0	0.791	0.336	-0.307	0.494	1.213	1.158	0.336	1.081	2.173	0.918	1.289	-0.449	0.000	0.735	-0.521	-0.969	0.000	1.052	0.499	-1.188	3.102	0.699	1.013	0.987	0.622	1.050	0.712	0.661
+0	1.321	0.856	0.464	0.202	0.901	1.144	0.120	-1.651	0.000	0.803	0.577	-0.509	2.215	0.695	-0.114	0.423	2.548	0.621	1.852	-0.420	0.000	0.697	0.964	0.983	0.527	0.659	0.719	0.729
+0	0.563	2.081	0.913	0.982	-0.533	0.549	-0.481	-1.730	0.000	0.962	0.921	0.569	2.215	0.731	1.184	-0.679	1.274	0.918	0.931	-1.432	0.000	1.008	0.919	0.993	0.895	0.819	0.810	0.878
+1	1.148	0.345	0.953	0.921	0.617	0.991	1.103	-0.484	0.000	0.970	1.978	1.525	0.000	1.150	0.689	-0.757	2.548	0.517	0.995	1.245	0.000	1.093	1.140	0.998	1.006	0.756	0.864	0.838
+1	1.400	0.128	-1.695	1.169	1.070	1.094	-0.345	-0.249	0.000	1.224	0.364	-0.036	2.215	1.178	0.530	-1.544	0.000	1.334	0.933	1.604	0.000	0.560	1.267	1.073	0.716	0.780	0.832	0.792
+0	0.330	-2.133	1.403	0.628	0.379	1.686	-0.995	0.030	1.087	2.071	0.127	-0.457	0.000	4.662	-0.855	1.477	0.000	2.072	-0.917	-1.416	3.102	5.403	3.074	0.977	0.936	1.910	2.325	1.702
+0	0.989	0.473	0.968	1.970	1.368	0.844	0.574	-0.290	2.173	0.866	-0.345	-1.019	0.000	1.130	0.605	-0.752	0.000	0.956	-0.888	0.870	3.102	0.885	0.886	0.982	1.157	1.201	1.100	1.068
+1	0.773	0.418	0.753	1.388	1.070	1.104	-0.378	-0.758	0.000	1.027	0.397	-0.496	2.215	1.234	0.027	1.084	2.548	0.936	0.209	1.677	0.000	1.355	1.020	0.983	0.550	1.206	0.916	0.931
+0	0.319	2.015	1.534	0.570	-1.134	0.632	0.124	0.757	0.000	0.477	0.598	-1.109	1.107	0.449	0.438	-0.755	2.548	0.574	-0.659	0.691	0.000	0.440	0.749	0.985	0.517	0.158	0.505	0.522
+0	1.215	1.453	-1.386	1.276	1.298	0.643	0.570	-0.196	2.173	0.588	2.104	0.498	0.000	0.617	-0.296	-0.801	2.548	0.452	0.110	0.313	0.000	0.815	0.953	1.141	1.166	0.547	0.892	0.807
+1	1.257	-1.869	-0.060	0.265	0.653	1.527	-0.346	1.163	2.173	0.758	-2.119	-0.604	0.000	1.473	-1.133	-1.290	2.548	0.477	-0.428	-0.066	0.000	0.818	0.841	0.984	1.446	1.729	1.211	1.054
+1	1.449	0.464	1.585	1.418	-1.488	1.540	0.942	0.087	0.000	0.898	0.402	-0.631	2.215	0.753	0.039	-1.729	0.000	0.859	0.849	-1.054	0.000	0.791	0.677	0.995	0.687	0.527	0.703	0.606
+1	1.084	-1.997	0.900	1.333	1.024	0.872	-0.864	-1.500	2.173	1.072	-0.813	-0.421	2.215	0.924	0.478	0.304	0.000	0.992	-0.398	-1.022	0.000	0.741	1.085	0.980	1.221	1.176	1.032	0.961
+0	1.712	1.129	0.125	1.120	-1.402	1.749	0.951	-1.575	2.173	1.711	0.445	0.578	0.000	1.114	0.234	-1.011	0.000	1.577	-0.088	0.086	3.102	2.108	1.312	1.882	1.597	2.009	1.441	1.308
+0	0.530	0.248	1.622	1.450	-1.012	1.221	-1.154	-0.763	2.173	1.698	-0.586	0.733	0.000	0.889	1.042	1.038	1.274	0.657	0.008	0.701	0.000	0.430	1.005	0.983	0.930	2.264	1.357	1.146
+1	0.921	1.735	0.883	0.699	-1.614	0.821	1.463	0.319	1.087	1.099	0.814	-1.600	2.215	1.375	0.702	-0.691	0.000	0.869	1.326	-0.790	0.000	0.980	0.900	0.988	0.832	1.452	0.816	0.709
+0	2.485	-0.823	-0.297	0.886	-1.404	0.989	0.835	1.615	2.173	0.382	0.588	-0.224	0.000	1.029	-0.456	1.546	2.548	0.613	-0.359	-0.789	0.000	0.768	0.977	1.726	2.007	0.913	1.338	1.180
+1	0.657	-0.069	-0.078	1.107	1.549	0.804	1.335	-1.630	2.173	1.271	0.481	0.153	1.107	1.028	0.144	-0.762	0.000	1.098	0.132	1.570	0.000	0.830	0.979	1.175	1.069	1.624	1.000	0.868
+1	2.032	0.329	-1.003	0.493	-0.136	1.159	-0.224	0.750	1.087	0.396	0.546	0.587	0.000	0.620	1.805	0.982	0.000	1.236	0.744	-1.621	0.000	0.930	1.200	0.988	0.482	0.771	0.887	0.779
+0	0.524	-1.319	0.634	0.471	1.221	0.599	-0.588	-0.461	0.000	1.230	-1.504	-1.517	1.107	1.436	-0.035	0.104	2.548	0.629	1.997	-1.282	0.000	2.084	1.450	0.984	1.084	1.827	1.547	1.213
+1	0.871	0.618	-1.544	0.718	0.186	1.041	-1.180	0.434	2.173	1.133	1.558	-1.301	0.000	0.452	-0.595	0.522	0.000	0.665	0.567	0.130	3.102	1.872	1.114	1.095	1.398	0.979	1.472	1.168
+1	3.308	1.037	-0.634	0.690	-0.619	1.975	0.949	1.280	0.000	0.826	0.546	-0.139	2.215	0.635	-0.045	0.427	0.000	1.224	0.112	1.339	3.102	1.756	1.050	0.992	0.738	0.903	0.968	1.238
+0	0.588	2.104	-0.872	1.136	1.743	0.842	0.638	0.015	0.000	0.481	0.928	1.000	2.215	0.595	0.125	1.429	0.000	0.951	-1.140	-0.511	3.102	1.031	1.057	0.979	0.673	1.064	1.001	0.891
+0	0.289	0.823	0.013	0.615	-1.601	0.177	2.403	-0.015	0.000	0.258	1.151	1.036	2.215	0.694	0.553	-1.326	2.548	0.411	0.366	0.106	0.000	0.482	0.562	0.989	0.670	0.404	0.516	0.561
+1	0.294	-0.660	-1.162	1.752	0.384	0.860	0.513	1.119	0.000	2.416	0.107	-1.342	0.000	1.398	0.361	-0.350	2.548	1.126	-0.902	0.040	1.551	0.650	1.125	0.988	0.531	0.843	0.912	0.911
+0	0.599	-0.616	1.526	1.381	0.507	0.955	-0.646	-0.085	2.173	0.775	-0.533	1.116	2.215	0.789	-0.136	-1.176	0.000	2.449	1.435	-1.433	0.000	1.692	1.699	1.000	0.869	1.119	1.508	1.303
+1	1.100	-1.174	-1.114	1.601	-1.576	1.056	-1.343	0.547	2.173	0.555	0.367	0.592	2.215	0.580	-1.862	-0.914	0.000	0.904	0.508	-0.444	0.000	1.439	1.105	0.986	1.408	1.104	1.190	1.094
+1	2.237	-0.701	1.470	0.719	-0.199	0.745	-0.132	-0.737	1.087	0.976	-0.227	0.093	2.215	0.699	0.057	1.133	0.000	0.661	0.573	-0.679	0.000	0.785	0.772	1.752	1.235	0.856	0.990	0.825
+1	0.455	-0.880	-1.482	1.260	-0.178	1.499	0.158	1.022	0.000	1.867	-0.435	-0.675	2.215	1.234	0.783	1.586	0.000	0.641	-0.454	-0.409	3.102	1.002	0.964	0.986	0.761	0.240	1.190	0.995
+1	1.158	-0.778	-0.159	0.823	1.641	1.341	-0.830	-1.169	2.173	0.840	-1.554	0.934	0.000	0.693	0.488	-1.218	2.548	1.042	1.395	0.276	0.000	0.946	0.785	1.350	1.079	0.893	1.267	1.151
+1	0.902	-0.078	-0.055	0.872	-0.012	0.843	1.276	1.739	2.173	0.838	1.492	0.918	0.000	0.626	0.904	-0.648	2.548	0.412	-2.027	-0.883	0.000	2.838	1.664	0.988	1.803	0.768	1.244	1.280
+1	0.649	-1.028	-1.521	1.097	0.774	1.216	-0.383	-0.318	2.173	1.643	-0.285	-1.705	0.000	0.911	-0.091	0.341	0.000	0.592	0.537	0.732	3.102	0.911	0.856	1.027	1.160	0.874	0.986	0.893
+1	1.192	1.846	-0.781	1.326	-0.747	1.550	1.177	1.366	0.000	1.196	0.151	0.387	2.215	0.527	2.261	-0.190	0.000	0.390	1.474	0.381	0.000	0.986	1.025	1.004	1.392	0.761	0.965	1.043
+0	0.438	-0.358	-1.549	0.836	0.436	0.818	0.276	-0.708	2.173	0.707	0.826	0.392	0.000	1.050	1.741	-1.066	0.000	1.276	-1.583	0.842	0.000	1.475	1.273	0.986	0.853	1.593	1.255	1.226
+1	1.083	0.142	1.701	0.605	-0.253	1.237	0.791	1.183	2.173	0.842	2.850	-0.082	0.000	0.724	-0.464	-0.694	0.000	1.499	0.456	-0.226	3.102	0.601	0.799	1.102	0.995	1.389	1.013	0.851
+0	0.828	1.897	-0.615	0.572	-0.545	0.572	0.461	0.464	2.173	0.393	0.356	1.069	2.215	1.840	0.088	1.500	0.000	0.407	-0.663	-0.787	0.000	0.950	0.965	0.979	0.733	0.363	0.618	0.733
+0	0.735	1.438	1.197	1.123	-0.214	0.641	0.949	0.858	0.000	1.162	0.524	-0.896	2.215	0.992	0.454	-1.475	2.548	0.902	1.079	0.019	0.000	0.822	0.917	1.203	1.032	0.569	0.780	0.764
+0	0.437	-2.102	0.044	1.779	-1.042	1.231	-0.181	-0.515	1.087	2.666	0.863	1.466	2.215	1.370	0.345	-1.371	0.000	0.906	0.363	1.611	0.000	1.140	1.362	1.013	3.931	3.004	2.724	2.028
+1	0.881	1.814	-0.987	0.384	0.800	2.384	1.422	0.640	0.000	1.528	0.292	-0.962	1.107	2.126	-0.371	-1.401	2.548	0.700	0.109	0.203	0.000	0.450	0.813	0.985	0.956	1.013	0.993	0.774
+1	0.630	0.408	0.152	0.194	0.316	0.710	-0.824	-0.358	2.173	0.741	0.535	-0.851	2.215	0.933	0.406	1.148	0.000	0.523	-0.479	-0.625	0.000	0.873	0.960	0.988	0.830	0.921	0.711	0.661
+1	0.870	-0.448	-1.134	0.616	0.135	0.600	0.649	-0.622	2.173	0.768	0.709	-0.123	0.000	1.308	0.500	1.468	0.000	1.973	-0.286	1.462	3.102	0.909	0.944	0.990	0.835	1.250	0.798	0.776
+0	1.290	0.552	1.330	0.615	-1.353	0.661	0.240	-0.393	0.000	0.531	0.053	-1.588	0.000	0.675	0.839	-0.345	1.274	1.597	0.020	0.536	3.102	1.114	0.964	0.987	0.783	0.675	0.662	0.675
+1	0.943	0.936	1.068	1.373	0.671	2.170	-2.011	-1.032	0.000	0.640	0.361	-0.806	0.000	2.239	-0.083	0.590	2.548	1.224	0.646	-1.723	0.000	0.879	0.834	0.981	1.436	0.568	0.916	0.931
+1	0.431	1.686	-1.053	0.388	1.739	0.457	-0.471	-0.743	2.173	0.786	1.432	-0.547	2.215	0.537	-0.413	1.256	0.000	0.413	2.311	-0.408	0.000	1.355	1.017	0.982	0.689	1.014	0.821	0.715
+0	1.620	-0.055	-0.862	1.341	-1.571	0.634	-0.906	0.935	2.173	0.501	-2.198	-0.525	0.000	0.778	-0.708	-0.060	0.000	0.988	-0.621	0.489	3.102	0.870	0.956	1.216	0.992	0.336	0.871	0.889
+1	0.549	0.304	-1.443	1.309	-0.312	1.116	0.644	1.519	2.173	1.078	-0.303	-0.736	0.000	1.261	0.387	0.628	2.548	0.945	-0.190	0.090	0.000	0.893	1.043	1.000	1.124	1.077	1.026	0.886
+0	0.412	-0.618	-1.486	1.133	-0.665	0.646	0.436	1.520	0.000	0.993	0.976	0.106	2.215	0.832	0.091	0.164	2.548	0.672	-0.650	1.256	0.000	0.695	1.131	0.991	1.017	0.455	1.226	1.087
+0	1.183	-0.084	1.644	1.389	0.967	0.843	0.938	-0.670	0.000	0.480	0.256	0.123	2.215	0.437	1.644	0.491	0.000	0.501	-0.416	0.101	3.102	1.060	0.804	1.017	0.775	0.173	0.535	0.760
+0	1.629	-1.486	-0.683	2.786	-0.492	1.347	-2.638	1.453	0.000	1.857	0.208	0.873	0.000	0.519	-1.265	-1.602	1.274	0.903	-1.102	-0.329	1.551	6.892	3.522	0.998	0.570	0.477	2.039	2.006
+1	2.045	-0.671	-1.235	0.490	-0.952	0.525	-1.252	1.289	0.000	1.088	-0.993	0.648	2.215	0.975	-0.109	-0.254	2.548	0.556	-1.095	-0.194	0.000	0.803	0.861	0.980	1.282	0.945	0.925	0.811
+0	0.448	-0.058	-0.974	0.945	-1.633	1.181	-1.139	0.266	2.173	1.118	-0.761	1.502	1.107	1.706	0.585	-0.680	0.000	0.487	-1.951	0.945	0.000	2.347	1.754	0.993	1.161	1.549	1.414	1.176
+0	0.551	0.519	0.448	2.183	1.293	1.220	0.628	-0.627	2.173	1.019	-0.002	-0.652	0.000	1.843	-0.386	1.042	2.548	0.400	-1.102	-1.014	0.000	0.648	0.792	1.049	0.888	2.132	1.262	1.096
+0	1.624	0.488	1.403	0.760	0.559	0.812	0.777	-1.244	2.173	0.613	0.589	-0.030	2.215	0.692	1.058	0.683	0.000	1.054	1.165	-0.765	0.000	0.915	0.875	1.059	0.821	0.927	0.792	0.721
+1	0.774	0.444	1.257	0.515	-0.689	0.515	1.448	-1.271	0.000	0.793	0.118	0.811	1.107	0.679	0.326	-0.426	0.000	1.066	-0.865	-0.049	3.102	0.960	1.046	0.986	0.716	0.772	0.855	0.732
+1	2.093	-1.240	1.615	0.918	-1.202	1.412	-0.541	0.640	1.087	2.019	0.872	-0.639	0.000	0.672	-0.936	0.972	0.000	0.896	0.235	0.212	0.000	0.810	0.700	1.090	0.797	0.862	1.049	0.874
+1	0.908	1.069	0.283	0.400	1.293	0.609	1.452	-1.136	0.000	0.623	0.417	-0.098	2.215	1.023	0.775	1.054	1.274	0.706	2.346	-1.305	0.000	0.744	1.006	0.991	0.606	0.753	0.796	0.753
+0	0.403	-1.328	-0.065	0.901	1.052	0.708	-0.354	-0.718	2.173	0.892	0.633	1.684	2.215	0.999	-1.205	0.941	0.000	0.930	1.072	-0.809	0.000	2.105	1.430	0.989	0.838	1.147	1.042	0.883
+0	1.447	0.453	0.118	1.731	0.650	0.771	0.446	-1.564	0.000	0.973	-2.014	0.354	0.000	1.949	-0.643	-1.531	1.274	1.106	-0.334	-1.163	0.000	0.795	0.821	1.013	1.699	0.918	1.118	1.018
+1	1.794	0.123	-0.454	0.057	1.489	0.966	-1.190	1.090	1.087	0.539	-0.535	1.035	0.000	1.096	-1.069	-1.236	2.548	0.659	-1.196	-0.283	0.000	0.803	0.756	0.985	1.343	1.109	0.993	0.806
+0	1.484	-2.047	0.813	0.591	-0.295	0.923	0.312	-1.164	2.173	0.654	-0.316	0.752	2.215	0.599	1.966	-1.128	0.000	0.626	-0.304	-1.431	0.000	1.112	0.910	1.090	0.986	1.189	1.350	1.472
+0	0.417	-2.016	0.849	1.817	0.040	1.201	-1.676	-1.394	0.000	0.792	0.537	0.641	2.215	0.794	-1.222	0.187	0.000	0.825	-0.217	1.334	3.102	1.470	0.931	0.987	1.203	0.525	0.833	0.827
+1	0.603	1.009	0.033	0.486	1.225	0.884	-0.617	-1.058	0.000	0.500	-1.407	-0.567	0.000	1.476	-0.876	0.605	2.548	0.970	0.560	1.092	3.102	0.853	1.153	0.988	0.846	0.920	0.944	0.835
+1	1.381	-0.326	0.552	0.417	-0.027	1.030	-0.835	-1.287	2.173	0.941	-0.421	1.519	2.215	0.615	-1.650	0.377	0.000	0.606	0.644	0.650	0.000	1.146	0.970	0.990	1.191	0.884	0.897	0.826
+1	0.632	1.200	-0.703	0.438	-1.700	0.779	-0.731	0.958	1.087	0.605	0.393	-1.376	0.000	0.670	-0.827	-1.315	2.548	0.626	-0.501	0.417	0.000	0.904	0.903	0.998	0.673	0.803	0.722	0.640
+1	1.561	-0.569	1.580	0.329	0.237	1.059	0.731	0.415	2.173	0.454	0.016	-0.828	0.000	0.587	0.008	-0.291	1.274	0.597	1.119	1.191	0.000	0.815	0.908	0.988	0.733	0.690	0.892	0.764
+1	2.102	0.087	0.449	1.164	-0.390	1.085	-0.408	-1.116	2.173	0.578	0.197	-0.137	0.000	1.202	0.917	1.523	0.000	0.959	-0.832	1.404	3.102	1.380	1.109	1.486	1.496	0.886	1.066	1.025
+1	1.698	-0.489	-0.552	0.976	-1.009	1.620	-0.721	0.648	1.087	1.481	-1.860	-1.354	0.000	1.142	-1.140	1.401	2.548	1.000	-1.274	-0.158	0.000	1.430	1.130	0.987	1.629	1.154	1.303	1.223
+1	1.111	-0.249	-1.457	0.421	0.939	0.646	-2.076	0.362	0.000	1.315	0.796	-1.436	2.215	0.780	0.130	0.055	0.000	1.662	-0.834	0.461	0.000	0.920	0.948	0.990	1.046	0.905	1.493	1.169
+1	0.945	0.390	-1.159	1.675	0.437	0.356	0.261	0.543	1.087	0.574	0.838	1.599	2.215	0.496	-1.220	-0.022	0.000	0.558	-2.454	1.440	0.000	0.763	0.983	1.728	1.000	0.578	0.922	1.003
+1	2.076	0.014	-1.314	0.854	-0.306	3.446	1.341	0.598	0.000	2.086	0.227	-0.747	2.215	1.564	-0.216	1.649	2.548	0.965	-0.857	-1.062	0.000	0.477	0.734	1.456	1.003	1.660	1.001	0.908
+1	1.992	0.192	-0.103	0.108	-1.599	0.938	0.595	-1.360	2.173	0.869	-1.012	1.432	0.000	1.302	0.850	0.436	2.548	0.487	1.051	-1.027	0.000	0.502	0.829	0.983	1.110	1.394	0.904	0.836
+0	0.460	1.625	1.485	1.331	1.242	0.675	-0.329	-1.039	1.087	0.671	-1.028	-0.514	0.000	1.265	-0.788	0.415	1.274	0.570	-0.683	-1.738	0.000	0.725	0.758	1.004	1.024	1.156	0.944	0.833
+0	0.871	0.839	-1.536	0.428	1.198	0.875	-1.256	-0.466	1.087	0.684	-0.768	0.150	0.000	0.556	-1.793	0.389	0.000	0.942	-1.126	1.339	1.551	0.624	0.734	0.986	1.357	0.960	1.474	1.294
+1	0.951	1.651	0.576	1.273	1.495	0.834	0.048	-0.578	2.173	0.386	-0.056	-1.448	0.000	0.597	-0.196	0.162	2.548	0.524	1.649	1.625	0.000	0.737	0.901	1.124	1.014	0.556	1.039	0.845
+1	1.049	-0.223	0.685	0.256	-1.191	2.506	0.238	-0.359	0.000	1.510	-0.904	1.158	1.107	2.733	-0.902	1.679	2.548	0.407	-0.474	-1.572	0.000	1.513	2.472	0.982	1.238	0.978	1.985	1.510
+0	0.455	-0.028	0.265	1.286	1.373	0.459	0.331	-0.922	0.000	0.343	0.634	0.430	0.000	0.279	-0.084	-0.272	0.000	0.475	0.926	-0.123	3.102	0.803	0.495	0.987	0.587	0.211	0.417	0.445
+1	2.074	0.388	0.878	1.110	1.557	1.077	-0.226	-0.295	2.173	0.865	-0.319	-1.116	2.215	0.707	-0.835	0.722	0.000	0.632	-0.608	-0.728	0.000	0.715	0.802	1.207	1.190	0.960	1.143	0.926
+1	1.390	0.265	1.196	0.919	-1.371	1.858	0.506	0.786	0.000	1.280	-1.367	-0.720	2.215	1.483	-0.441	-0.675	2.548	1.076	0.294	-0.539	0.000	1.126	0.830	1.155	1.551	0.702	1.103	0.933
+1	1.014	-0.079	1.597	1.038	-0.281	1.135	-0.722	-0.177	2.173	0.544	-1.475	-1.501	0.000	1.257	-1.315	1.212	0.000	0.496	-0.060	1.180	1.551	0.815	0.611	1.411	1.110	0.792	0.846	0.853
+0	0.335	1.267	-1.154	2.011	-0.574	0.753	0.618	1.411	0.000	0.474	0.748	0.681	2.215	0.608	-0.446	-0.354	2.548	0.399	1.295	-0.581	0.000	0.911	0.882	0.975	0.832	0.598	0.580	0.678
+1	0.729	-0.189	1.182	0.293	1.310	0.412	0.459	-0.632	0.000	0.869	-1.128	-0.625	2.215	1.173	-0.893	0.478	2.548	0.584	-2.394	-1.727	0.000	2.016	1.272	0.995	1.034	0.905	0.966	1.038
+1	1.225	-1.215	-0.088	0.881	-0.237	0.600	-0.976	1.462	2.173	0.876	0.506	1.583	2.215	0.718	1.228	-0.031	0.000	0.653	-1.292	1.216	0.000	0.838	1.108	0.981	1.805	0.890	1.251	1.197
+1	2.685	-0.444	0.847	0.253	0.183	0.641	-1.541	-0.873	2.173	0.417	2.874	-0.551	0.000	0.706	-1.431	0.764	0.000	1.390	-0.596	-1.397	0.000	0.894	0.829	0.993	0.789	0.654	0.883	0.746
+0	0.638	-0.481	0.683	1.457	-1.024	0.707	-1.338	1.498	0.000	0.980	0.518	0.289	2.215	0.964	-0.531	-0.423	0.000	0.694	-0.654	-1.314	3.102	0.807	1.283	1.335	0.658	0.907	0.797	0.772
+1	1.789	-0.765	-0.732	0.421	-0.020	1.142	-1.353	1.439	2.173	0.725	-1.518	-1.261	0.000	0.812	-2.597	-0.463	0.000	1.203	-0.120	1.001	0.000	0.978	0.673	0.985	1.303	1.400	1.078	0.983
+1	0.784	-1.431	1.724	0.848	0.559	0.615	-1.643	-1.456	0.000	1.339	-0.513	0.040	2.215	0.394	-2.483	1.304	0.000	0.987	0.889	-0.339	0.000	0.732	0.713	0.987	0.973	0.705	0.875	0.759
+1	0.911	1.098	-1.289	0.421	0.823	1.218	-0.503	0.431	0.000	0.775	0.432	-1.680	0.000	0.855	-0.226	-0.460	2.548	0.646	-0.947	-1.243	1.551	2.201	1.349	0.985	0.730	0.451	0.877	0.825
+1	0.959	0.372	-0.269	1.255	0.702	1.151	0.097	0.805	2.173	0.993	1.011	0.767	2.215	1.096	0.185	0.381	0.000	1.001	-0.205	0.059	0.000	0.979	0.997	1.168	0.796	0.771	0.839	0.776
+0	0.283	-1.864	-1.663	0.219	1.624	0.955	-1.213	0.932	2.173	0.889	0.395	-0.268	0.000	0.597	-1.083	-0.921	2.548	0.584	1.325	-1.072	0.000	0.856	0.927	0.996	0.937	0.936	1.095	0.892
+0	2.017	-0.488	-0.466	1.029	-0.870	3.157	0.059	-0.343	2.173	3.881	0.872	1.502	1.107	3.631	1.720	0.963	0.000	0.633	-1.264	-1.734	0.000	4.572	3.339	1.005	1.407	5.590	3.614	3.110
+1	1.088	0.414	-0.841	0.485	0.605	0.860	1.110	-0.568	0.000	1.152	-0.325	1.203	2.215	0.324	1.652	-0.104	0.000	0.510	1.095	-1.728	0.000	0.880	0.722	0.989	0.977	0.711	0.888	0.762
+0	0.409	-1.717	0.712	0.809	-1.301	0.701	-1.529	-1.411	0.000	1.191	-0.582	0.438	2.215	1.147	0.813	-0.571	2.548	1.039	0.543	0.892	0.000	0.636	0.810	0.986	0.861	1.411	0.907	0.756
+1	1.094	1.577	-0.988	0.497	-0.149	0.891	-2.459	1.034	0.000	0.646	0.792	-1.022	0.000	1.573	0.254	-0.053	2.548	1.428	0.190	-1.641	3.102	4.322	2.687	0.985	0.881	1.135	1.907	1.831
+1	0.613	1.993	-0.280	0.544	0.931	0.909	1.526	1.559	0.000	0.840	1.473	-0.483	2.215	0.856	0.352	0.408	2.548	1.058	1.733	-1.396	0.000	0.801	1.066	0.984	0.639	0.841	0.871	0.748
+0	0.958	-1.202	0.600	0.434	0.170	0.783	-0.214	1.319	0.000	0.835	-0.454	-0.615	2.215	0.658	-1.858	-0.891	0.000	0.640	0.172	-1.204	3.102	1.790	1.086	0.997	0.804	0.403	0.793	0.756
+1	1.998	-0.238	0.972	0.058	0.266	0.759	1.576	-0.357	2.173	1.004	-0.349	-0.747	2.215	0.962	0.490	-0.453	0.000	1.592	0.661	-1.405	0.000	0.874	1.086	0.990	1.436	1.527	1.177	0.993
+1	0.796	-0.171	-0.818	0.574	-1.625	1.201	-0.737	1.451	2.173	0.651	0.404	-0.452	0.000	1.150	-0.652	-0.120	0.000	1.008	-0.093	0.531	3.102	0.884	0.706	0.979	1.193	0.937	0.943	0.881
+1	0.773	1.023	0.527	1.537	-0.201	2.967	-0.574	-1.534	2.173	2.346	-0.307	0.394	2.215	1.393	0.135	-0.027	0.000	3.015	0.187	0.516	0.000	0.819	1.260	0.982	2.552	3.862	2.179	1.786
+0	1.823	1.008	-1.489	0.234	-0.962	0.591	0.461	0.996	2.173	0.568	-1.297	-0.410	0.000	0.887	2.157	1.194	0.000	2.079	0.369	-0.085	3.102	0.770	0.945	0.995	1.179	0.971	0.925	0.983
+0	0.780	0.640	0.490	0.680	-1.301	0.715	-0.137	0.152	2.173	0.616	-0.831	1.668	0.000	1.958	0.528	-0.982	2.548	0.966	-1.551	0.462	0.000	1.034	1.079	1.008	0.827	1.369	1.152	0.983
+1	0.543	0.801	1.543	1.134	-0.772	0.954	-0.849	0.410	1.087	0.851	-1.988	1.686	0.000	0.799	-0.912	-1.156	0.000	0.479	0.097	1.334	0.000	0.923	0.597	0.989	1.231	0.759	0.975	0.867
+0	1.241	-0.014	0.129	1.158	0.670	0.445	-0.732	1.739	2.173	0.918	0.659	-1.340	2.215	0.557	2.410	-1.404	0.000	0.966	-1.545	-1.120	0.000	0.874	0.918	0.987	1.001	0.798	0.904	0.937
+0	1.751	-0.266	-1.575	0.489	1.292	1.112	1.533	0.137	2.173	1.204	-0.414	-0.928	0.000	0.879	1.237	-0.415	2.548	1.479	1.469	0.913	0.000	2.884	1.747	0.989	1.742	0.600	1.363	1.293
+1	1.505	1.208	-1.476	0.995	-0.836	2.800	-1.600	0.111	0.000	2.157	1.241	1.110	2.215	1.076	2.619	-0.913	0.000	1.678	2.204	-1.575	0.000	0.849	1.224	0.990	1.412	0.976	1.271	1.105
+0	0.816	0.611	0.779	1.694	0.278	0.575	-0.787	1.592	2.173	1.148	1.076	-0.831	2.215	0.421	1.316	0.632	0.000	0.589	0.452	-1.466	0.000	0.779	0.909	0.990	1.146	1.639	1.236	0.949
+1	0.551	-0.808	0.330	1.188	-0.294	0.447	-0.035	-0.993	0.000	0.432	-0.276	-0.481	2.215	1.959	-0.288	1.195	2.548	0.638	0.583	1.107	0.000	0.832	0.924	0.993	0.723	0.976	0.968	0.895
+0	1.316	-0.093	0.995	0.860	-0.621	0.593	-0.560	-1.599	2.173	0.524	-0.318	-0.240	2.215	0.566	0.759	-0.368	0.000	0.483	-2.030	-1.104	0.000	1.468	1.041	1.464	0.811	0.778	0.690	0.722
+1	1.528	0.067	-0.855	0.959	-1.464	1.143	-0.082	1.023	0.000	0.702	-0.763	-0.244	0.000	0.935	-0.881	0.206	2.548	0.614	-0.831	1.657	3.102	1.680	1.105	0.983	1.078	0.559	0.801	0.809
+0	0.558	-0.833	-0.598	1.436	-1.724	1.316	-0.661	1.593	2.173	1.148	-0.503	-0.132	1.107	1.584	-0.125	0.380	0.000	1.110	-1.216	-0.181	0.000	1.258	0.860	1.053	0.790	1.814	1.159	1.007
+1	0.819	0.879	1.221	0.598	-1.450	0.754	0.417	-0.369	2.173	0.477	1.199	0.274	0.000	1.073	0.368	0.273	2.548	1.599	2.047	1.690	0.000	0.933	0.984	0.983	0.788	0.613	0.728	0.717
+0	0.981	-1.007	0.489	0.923	1.261	0.436	-0.698	-0.506	2.173	0.764	-1.105	-1.241	2.215	0.577	-2.573	-0.036	0.000	0.565	-1.628	1.610	0.000	0.688	0.801	0.991	0.871	0.554	0.691	0.656
+0	2.888	0.568	-1.416	1.461	-1.157	1.756	-0.900	0.522	0.000	0.657	0.409	1.076	2.215	1.419	0.672	-0.019	0.000	1.436	-0.184	-0.980	3.102	0.946	0.919	0.995	1.069	0.890	0.834	0.856
+1	0.522	1.805	-0.963	1.136	0.418	0.727	-0.195	-1.695	2.173	0.309	2.559	-0.178	0.000	0.521	1.794	0.919	0.000	0.788	0.174	-0.406	3.102	0.555	0.729	1.011	1.385	0.753	0.927	0.832
+1	0.793	-0.162	-1.643	0.634	0.337	0.898	-0.633	1.689	0.000	0.806	-0.826	-0.356	2.215	0.890	-0.142	-1.268	0.000	1.293	0.574	0.725	0.000	0.833	1.077	0.988	0.721	0.679	0.867	0.753
+0	1.298	1.098	0.280	0.371	-0.373	0.855	-0.306	-1.186	0.000	0.977	-0.421	1.003	0.000	0.978	0.956	-1.249	2.548	0.735	0.577	-0.037	3.102	0.974	1.002	0.992	0.549	0.587	0.725	0.954
+1	0.751	-0.520	-1.653	0.168	-0.419	0.878	-1.023	-1.364	2.173	1.310	-0.667	0.863	0.000	1.196	-0.827	0.358	0.000	1.154	-0.165	-0.360	1.551	0.871	0.950	0.983	0.907	0.955	0.959	0.874
+0	1.730	0.666	-1.432	0.446	1.302	0.921	-0.203	0.621	0.000	1.171	-0.365	-0.611	1.107	0.585	0.807	1.150	0.000	0.415	-0.843	1.311	0.000	0.968	0.786	0.986	1.059	0.371	0.790	0.848
+1	0.596	-1.486	0.690	1.045	-1.344	0.928	0.867	0.820	2.173	0.610	0.999	-1.329	2.215	0.883	-0.001	-0.106	0.000	1.145	2.184	-0.808	0.000	2.019	1.256	1.056	1.751	1.037	1.298	1.518
+1	0.656	-1.993	-0.519	1.643	-0.143	0.815	0.256	1.220	1.087	0.399	-1.184	-1.458	0.000	0.738	1.361	-1.443	0.000	0.842	0.033	0.293	0.000	0.910	0.891	0.993	0.668	0.562	0.958	0.787
+1	1.127	-0.542	0.645	0.318	-1.496	0.661	-0.640	0.369	2.173	0.992	0.358	1.702	0.000	1.004	0.316	-1.109	0.000	1.616	-0.936	-0.707	1.551	0.875	1.191	0.985	0.651	0.940	0.969	0.834
+0	0.916	-1.423	-1.490	1.248	-0.538	0.625	-0.535	-0.174	0.000	0.769	-0.389	1.608	2.215	0.667	-1.138	-1.738	1.274	0.877	-0.019	0.482	0.000	0.696	0.917	1.121	0.678	0.347	0.647	0.722
+1	2.756	-0.637	-1.715	1.331	1.124	0.913	-0.296	-0.491	0.000	0.983	-0.831	0.000	2.215	1.180	-0.428	0.742	0.000	1.113	0.005	-1.157	1.551	1.681	1.096	1.462	0.976	0.917	1.009	1.040
+0	0.755	1.754	0.701	2.111	0.256	1.243	0.057	-1.502	2.173	0.565	-0.034	-1.078	1.107	0.529	1.696	-1.090	0.000	0.665	0.292	0.107	0.000	0.870	0.780	0.990	2.775	0.465	1.876	1.758
+1	0.593	-0.762	1.743	0.908	0.442	0.773	-1.357	-0.768	2.173	0.432	1.421	1.236	0.000	0.579	0.291	-0.403	0.000	0.966	-0.309	1.016	3.102	0.893	0.743	0.989	0.857	1.030	0.943	0.854
+1	0.891	-1.151	-1.269	0.504	-0.622	0.893	-0.549	0.700	0.000	0.828	-0.825	0.154	2.215	1.083	0.632	-1.141	0.000	1.059	-0.557	1.526	3.102	2.117	1.281	0.987	0.819	0.802	0.917	0.828
+1	2.358	-0.248	0.080	0.747	-0.975	1.019	1.374	1.363	0.000	0.935	0.127	-1.707	2.215	0.312	-0.827	0.017	0.000	0.737	1.059	-0.327	0.000	0.716	0.828	1.495	0.953	0.704	0.880	0.745
+0	0.660	-0.017	-1.138	0.453	1.002	0.645	0.518	0.703	2.173	0.751	0.705	-0.592	2.215	0.744	-0.909	-1.596	0.000	0.410	-1.135	0.481	0.000	0.592	0.922	0.989	0.897	0.948	0.777	0.701
+1	0.718	0.518	0.225	1.710	-0.022	1.888	-0.424	1.092	0.000	4.134	0.185	-1.366	0.000	1.415	1.293	0.242	2.548	2.351	0.264	-0.057	3.102	0.830	1.630	0.976	1.215	0.890	1.422	1.215
+1	1.160	0.203	0.941	0.594	0.212	0.636	-0.556	0.679	2.173	1.089	-0.481	-1.008	1.107	1.245	-0.056	-1.357	0.000	0.587	1.007	0.056	0.000	1.106	0.901	0.987	0.786	1.224	0.914	0.837
+1	0.697	0.542	0.619	0.985	1.481	0.745	0.415	1.644	2.173	0.903	0.495	-0.958	2.215	1.165	1.195	0.346	0.000	1.067	-0.881	-0.264	0.000	0.830	1.025	0.987	0.690	0.863	0.894	0.867
+0	1.430	0.190	-0.700	0.246	0.518	1.302	0.660	-0.247	2.173	1.185	-0.539	1.504	0.000	1.976	-0.401	1.079	0.000	0.855	-0.958	-1.110	3.102	0.886	0.953	0.993	0.889	1.400	1.376	1.119
+1	1.122	-0.795	0.202	0.397	-1.553	0.597	-1.459	-0.734	2.173	0.522	1.044	1.027	2.215	0.783	-1.243	1.701	0.000	0.371	1.737	0.199	0.000	1.719	1.176	0.988	0.723	1.583	1.063	0.914
+0	1.153	0.526	1.236	0.266	0.001	1.139	-1.236	-0.585	2.173	1.337	-0.215	-1.356	2.215	1.780	1.129	0.902	0.000	1.608	-0.391	-0.161	0.000	1.441	1.633	0.990	1.838	1.516	1.635	1.373
+1	0.760	1.012	0.758	0.937	0.051	0.941	0.687	-1.247	2.173	1.288	-0.743	0.822	0.000	1.552	1.782	-1.533	0.000	0.767	1.349	0.168	0.000	0.716	0.862	0.988	0.595	0.359	0.697	0.623
+1	1.756	-1.469	1.395	1.345	-1.595	0.817	0.017	-0.741	2.173	0.483	-0.008	0.293	0.000	1.768	-0.663	0.438	1.274	1.202	-1.387	-0.222	0.000	1.022	1.058	0.992	1.407	1.427	1.356	1.133
+0	0.397	0.582	-0.758	1.260	-1.735	0.889	-0.515	1.139	2.173	0.973	1.616	0.460	0.000	1.308	1.001	-0.709	2.548	0.858	0.995	-0.231	0.000	0.749	0.888	0.979	1.487	1.804	1.208	1.079
+0	0.515	-0.984	0.425	1.114	-0.439	1.999	0.818	1.561	0.000	1.407	0.009	-0.380	0.000	1.332	0.230	0.397	0.000	1.356	-0.616	-1.057	3.102	0.978	1.017	0.990	1.118	0.862	0.835	0.919
+1	1.368	-0.921	-0.866	0.842	-0.598	0.456	-1.176	1.219	1.087	0.419	-1.974	-0.819	0.000	0.791	-1.640	0.881	0.000	1.295	-0.782	0.442	3.102	0.945	0.761	0.974	0.915	0.535	0.733	0.651
+0	2.276	0.134	0.399	2.525	0.376	1.111	-1.078	-1.571	0.000	0.657	2.215	-0.900	0.000	1.183	-0.662	-0.508	2.548	1.436	-0.517	0.960	3.102	0.569	0.931	0.993	1.170	0.967	0.879	1.207
+0	0.849	0.907	0.124	0.652	1.585	0.715	0.355	-1.200	0.000	0.599	-0.892	1.301	0.000	1.106	1.151	0.582	0.000	1.895	-0.279	-0.568	3.102	0.881	0.945	0.998	0.559	0.649	0.638	0.660
+1	2.105	0.248	-0.797	0.530	0.206	1.957	-2.175	0.797	0.000	1.193	0.637	-1.646	2.215	0.881	1.111	-1.046	0.000	0.872	-0.185	1.085	1.551	0.986	1.343	1.151	1.069	0.714	2.063	1.951
+1	1.838	1.060	1.637	1.017	1.370	0.913	0.461	-0.609	1.087	0.766	-0.461	0.303	2.215	0.724	-0.061	0.886	0.000	0.941	1.123	-0.745	0.000	0.858	0.847	0.979	1.313	1.083	1.094	0.910
+0	0.364	1.274	1.066	1.570	-0.394	0.485	0.012	-1.716	0.000	0.317	-1.233	0.534	2.215	0.548	-2.165	0.762	0.000	0.729	0.169	-0.318	3.102	0.892	0.944	1.013	0.594	0.461	0.688	0.715
+1	0.503	1.343	-0.031	1.134	-1.204	0.590	-0.309	0.174	2.173	0.408	2.372	-0.628	0.000	1.850	0.400	1.147	2.548	0.664	-0.458	-0.885	0.000	1.445	1.283	0.989	1.280	1.118	1.127	1.026
+0	1.873	0.258	0.103	2.491	0.530	1.678	0.644	-1.738	2.173	1.432	0.848	-1.340	0.000	0.621	1.323	-1.316	0.000	0.628	0.789	-0.206	1.551	0.426	0.802	1.125	0.688	1.079	1.338	1.239
+1	0.826	-0.732	1.587	0.582	-1.236	0.495	0.757	-0.741	2.173	0.940	1.474	0.354	2.215	0.474	1.055	-1.657	0.000	0.415	1.758	0.841	0.000	0.451	0.578	0.984	0.757	0.922	0.860	0.696
+0	0.935	-1.614	-0.597	0.299	1.223	0.707	-0.853	-1.026	0.000	0.751	0.007	-1.691	0.000	1.062	-0.125	0.976	2.548	0.877	1.275	0.646	0.000	0.962	1.074	0.980	0.608	0.726	0.741	0.662
+1	0.643	0.542	-1.285	0.474	-0.366	0.667	-0.446	1.195	2.173	1.076	0.145	-0.126	0.000	0.970	-0.661	0.394	1.274	1.218	-0.184	-1.722	0.000	1.331	1.019	0.985	1.192	0.677	0.973	0.910
+0	0.713	0.164	1.080	1.427	-0.460	0.960	-0.152	-0.940	2.173	1.427	-0.901	1.036	1.107	0.440	-1.269	-0.194	0.000	0.452	1.932	-0.532	0.000	1.542	1.210	1.374	1.319	1.818	1.220	1.050
+0	0.876	-0.463	-1.224	2.458	-1.689	1.007	-0.752	0.398	0.000	2.456	-1.285	-0.152	1.107	1.641	1.838	1.717	0.000	0.458	0.194	0.488	3.102	4.848	2.463	0.986	1.981	0.974	2.642	2.258
+1	0.384	-0.275	0.387	1.403	-0.994	0.620	-1.529	1.685	0.000	1.091	-1.644	1.078	0.000	0.781	-1.311	0.326	2.548	1.228	-0.728	-0.633	1.551	0.920	0.854	0.987	0.646	0.609	0.740	0.884
+0	0.318	-1.818	-1.008	0.977	1.268	0.457	2.451	-1.522	0.000	0.881	1.351	0.461	2.215	0.929	0.239	-0.380	2.548	0.382	-0.613	1.330	0.000	1.563	1.193	0.994	0.829	0.874	0.901	1.026
+1	0.612	-1.120	1.098	0.402	-0.480	0.818	0.188	1.511	0.000	0.800	-0.253	0.977	0.000	1.175	0.271	-1.289	1.274	2.531	0.226	-0.409	3.102	0.889	0.947	0.979	1.486	0.940	1.152	1.119
+1	0.587	-0.737	-0.228	0.970	1.119	0.823	0.184	1.594	0.000	1.104	0.301	-0.818	2.215	0.819	0.712	-0.560	0.000	2.240	-0.419	0.340	3.102	1.445	1.103	0.988	0.715	1.363	1.019	0.926
+0	1.030	-0.694	-1.638	0.893	-1.074	1.160	-0.766	0.485	0.000	1.632	-0.698	-1.142	2.215	1.050	-1.092	0.952	0.000	1.475	0.286	0.125	3.102	0.914	1.075	0.982	0.732	1.493	1.219	1.079
+1	2.142	0.617	1.517	0.387	-0.862	0.345	1.203	-1.014	2.173	0.609	1.092	0.275	0.000	1.331	0.582	-0.183	2.548	0.557	1.540	-1.642	0.000	0.801	0.737	1.060	0.715	0.626	0.749	0.674
+0	1.076	0.240	-0.246	0.871	-1.241	0.496	0.282	0.746	2.173	1.095	-0.648	1.100	2.215	0.446	-1.756	0.764	0.000	0.434	0.788	-0.991	0.000	1.079	0.868	1.047	0.818	0.634	0.795	0.733
+0	1.400	0.901	-1.617	0.625	-0.163	0.661	-0.411	-1.616	2.173	0.685	0.524	0.425	0.000	0.881	-0.766	0.312	0.000	0.979	0.255	-0.667	3.102	0.898	1.105	1.253	0.730	0.716	0.738	0.795
+0	3.302	1.132	1.051	0.658	0.768	1.308	0.251	-0.374	1.087	1.673	0.015	-0.898	0.000	0.688	-0.535	1.363	1.274	0.871	1.325	-1.583	0.000	1.646	1.249	0.995	1.919	1.288	1.330	1.329
+0	1.757	0.202	0.750	0.767	-0.362	0.932	-1.033	-1.366	0.000	1.529	-1.012	-0.771	0.000	1.161	-0.287	0.059	0.000	2.185	1.147	1.099	3.102	0.795	0.529	1.354	1.144	1.491	1.319	1.161
+0	1.290	0.905	-1.711	1.017	-0.695	1.008	-1.038	0.693	2.173	1.202	-0.595	0.187	0.000	1.011	0.139	-1.607	0.000	0.789	-0.613	-1.041	3.102	1.304	0.895	1.259	1.866	0.955	1.211	1.200
+1	1.125	-0.004	1.694	0.373	0.329	0.978	0.640	-0.391	0.000	1.122	-0.376	1.521	2.215	0.432	2.413	-1.259	0.000	0.969	0.730	0.512	3.102	0.716	0.773	0.991	0.624	0.977	0.981	0.875
+0	1.081	0.861	1.252	1.621	1.474	1.293	0.600	0.630	0.000	1.991	-0.090	-0.675	2.215	0.861	1.105	-0.201	0.000	1.135	2.489	-1.659	0.000	1.089	0.657	0.991	2.179	0.412	1.334	1.071
+1	0.652	-0.294	1.241	1.034	0.490	1.033	0.551	-0.963	2.173	0.661	1.031	-1.654	2.215	1.376	-0.018	0.843	0.000	0.943	-0.329	-0.269	0.000	1.085	1.067	0.991	1.504	0.773	1.135	0.993
+1	1.408	-1.028	-1.018	0.252	-0.242	0.465	-0.364	-0.200	0.000	1.466	0.669	0.739	1.107	1.031	0.415	-1.468	2.548	0.457	-1.091	-1.722	0.000	0.771	0.811	0.979	1.459	1.204	1.041	0.866
+1	0.781	-1.143	-0.659	0.961	1.266	1.183	-0.686	0.119	2.173	1.126	-0.064	1.447	0.000	0.730	1.430	-1.535	0.000	1.601	0.513	1.658	0.000	0.871	1.345	1.184	1.058	0.620	1.107	0.978
+1	1.300	-0.616	1.032	0.751	-0.731	0.961	-0.716	1.592	0.000	2.079	-1.063	-0.271	2.215	0.475	0.518	1.695	1.274	0.395	-2.204	0.349	0.000	1.350	0.983	1.369	1.265	1.428	1.135	0.982
+1	0.833	0.809	1.657	1.637	1.019	0.705	1.077	-0.968	2.173	1.261	0.114	-0.298	1.107	1.032	0.017	0.236	0.000	0.640	-0.026	-1.598	0.000	0.894	0.982	0.981	1.250	1.054	1.018	0.853
+1	1.686	-1.090	-0.301	0.890	0.557	1.304	-0.284	-1.393	2.173	0.388	2.118	0.513	0.000	0.514	-0.015	0.891	0.000	0.460	0.547	0.627	3.102	0.942	0.524	1.186	1.528	0.889	1.015	1.122
+1	0.551	0.911	0.879	0.379	-0.796	1.154	-0.808	-0.966	0.000	1.168	-0.513	0.355	2.215	0.646	-1.309	0.773	0.000	0.544	-0.283	1.301	3.102	0.847	0.705	0.990	0.772	0.546	0.790	0.719
+1	1.597	0.793	-1.119	0.691	-1.455	0.370	0.337	1.354	0.000	0.646	-1.005	0.732	2.215	1.019	0.040	0.209	0.000	0.545	0.958	0.239	3.102	0.962	0.793	0.994	0.719	0.745	0.812	0.739
+0	1.033	-1.193	-0.452	0.247	0.970	0.503	-1.424	1.362	0.000	1.062	-0.416	-1.156	2.215	0.935	-0.023	0.555	2.548	0.410	-1.766	0.379	0.000	0.590	0.953	0.991	0.717	1.081	0.763	0.690
+1	0.859	-1.004	1.521	0.781	-0.993	0.677	0.643	-0.338	2.173	0.486	0.409	1.283	0.000	0.679	0.110	0.285	0.000	0.715	-0.735	-0.157	1.551	0.702	0.773	0.984	0.627	0.633	0.694	0.643
+0	0.612	-1.127	1.074	1.225	-0.426	0.927	-2.141	-0.473	0.000	1.290	-0.927	-1.085	2.215	1.183	1.981	-1.687	0.000	2.176	0.406	-1.581	0.000	0.945	0.651	1.170	0.895	1.604	1.179	1.142
+1	0.535	0.321	-1.095	0.281	-0.960	0.876	-0.709	-0.076	0.000	1.563	-0.666	1.536	2.215	0.773	-0.321	0.435	0.000	0.682	-0.801	-0.952	3.102	0.711	0.667	0.985	0.888	0.741	0.872	0.758
+1	0.745	1.586	1.578	0.863	-1.423	0.530	1.714	1.085	0.000	1.174	0.679	1.015	0.000	1.158	0.609	-1.186	2.548	1.851	0.832	-0.248	3.102	0.910	1.164	0.983	0.947	0.858	0.928	0.823
+0	0.677	-1.014	-1.648	1.455	1.461	0.596	-2.358	0.517	0.000	0.800	0.849	-0.743	2.215	1.024	-0.282	-1.004	0.000	1.846	-0.977	0.378	3.102	2.210	1.423	0.982	1.074	1.623	1.417	1.258
+1	0.815	-1.263	0.057	1.018	-0.208	0.339	-0.347	-1.646	2.173	1.223	0.600	-1.658	2.215	1.435	0.042	0.926	0.000	0.777	1.698	-0.698	0.000	1.022	1.058	1.000	0.784	0.477	0.886	0.836
+0	3.512	-1.094	-0.220	0.338	-0.328	1.962	-1.099	1.544	1.087	1.461	-1.305	-0.922	2.215	1.219	-1.289	0.400	0.000	0.731	0.155	1.249	0.000	1.173	1.366	0.993	2.259	2.000	1.626	1.349
+0	0.904	1.248	0.325	0.317	-1.624	0.685	-0.538	1.665	2.173	0.685	-2.145	-1.106	0.000	0.632	-1.460	1.017	0.000	1.085	-0.182	0.162	3.102	0.885	0.801	0.989	0.930	0.904	1.012	0.961

contrib/gbdt/lightgbm/lightgbm-example.ipynb

@@ -0,0 +1,270 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Copyright (c) Microsoft Corporation. All rights reserved.  \n",
+        "Licensed under the MIT License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/contrib/gbdt/lightgbm/lightgbm-example.png)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# Use LightGBM Estimator in Azure Machine Learning\n",
+        "In this notebook we will demonstrate how to run a training job using LightGBM Estimator. [LightGBM](https://lightgbm.readthedocs.io/en/latest/) is a gradient boosting framework that uses tree based learning algorithms. "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Prerequisites\n",
+        "This notebook uses azureml-contrib-gbdt package, if you don't already have the package, please install by uncommenting below cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "#!pip install azureml-contrib-gbdt"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core import Workspace, Run, Experiment\n",
+        "import shutil, os\n",
+        "from azureml.widgets import RunDetails\n",
+        "from azureml.contrib.gbdt import LightGBM\n",
+        "from azureml.train.dnn import Mpi\n",
+        "from azureml.core.compute import AmlCompute, ComputeTarget\n",
+        "from azureml.core.compute_target import ComputeTargetException"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "If you are using an AzureML Compute Instance, you are all set. Otherwise, go through the [configuration.ipynb](../../../configuration.ipynb) notebook to install the Azure Machine Learning Python SDK and create an Azure ML Workspace"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Set up machine learning resources"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "ws = Workspace.from_config()\n",
+        "\n",
+        "print('Workspace name: ' + ws.name, \n",
+        "      'Azure region: ' + ws.location, \n",
+        "      'Subscription id: ' + ws.subscription_id, \n",
+        "      'Resource group: ' + ws.resource_group, sep = '\\n')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "cluster_vm_size = \"STANDARD_DS14_V2\"\n",
+        "cluster_min_nodes = 0\n",
+        "cluster_max_nodes = 20\n",
+        "cpu_cluster_name = 'TrainingCompute2' \n",
+        "\n",
+        "try:\n",
+        "    cpu_cluster = AmlCompute(ws, cpu_cluster_name)\n",
+        "    if cpu_cluster and type(cpu_cluster) is AmlCompute:\n",
+        "        print('found compute target: ' + cpu_cluster_name)\n",
+        "except ComputeTargetException:\n",
+        "    print('creating a new compute target...')\n",
+        "    provisioning_config = AmlCompute.provisioning_configuration(vm_size = cluster_vm_size, \n",
+        "                                                                vm_priority = 'lowpriority', \n",
+        "                                                                min_nodes = cluster_min_nodes, \n",
+        "                                                                max_nodes = cluster_max_nodes)\n",
+        "    cpu_cluster = ComputeTarget.create(ws, cpu_cluster_name, provisioning_config)\n",
+        "    \n",
+        "    # can poll for a minimum number of nodes and for a specific timeout. \n",
+        "    # if no min node count is provided it will use the scale settings for the cluster\n",
+        "    cpu_cluster.wait_for_completion(show_output=True, min_node_count=None, timeout_in_minutes=20)\n",
+        "    \n",
+        "     # For a more detailed view of current Azure Machine Learning Compute  status, use get_status()\n",
+        "    print(cpu_cluster.get_status().serialize())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "From this point, you can either upload training data file directly or use Datastore for training data storage\n",
+        "## Upload training file from local"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "scripts_folder = \"scripts_folder\"\n",
+        "if not os.path.isdir(scripts_folder):\n",
+        "    os.mkdir(scripts_folder)\n",
+        "shutil.copy('./train.conf', os.path.join(scripts_folder, 'train.conf'))\n",
+        "shutil.copy('./binary0.train', os.path.join(scripts_folder, 'binary0.train'))\n",
+        "shutil.copy('./binary1.train', os.path.join(scripts_folder, 'binary1.train'))\n",
+        "shutil.copy('./binary0.test', os.path.join(scripts_folder, 'binary0.test'))\n",
+        "shutil.copy('./binary1.test', os.path.join(scripts_folder, 'binary1.test'))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "training_data_list=[\"binary0.train\", \"binary1.train\"]\n",
+        "validation_data_list = [\"binary0.test\", \"binary1.test\"]\n",
+        "lgbm = LightGBM(source_directory=scripts_folder, \n",
+        "                compute_target=cpu_cluster, \n",
+        "                distributed_training=Mpi(),\n",
+        "                node_count=2,\n",
+        "                lightgbm_config='train.conf',\n",
+        "                data=training_data_list,\n",
+        "                valid=validation_data_list\n",
+        "               )\n",
+        "experiment_name = 'lightgbm-estimator-test'\n",
+        "experiment = Experiment(ws, name=experiment_name)\n",
+        "run = experiment.submit(lgbm, tags={\"test public docker image\": None})\n",
+        "RunDetails(run).show()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "run.wait_for_completion(show_output=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Use data reference"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.datastore import Datastore\n",
+        "from azureml.data.data_reference import DataReference\n",
+        "datastore = ws.get_default_datastore()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "datastore.upload(src_dir='.',\n",
+        "                 target_path='.',\n",
+        "                 show_progress=True)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "training_data_list=[\"binary0.train\", \"binary1.train\"]\n",
+        "validation_data_list = [\"binary0.test\", \"binary1.test\"]\n",
+        "lgbm = LightGBM(source_directory='.', \n",
+        "                compute_target=cpu_cluster, \n",
+        "                distributed_training=Mpi(),\n",
+        "                node_count=2,\n",
+        "                inputs=[datastore.as_mount()],\n",
+        "                lightgbm_config='train.conf',\n",
+        "                data=training_data_list,\n",
+        "                valid=validation_data_list\n",
+        "               )\n",
+        "experiment_name = 'lightgbm-estimator-test'\n",
+        "experiment = Experiment(ws, name=experiment_name)\n",
+        "run = experiment.submit(lgbm, tags={\"use datastore.as_mount()\": None})\n",
+        "RunDetails(run).show()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "run.wait_for_completion(show_output=True)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# uncomment below and run if compute resources are no longer needed\n",
+        "# cpu_cluster.delete() "
+      ]
+    }
+  ],
+  "metadata": {
+    "authors": [
+      {
+        "name": "jingywa"
+      }
+    ],
+    "kernelspec": {
+      "display_name": "Python 3.6",
+      "language": "python",
+      "name": "python36"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.6.9"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 2
+}

contrib/gbdt/lightgbm/lightgbm-example.yml

@@ -0,0 +1,7 @@
+name: lightgbm-example
+dependencies:
+- pip:
+  - azureml-sdk
+  - azureml-contrib-gbdt
+  - azureml-widgets
+  - azureml-core

contrib/gbdt/lightgbm/train.conf

@@ -0,0 +1,111 @@
+# task type, support train and predict
+task = train
+
+# boosting type, support gbdt for now, alias: boosting, boost
+boosting_type = gbdt
+
+# application type, support following application
+# regression , regression task
+# binary , binary classification task
+# lambdarank , lambdarank task
+# alias: application, app
+objective = binary
+
+# eval metrics, support multi metric, delimite by ',' , support following metrics
+# l1 
+# l2 , default metric for regression
+# ndcg , default metric for lambdarank
+# auc 
+# binary_logloss , default metric for binary
+# binary_error
+metric = binary_logloss,auc
+
+# frequence for metric output
+metric_freq = 1
+
+# true if need output metric for training data, alias: tranining_metric, train_metric
+is_training_metric = true
+
+# number of bins for feature bucket, 255 is a recommend setting, it can save memories, and also has good accuracy. 
+max_bin = 255
+
+# training data
+# if exsting weight file, should name to "binary.train.weight"
+# alias: train_data, train
+data = binary.train
+
+# validation data, support multi validation data, separated by ','
+# if exsting weight file, should name to "binary.test.weight"
+# alias: valid, test, test_data, 
+valid_data = binary.test
+
+# number of trees(iterations), alias: num_tree, num_iteration, num_iterations, num_round, num_rounds
+num_trees = 100
+
+# shrinkage rate , alias: shrinkage_rate
+learning_rate = 0.1
+
+# number of leaves for one tree, alias: num_leaf
+num_leaves = 63
+
+# type of tree learner, support following types:
+# serial , single machine version
+# feature , use feature parallel to train
+# data , use data parallel to train
+# voting , use voting based parallel to train
+# alias: tree
+tree_learner = feature
+
+# number of threads for multi-threading. One thread will use one CPU, defalut is setted to #cpu. 
+# num_threads = 8
+
+# feature sub-sample, will random select 80% feature to train on each iteration 
+# alias: sub_feature
+feature_fraction = 0.8
+
+# Support bagging (data sub-sample), will perform bagging every 5 iterations
+bagging_freq = 5
+
+# Bagging farction, will random select 80% data on bagging
+# alias: sub_row
+bagging_fraction = 0.8
+
+# minimal number data for one leaf, use this to deal with over-fit
+# alias : min_data_per_leaf, min_data
+min_data_in_leaf = 50
+
+# minimal sum hessians for one leaf, use this to deal with over-fit
+min_sum_hessian_in_leaf = 5.0
+
+# save memory and faster speed for sparse feature, alias: is_sparse
+is_enable_sparse = true
+
+# when data is bigger than memory size, set this to true. otherwise set false will have faster speed
+# alias: two_round_loading, two_round
+use_two_round_loading = false
+
+# true if need to save data to binary file and application will auto load data from binary file next time
+# alias: is_save_binary, save_binary
+is_save_binary_file = false
+
+# output model file
+output_model = LightGBM_model.txt
+
+# support continuous train from trained gbdt model
+# input_model= trained_model.txt
+
+# output prediction file for predict task
+# output_result= prediction.txt
+
+# support continuous train from initial score file
+# input_init_score= init_score.txt
+
+
+# number of machines in parallel training, alias: num_machine
+num_machines = 2
+
+# local listening port in parallel training, alias: local_port
+local_listen_port = 12400
+
+# machines list file for parallel training, alias: mlist
+machine_list_file = mlist.txt

how-to-use-azureml/README.md

@@ -8,10 +8,9 @@ As a pre-requisite, run the [configuration Notebook](../configuration.ipynb) not
 * [train-on-local](./training/train-on-local): Learn how to submit a run to local computer and use Azure ML managed run configuration.
 * [train-on-amlcompute](./training/train-on-amlcompute): Use a 1-n node Azure ML managed compute cluster for remote runs on Azure CPU or GPU infrastructure.
 * [train-on-remote-vm](./training/train-on-remote-vm): Use Data Science Virtual Machine as a target for remote runs.
-* [logging-api](./training/logging-api): Learn about the details of logging metrics to run history.
+* [logging-api](./track-and-monitor-experiments/logging-api): Learn about the details of logging metrics to run history.
 * [register-model-create-image-deploy-service](./deployment/register-model-create-image-deploy-service): Learn about the details of model management.
 * [production-deploy-to-aks](./deployment/production-deploy-to-aks) Deploy a model to production at scale on Azure Kubernetes Service.
-* [enable-data-collection-for-models-in-aks](./deployment/enable-data-collection-for-models-in-aks) Learn about data collection APIs for deployed model.
 * [enable-app-insights-in-production-service](./deployment/enable-app-insights-in-production-service) Learn how to use App Insights with production web service.
  
 Find quickstarts, end-to-end tutorials, and how-tos on the [official documentation site for Azure Machine Learning service](https://docs.microsoft.com/en-us/azure/machine-learning/service/).

how-to-use-azureml/automated-machine-learning/README.md

@@ -1,8 +1,8 @@
 # Table of Contents
 1. [Automated ML Introduction](#introduction)
-1. [Running samples in Azure Notebooks](#jupyter)
-1. [Running samples in Azure Databricks](#databricks)
-1. [Running samples in a Local Conda environment](#localconda)
+1. [Setup using Azure Notebooks](#jupyter)
+1. [Setup using Azure Databricks](#databricks)
+1. [Setup using a Local Conda environment](#localconda)
 1. [Automated ML SDK Sample Notebooks](#samples)
 1. [Documentation](#documentation)
 1. [Running using python command](#pythoncommand)
@@ -13,33 +13,25 @@
 Automated machine learning (automated ML) builds high quality machine learning models for you by automating model and hyperparameter selection. Bring a labelled dataset that you want to build a model for, automated ML will give you a high quality machine learning model that you can use for predictions.
 
 
-If you are new to Data Science, AutoML will help you get jumpstarted by simplifying machine learning model building. It abstracts you from needing to perform model selection, hyperparameter selection and in one step creates a high quality trained model for you to use.
+If you are new to Data Science, automated ML will help you get jumpstarted by simplifying machine learning model building. It abstracts you from needing to perform model selection, hyperparameter selection and in one step creates a high quality trained model for you to use.
 
-If you are an experienced data scientist, AutoML will help increase your productivity by intelligently performing the model and hyperparameter selection for your training and generates high quality models much quicker than manually specifying several combinations of the parameters and running training jobs. AutoML provides visibility and access to all the training jobs and the performance characteristics of the models to help you further tune the pipeline if you desire.
+If you are an experienced data scientist, automated ML will help increase your productivity by intelligently performing the model and hyperparameter selection for your training and generates high quality models much quicker than manually specifying several combinations of the parameters and running training jobs. Automated ML provides visibility and access to all the training jobs and the performance characteristics of the models to help you further tune the pipeline if you desire.
 
-Below are the three execution environments supported by AutoML.
+Below are the three execution environments supported by automated ML.
 
 
  <a name="jupyter"></a>
-## Running samples in Azure Notebooks - Jupyter based notebooks in the Azure cloud
+## Setup using Notebook VMs - Jupyter based notebooks from a Azure VM
 
-1. [![Azure Notebooks](https://notebooks.azure.com/launch.png)](https://aka.ms/aml-clone-azure-notebooks)
-[Import sample notebooks ](https://aka.ms/aml-clone-azure-notebooks) into Azure Notebooks.
-1. Follow the instructions in the [configuration](../../configuration.ipynb) notebook to create and connect to a workspace.
-1. Open one of the sample notebooks.
-
- <a name="databricks"></a>
-## Running samples in Azure Databricks
-
-**NOTE**: Please create your Azure Databricks cluster as v4.x (high concurrency preferred) with **Python 3** (dropdown).
-**NOTE**: You should at least have contributor access to your Azure subcription to run the notebook.
-- Please remove the previous SDK version if there is any and install the latest SDK by installing **azureml-sdk[automl_databricks]** as a PyPi library in Azure Databricks workspace.
-- You can find the detail Readme instructions at [GitHub](https://github.com/Azure/MachineLearningNotebooks/tree/master/how-to-use-azureml/azure-databricks).
-- Download the sample notebook automl-databricks-local-01.ipynb from [GitHub](https://github.com/Azure/MachineLearningNotebooks/tree/master/how-to-use-azureml/azure-databricks) and import into the Azure databricks workspace.
-- Attach the notebook to the cluster.
+1. Open the [ML Azure portal](https://ml.azure.com)
+1. Select Compute
+1. Select Notebook VMs
+1. Click New
+1. Type a name for the Vm and select a VM type
+1. Click Create
 
 <a name="localconda"></a>
-## Running samples in a Local Conda environment
+## Setup using a Local Conda environment
 
 To run these notebook on your own notebook server, use these installation instructions.
 The instructions below will install everything you need and then start a Jupyter notebook.
@@ -49,11 +41,15 @@ The instructions below will install everything you need and then start a Jupyter
 There's no need to install mini-conda specifically.
 
 ### 2. Downloading the sample notebooks
-- Download the sample notebooks from [GitHub](https://github.com/Azure/MachineLearningNotebooks) as zip and extract the contents to a local directory.  The AutoML sample notebooks are in the "automl" folder.
+- Download the sample notebooks from [GitHub](https://github.com/Azure/MachineLearningNotebooks) as zip and extract the contents to a local directory.  The automated ML sample notebooks are in the "automated-machine-learning" folder.
 
 ### 3. Setup a new conda environment
-The **automl/automl_setup** script creates a new conda environment, installs the necessary packages, configures the widget and starts a jupyter notebook.
-It takes the conda environment name as an optional parameter.  The default conda environment name is azure_automl.  The exact command depends on the operating system.  See the specific sections below for Windows, Mac and Linux.  It can take about 10 minutes to execute.
+The **automl_setup** script creates a new conda environment, installs the necessary packages, configures the widget and starts a jupyter notebook. It takes the conda environment name as an optional parameter.  The default conda environment name is azure_automl.  The exact command depends on the operating system.  See the specific sections below for Windows, Mac and Linux.  It can take about 10 minutes to execute.
+
+Packages installed by the **automl_setup** script:
+    <ul><li>python</li><li>nb_conda</li><li>matplotlib</li><li>numpy</li><li>cython</li><li>urllib3</li><li>scipy</li><li>scikit-learn</li><li>pandas</li><li>tensorflow</li><li>py-xgboost</li><li>azureml-sdk</li><li>azureml-widgets</li><li>pandas-ml</li></ul>
+
+For more details refer to the [automl_env.yml](./automl_env.yml)
 ## Windows
 Start an **Anaconda Prompt** window, cd to the **how-to-use-azureml/automated-machine-learning** folder where the sample notebooks were extracted and then run:
 ```
@@ -81,7 +77,7 @@ bash automl_setup_linux.sh
 
 ### 5. Running Samples
 - Please make sure you use the Python [conda env:azure_automl] kernel when trying the sample Notebooks.
-- Follow the instructions in the individual notebooks to explore various features in AutoML
+- Follow the instructions in the individual notebooks to explore various features in automated ML.
 
 ### 6. Starting jupyter notebook manually
 To start your Jupyter notebook manually, use:
@@ -98,101 +94,71 @@ source activate azure_automl
 jupyter notebook
 ```
 
+ <a name="databricks"></a>
+## Setup using Azure Databricks
+
+**NOTE**: Please create your Azure Databricks cluster as v6.0 (high concurrency preferred) with **Python 3** (dropdown).
+**NOTE**: You should at least have contributor access to your Azure subcription to run the notebook.
+- Please remove the previous SDK version if there is any and install the latest SDK by installing **azureml-sdk[automl]** as a PyPi library in Azure Databricks workspace.
+- You can find the detail Readme instructions at [GitHub](https://github.com/Azure/MachineLearningNotebooks/tree/master/how-to-use-azureml/azure-databricks).
+- Download the sample notebook automl-databricks-local-01.ipynb from [GitHub](https://github.com/Azure/MachineLearningNotebooks/tree/master/how-to-use-azureml/azure-databricks) and import into the Azure databricks workspace.
+- Attach the notebook to the cluster.
+
 <a name="samples"></a>
 # Automated ML SDK Sample Notebooks
 
-- [auto-ml-classification.ipynb](classification/auto-ml-classification.ipynb)
-    - Dataset: scikit learn's [digit dataset](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_digits.html#sklearn.datasets.load_digits)
-    - Simple example of using Auto ML for classification
-    - Uses local compute for training
+- [auto-ml-classification-credit-card-fraud.ipynb](classification-credit-card-fraud/auto-ml-classification-credit-card-fraud.ipynb)
+    - Dataset: Kaggle's [credit card fraud detection dataset](https://www.kaggle.com/mlg-ulb/creditcardfraud)
+    - Simple example of using automated ML for classification to fraudulent credit card transactions
+    - Uses azure compute for training
 
 - [auto-ml-regression.ipynb](regression/auto-ml-regression.ipynb)
-    - Dataset: scikit learn's [diabetes dataset](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_diabetes.html)
-    - Simple example of using Auto ML for regression
-    - Uses local compute for training
+    - Dataset: Hardware Performance Dataset
+    - Simple example of using automated ML for regression
+    - Uses azure compute for training
 
-- [auto-ml-remote-execution.ipynb](remote-execution/auto-ml-remote-execution.ipynb)
-    - Dataset: scikit learn's [digit dataset](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_digits.html#sklearn.datasets.load_digits)
-    - Example of using Auto ML for classification using a remote linux DSVM for training
-    - Parallel execution of iterations
-    - Async tracking of progress
-    - Cancelling individual iterations or entire run
-    - Retrieving models for any iteration or logged metric
-    - Specify automl settings as kwargs
-
-- [auto-ml-remote-amlcompute.ipynb](remote-batchai/auto-ml-remote-amlcompute.ipynb)
-    - Dataset: scikit learn's [digit dataset](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_digits.html#sklearn.datasets.load_digits)
-    - Example of using automated ML for classification using remote AmlCompute for training
-    - Parallel execution of iterations
-    - Async tracking of progress
-    - Cancelling individual iterations or entire run
-    - Retrieving models for any iteration or logged metric
-    - Specify automl settings as kwargs
-
-- [auto-ml-remote-attach.ipynb](remote-attach/auto-ml-remote-attach.ipynb)
-    - Dataset: Scikit learn's [20newsgroup](http://scikit-learn.org/stable/datasets/twenty_newsgroups.html)
-    - handling text data with preprocess flag
-    - Reading data from a blob store for remote executions
-    - using pandas dataframes for reading data
-
-- [auto-ml-missing-data-blacklist-early-termination.ipynb](missing-data-blacklist-early-termination/auto-ml-missing-data-blacklist-early-termination.ipynb)
-    - Dataset: scikit learn's [digit dataset](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_digits.html#sklearn.datasets.load_digits)
-    - Blacklist certain pipelines
-    - Specify a target metrics to indicate stopping criteria
-    - Handling Missing Data in the input
-
-- [auto-ml-sparse-data-train-test-split.ipynb](sparse-data-train-test-split/auto-ml-sparse-data-train-test-split.ipynb)
-    - Dataset: Scikit learn's [20newsgroup](http://scikit-learn.org/stable/datasets/twenty_newsgroups.html)
-    - Handle sparse datasets
-    - Specify custom train and validation set
-
-- [auto-ml-exploring-previous-runs.ipynb](exploring-previous-runs/auto-ml-exploring-previous-runs.ipynb)
-    - List all projects for the workspace
-    - List all AutoML Runs for a given project
-    - Get details for a AutoML Run. (Automl settings, run widget & all metrics)
-    - Download fitted pipeline for any iteration
-
-- [auto-ml-remote-execution-with-datastore.ipynb](remote-execution-with-datastore/auto-ml-remote-execution-with-datastore.ipynb)
-    - Dataset: Scikit learn's [20newsgroup](http://scikit-learn.org/stable/datasets/twenty_newsgroups.html)
-    - Download the data and store it in DataStore.
-
-- [auto-ml-classification-with-deployment.ipynb](classification-with-deployment/auto-ml-classification-with-deployment.ipynb)
-    - Dataset: scikit learn's [digit dataset](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_digits.html#sklearn.datasets.load_digits)
-    - Simple example of using Auto ML for classification
-    - Registering the model
-    - Creating Image and creating aci service
-    - Testing the aci service
-
-- [auto-ml-sample-weight.ipynb](sample-weight/auto-ml-sample-weight.ipynb)
-    - How to specifying sample_weight
-    - The difference that it makes to test results
-
-- [auto-ml-subsampling-local.ipynb](subsampling/auto-ml-subsampling-local.ipynb)
-    - How to enable subsampling
-
-- [auto-ml-dataprep.ipynb](dataprep/auto-ml-dataprep.ipynb)
-    - Using DataPrep for reading data
-
-- [auto-ml-dataprep-remote-execution.ipynb](dataprep-remote-execution/auto-ml-dataprep-remote-execution.ipynb)
-    - Using DataPrep for reading data with remote execution
-
-- [auto-ml-classification-with-whitelisting.ipynb](classification-with-whitelisting/auto-ml-classification-with-whitelisting.ipynb)
-    - Dataset: scikit learn's [digit dataset](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_digits.html#sklearn.datasets.load_digits)
-    - Simple example of using Auto ML for classification with whitelisting tensorflow models.
-    - Uses local compute for training
+- [auto-ml-regression-hardware-performance-explanation-and-featurization.ipynb](regression-hardware-performance-explanation-and-featurization/auto-ml-regression-hardware-performance-explanation-and-featurization.ipynb)
+    - Dataset: Hardware Performance Dataset
+    - Shows featurization and excplanation
+    - Uses azure compute for training
 
 - [auto-ml-forecasting-energy-demand.ipynb](forecasting-energy-demand/auto-ml-forecasting-energy-demand.ipynb)
     - Dataset: [NYC energy demand data](forecasting-a/nyc_energy.csv)
-    - Example of using AutoML for training a forecasting model
+    - Example of using automated ML for training a forecasting model
+
+- [auto-ml-classification-credit-card-fraud-local.ipynb](local-run-classification-credit-card-fraud/auto-ml-classification-credit-card-fraud-local.ipynb)
+    - Dataset: Kaggle's [credit card fraud detection dataset](https://www.kaggle.com/mlg-ulb/creditcardfraud)
+    - Simple example of using automated ML for classification to fraudulent credit card transactions
+    - Uses local compute for training
+
+- [auto-ml-classification-bank-marketing-all-features.ipynb](classification-bank-marketing-all-features/auto-ml-classification-bank-marketing-all-features.ipynb)
+    - Dataset: UCI's [bank marketing dataset](https://www.kaggle.com/janiobachmann/bank-marketing-dataset)
+    - Simple example of using automated ML for classification to predict term deposit subscriptions for a bank
+    - Uses azure compute for training
 
 - [auto-ml-forecasting-orange-juice-sales.ipynb](forecasting-orange-juice-sales/auto-ml-forecasting-orange-juice-sales.ipynb)
     - Dataset: [Dominick's grocery sales of orange juice](forecasting-b/dominicks_OJ.csv)
-    - Example of training an AutoML forecasting model on multiple time-series
+    - Example of training an automated ML forecasting model on multiple time-series
 
-- [auto-ml-classification-with-onnx.ipynb](classification-with-onnx/auto-ml-classification-with-onnx.ipynb)
-    - Dataset: scikit learn's [digit dataset](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_digits.html#sklearn.datasets.load_digits)
-    - Simple example of using Auto ML for classification with ONNX models
-    - Uses local compute for training
+- [auto-ml-forecasting-bike-share.ipynb](forecasting-bike-share/auto-ml-forecasting-bike-share.ipynb)
+    - Dataset: forecasting for a bike-sharing
+    - Example of training an automated ML forecasting model on multiple time-series
+
+- [automl-forecasting-function.ipynb](forecasting-high-frequency/automl-forecasting-function.ipynb)
+    - Example of training an automated ML forecasting model on multiple time-series
+
+- [auto-ml-forecasting-beer-remote.ipynb](forecasting-beer-remote/auto-ml-forecasting-beer-remote.ipynb)
+    - Example of training an automated ML forecasting model on multiple time-series
+    - Beer Production Forecasting
+
+- [auto-ml-continuous-retraining.ipynb](continuous-retraining/auto-ml-continuous-retraining.ipynb)
+    - Continous retraining using Pipelines and Time-Series TabularDataset
+
+- [auto-ml-classification-text-dnn.ipynb](classification-text-dnn/auto-ml-classification-text-dnn.ipynb)
+    - Classification with text data using deep learning in AutoML
+    - AutoML highlights here include using deep neural networks (DNNs) to create embedded features from text data.
+    - Depending on the compute cluster the user provides, AutoML tried out Bidirectional Encoder Representations from Transformers (BERT) when a GPU compute is used.
+    - Bidirectional Long-Short Term neural network (BiLSTM) when a CPU compute is used, thereby optimizing the choice of DNN for the uesr's setup.
 
 <a name="documentation"></a>
 See [Configure automated machine learning experiments](https://docs.microsoft.com/azure/machine-learning/service/how-to-configure-auto-train) to learn how more about the the settings and features available for automated machine learning experiments.
@@ -211,10 +177,18 @@ The main code of the file must be indented so that it is under this condition.
 <a name="troubleshooting"></a>
 # Troubleshooting
 ## automl_setup fails
-1. On windows, make sure that you are running automl_setup from an Anconda Prompt window rather than a regular cmd window.  You can launch the "Anaconda Prompt" window by hitting the Start button and typing "Anaconda Prompt".  If you don't see the application "Anaconda Prompt", you might not have conda or mini conda installed.  In that case, you can install it [here](https://conda.io/miniconda.html)
+1. On Windows, make sure that you are running automl_setup from an Anconda Prompt window rather than a regular cmd window.  You can launch the "Anaconda Prompt" window by hitting the Start button and typing "Anaconda Prompt".  If you don't see the application "Anaconda Prompt", you might not have conda or mini conda installed.  In that case, you can install it [here](https://conda.io/miniconda.html)
 2. Check that you have conda 64-bit installed rather than 32-bit.  You can check this with the command `conda info`.  The `platform` should be `win-64` for Windows or `osx-64` for Mac.
 3. Check that you have conda 4.4.10 or later.  You can check the version with the command `conda -V`.  If you have a previous version installed, you can update it using the command: `conda update conda`.
-4. Pass a new name as the first parameter to automl_setup so that it creates a new conda environment. You can view existing conda environments using `conda env list` and remove them with `conda env remove -n <environmentname>`. 
+4. On Linux, if the error is `gcc: error trying to exec 'cc1plus': execvp: No such file or directory`, install build essentials using the command `sudo apt-get install build-essential`.
+5. Pass a new name as the first parameter to automl_setup so that it creates a new conda environment. You can view existing conda environments using `conda env list` and remove them with `conda env remove -n <environmentname>`.
+
+## automl_setup_linux.sh fails
+If automl_setup_linux.sh fails on Ubuntu Linux with the error: `unable to execute 'gcc': No such file or directory`
+1. Make sure that outbound ports 53 and 80 are enabled.  On an Azure VM, you can do this from the Azure Portal by selecting the VM and clicking on Networking.
+2. Run the command: `sudo apt-get update`
+3. Run the command: `sudo apt-get install build-essential --fix-missing`
+4. Run `automl_setup_linux.sh` again.
 
 ## configuration.ipynb fails
 1) For local conda, make sure that you have susccessfully run automl_setup first.
@@ -242,16 +216,16 @@ Some Windows environments see an error loading numpy with the latest Python vers
 Check the tensorflow version in the automated ml conda environment. Supported versions are < 1.13. Uninstall tensorflow from the environment if version is >= 1.13
 You may check the version of tensorflow and uninstall as follows
 1) start a command shell, activate conda environment where automated ml packages are installed
-2) enter `pip freeze` and look for `tensorflow` , if found, the version listed should be < 1.13 
-3) If the listed version is a not a supported version,  `pip uninstall tensorflow` in the command shell and enter y for confirmation. 
+2) enter `pip freeze` and look for `tensorflow` , if found, the version listed should be < 1.13
+3) If the listed version is a not a supported version,  `pip uninstall tensorflow` in the command shell and enter y for confirmation.
 
-## Remote run: DsvmCompute.create fails 
+## Remote run: DsvmCompute.create fails
 There are several reasons why the DsvmCompute.create can fail.  The reason is usually in the error message but you have to look at the end of the error message for the detailed reason.  Some common reasons are:
 1) `Compute name is invalid, it should start with a letter, be between 2 and 16 character, and only include letters (a-zA-Z), numbers (0-9) and \'-\'.`  Note that underscore is not allowed in the name.
-2) `The requested VM size xxxxx is not available in the current region.`  You can select a different region or vm_size. 
+2) `The requested VM size xxxxx is not available in the current region.`  You can select a different region or vm_size.
 
 ## Remote run: Unable to establish SSH connection
-AutoML uses the SSH protocol to communicate with remote DSVMs.  This defaults to port 22.  Possible causes for this error are:
+Automated ML uses the SSH protocol to communicate with remote DSVMs.  This defaults to port 22.  Possible causes for this error are:
 1) The DSVM is not ready for SSH connections.  When DSVM creation completes, the DSVM might still not be ready to acceept SSH connections.  The sample notebooks have a one minute delay to allow for this.
 2) Your Azure Subscription may restrict the IP address ranges that can access the DSVM on port 22.  You can check this in the Azure Portal by selecting the Virtual Machine and then clicking Networking.  The Virtual Machine name is the name that you provided in the notebook plus 10 alpha numeric characters to make the name unique.  The Inbound Port Rules define what can access the VM on specific ports.  Note that there is a priority priority order.  So, a Deny entry with a low priority number will override a Allow entry with a higher priority number.
 
@@ -262,16 +236,16 @@ This is often an issue with the `get_data` method.
 3) You can get to the error log for the setup iteration by clicking the `Click here to see the run in Azure portal` link, click `Back to Experiment`, click on the highest run number and then click on Logs.
 
 ## Remote run: disk full
-AutoML creates files under /tmp/azureml_runs for each iteration that it runs.  It creates a folder with the iteration id.  For example: AutoML_9a038a18-77cc-48f1-80fb-65abdbc33abe_93.  Under this, there is a azureml-logs folder, which contains logs.  If you run too many iterations on the same DSVM, these files can fill the disk.
+Automated ML creates files under /tmp/azureml_runs for each iteration that it runs.  It creates a folder with the iteration id.  For example: AutoML_9a038a18-77cc-48f1-80fb-65abdbc33abe_93.  Under this, there is a azureml-logs folder, which contains logs.  If you run too many iterations on the same DSVM, these files can fill the disk.
 You can delete the files under /tmp/azureml_runs or just delete the VM and create a new one.
 If your get_data downloads files, make sure the delete them or they can use disk space as well.
 When using DataStore, it is good to specify an absolute path for the files so that they are downloaded just once.  If you specify a relative path, it will download a file for each iteration.
 
 ## Remote run: Iterations fail and the log contains "MemoryError"
-This can be caused by insufficient memory on the DSVM.  AutoML loads all training data into memory.  So, the available memory should be more than the training data size.
+This can be caused by insufficient memory on the DSVM.  Automated ML loads all training data into memory.  So, the available memory should be more than the training data size.
 If you are using a remote DSVM, memory is needed for each concurrent iteration.  The max_concurrent_iterations setting specifies the maximum concurrent iterations.  For example, if the training data size is 8Gb and max_concurrent_iterations is set to 10, the minimum memory required is at least 80Gb.
 To resolve this issue, allocate a DSVM with more memory or reduce the value specified for max_concurrent_iterations.
 
 ## Remote run: Iterations show as "Not Responding" in the RunDetails widget.
 This can be caused by too many concurrent iterations for a remote DSVM.  Each concurrent iteration usually takes 100% of a core when it is running.  Some iterations can use multiple cores.  So, the max_concurrent_iterations setting should always be less than the number of cores of the DSVM.
-To resolve this issue, try reducing the value specified for the max_concurrent_iterations setting.
+To resolve this issue, try reducing the value specified for the max_concurrent_iterations setting.

how-to-use-azureml/automated-machine-learning/automl_env.yml

@@ -1,22 +1,38 @@
-name: azure_automl
-dependencies:
-  # The python interpreter version.
-  # Currently Azure ML only supports 3.5.2 and later.
-- python>=3.5.2,<3.6.8
-- nb_conda
-- matplotlib==2.1.0
-- numpy>=1.11.0,<1.15.0
-- cython
-- urllib3<1.24
-- scipy>=1.0.0,<=1.1.0
-- scikit-learn>=0.18.0,<=0.19.1
-- pandas>=0.22.0,<0.23.0
-- tensorflow>=1.12.0
-- py-xgboost<=0.80
-
-- pip:
-  # Required packages for AzureML execution, history, and data preparation.
-  - azureml-sdk[automl,explain]
-  - azureml-widgets
-  - pandas_ml
-  
+name: azure_automl
+dependencies:
+  # The python interpreter version.
+  # Currently Azure ML only supports 3.5.2 and later.
+- pip
+- python>=3.5.2,<3.6.8
+- nb_conda
+- matplotlib==2.1.0
+- numpy>=1.16.0,<=1.16.2
+- cython
+- urllib3<1.24
+- scipy>=1.0.0,<=1.1.0
+- scikit-learn>=0.19.0,<=0.20.3
+- pandas>=0.22.0,<=0.23.4
+- py-xgboost<=0.80
+- pyarrow>=0.11.0
+- fbprophet==0.5
+- pytorch=1.1.0
+- cudatoolkit=9.0
+
+- pip:
+  # Required packages for AzureML execution, history, and data preparation.
+  - azureml-defaults
+  - azureml-train-automl
+  - azureml-train
+  - azureml-widgets
+  - azureml-explain-model
+  - azureml-pipeline
+  - azureml-contrib-interpret
+  - pytorch-transformers==1.0.0
+  - spacy==2.1.8
+  - joblib
+  - onnxruntime==0.4.0
+  - https://aka.ms/automl-resources/packages/en_core_web_sm-2.1.0.tar.gz
+
+channels:
+- conda-forge
+- pytorch

how-to-use-azureml/automated-machine-learning/automl_env_mac.yml

@@ -1,23 +1,39 @@
-name: azure_automl
-dependencies:
-  # The python interpreter version.
-  # Currently Azure ML only supports 3.5.2 and later.
-- python>=3.5.2,<3.6.8
-- nb_conda
-- matplotlib==2.1.0
-- numpy>=1.15.3
-- cython
-- urllib3<1.24
-- scipy>=1.0.0,<=1.1.0
-- scikit-learn>=0.18.0,<=0.19.1
-- pandas>=0.22.0,<0.23.0
-- tensorflow>=1.12.0
-- py-xgboost<=0.80
-
-- pip:
-  # Required packages for AzureML execution, history, and data preparation.
-  - azureml-sdk[automl,explain]
-  - azureml-widgets
-  - pandas_ml
-  
-
+name: azure_automl
+dependencies:
+  # The python interpreter version.
+  # Currently Azure ML only supports 3.5.2 and later.
+- pip
+- nomkl
+- python>=3.5.2,<3.6.8
+- nb_conda
+- matplotlib==2.1.0
+- numpy>=1.16.0,<=1.16.2
+- cython
+- urllib3<1.24
+- scipy>=1.0.0,<=1.1.0
+- scikit-learn>=0.19.0,<=0.20.3
+- pandas>=0.22.0,<0.23.0
+- py-xgboost<=0.80
+- pyarrow>=0.11.0
+- fbprophet==0.5
+- pytorch=1.1.0
+- cudatoolkit=9.0
+
+- pip:
+  # Required packages for AzureML execution, history, and data preparation.
+  - azureml-defaults
+  - azureml-train-automl
+  - azureml-train
+  - azureml-widgets
+  - azureml-explain-model
+  - azureml-pipeline
+  - azureml-contrib-interpret
+  - pytorch-transformers==1.0.0
+  - spacy==2.1.8
+  - joblib
+  - onnxruntime==0.4.0
+  - https://aka.ms/automl-resources/packages/en_core_web_sm-2.1.0.tar.gz  
+
+channels:
+- conda-forge
+- pytorch

how-to-use-azureml/automated-machine-learning/automl_setup.cmd

@@ -1,51 +1,63 @@
-@echo off
-set conda_env_name=%1
-set automl_env_file=%2
-set options=%3
-set PIP_NO_WARN_SCRIPT_LOCATION=0
-
-IF "%conda_env_name%"=="" SET conda_env_name="azure_automl"
-IF "%automl_env_file%"=="" SET automl_env_file="automl_env.yml"
-
-IF NOT EXIST %automl_env_file% GOTO YmlMissing
-
-call conda activate %conda_env_name% 2>nul:
-
-if not errorlevel 1 (
-  echo Upgrading azureml-sdk[automl,notebooks,explain] in existing conda environment %conda_env_name%
-  call pip install --upgrade azureml-sdk[automl,notebooks,explain]
-  if errorlevel 1 goto ErrorExit
-) else (
-  call conda env create -f %automl_env_file% -n %conda_env_name%
-)
-
-call conda activate %conda_env_name% 2>nul:
-if errorlevel 1 goto ErrorExit
-
-call python -m ipykernel install --user --name %conda_env_name% --display-name "Python (%conda_env_name%)"
-
-REM azureml.widgets is now installed as part of the pip install under the conda env.
-REM Removing the old user install so that the notebooks will use the latest widget.
-call jupyter nbextension uninstall --user --py azureml.widgets
-
-echo.
-echo.
-echo ***************************************
-echo * AutoML setup completed successfully *
-echo ***************************************
-IF NOT "%options%"=="nolaunch" (
-  echo.
-  echo Starting jupyter notebook - please run the configuration notebook 
-  echo.
-  jupyter notebook --log-level=50 --notebook-dir='..\..'
-)
-
-goto End
-
-:YmlMissing
-echo File %automl_env_file% not found.
-
-:ErrorExit
-echo Install failed
-
+@echo off
+set conda_env_name=%1
+set automl_env_file=%2
+set options=%3
+set PIP_NO_WARN_SCRIPT_LOCATION=0
+
+IF "%conda_env_name%"=="" SET conda_env_name="azure_automl"
+IF "%automl_env_file%"=="" SET automl_env_file="automl_env.yml"
+
+IF NOT EXIST %automl_env_file% GOTO YmlMissing
+
+IF "%CONDA_EXE%"=="" GOTO CondaMissing
+
+call conda activate %conda_env_name% 2>nul:
+
+if not errorlevel 1 (
+  echo Upgrading existing conda environment %conda_env_name%
+  call pip uninstall azureml-train-automl -y -q
+  call conda env update --name %conda_env_name% --file %automl_env_file%
+  if errorlevel 1 goto ErrorExit
+) else (
+  call conda env create -f %automl_env_file% -n %conda_env_name%
+)
+
+call conda activate %conda_env_name% 2>nul:
+if errorlevel 1 goto ErrorExit
+
+call python -m ipykernel install --user --name %conda_env_name% --display-name "Python (%conda_env_name%)"
+
+REM azureml.widgets is now installed as part of the pip install under the conda env.
+REM Removing the old user install so that the notebooks will use the latest widget.
+call jupyter nbextension uninstall --user --py azureml.widgets
+
+echo.
+echo.
+echo ***************************************
+echo * AutoML setup completed successfully *
+echo ***************************************
+IF NOT "%options%"=="nolaunch" (
+  echo.
+  echo Starting jupyter notebook - please run the configuration notebook 
+  echo.
+  jupyter notebook --log-level=50 --notebook-dir='..\..'
+)
+
+goto End
+
+:CondaMissing
+echo Please run this script from an Anaconda Prompt window.
+echo You can start an Anaconda Prompt window by
+echo typing Anaconda Prompt on the Start menu.
+echo If you don't see the Anaconda Prompt app, install Miniconda.
+echo If you are running an older version of Miniconda or Anaconda,
+echo you can upgrade using the command: conda update conda
+goto End
+
+:YmlMissing
+echo File %automl_env_file% not found.
+
+:ErrorExit
+echo Install failed
+
 :End

how-to-use-azureml/automated-machine-learning/automl_setup_linux.sh

@@ -22,8 +22,9 @@ fi
 
 if source activate $CONDA_ENV_NAME 2> /dev/null
 then
-   echo "Upgrading azureml-sdk[automl,notebooks,explain] in existing conda environment" $CONDA_ENV_NAME
-   pip install --upgrade azureml-sdk[automl,notebooks,explain] &&
+   echo "Upgrading existing conda environment" $CONDA_ENV_NAME
+   pip uninstall azureml-train-automl -y -q
+   conda env update --name $CONDA_ENV_NAME --file $AUTOML_ENV_FILE &&
    jupyter nbextension uninstall --user --py azureml.widgets
 else
    conda env create -f $AUTOML_ENV_FILE -n $CONDA_ENV_NAME &&

how-to-use-azureml/automated-machine-learning/automl_setup_mac.sh

@@ -22,8 +22,9 @@ fi
 
 if source activate $CONDA_ENV_NAME 2> /dev/null
 then
-   echo "Upgrading azureml-sdk[automl,notebooks,explain] in existing conda environment" $CONDA_ENV_NAME
-   pip install --upgrade azureml-sdk[automl,notebooks,explain] &&
+   echo "Upgrading existing conda environment" $CONDA_ENV_NAME
+   pip uninstall azureml-train-automl -y -q
+   conda env update --name $CONDA_ENV_NAME --file $AUTOML_ENV_FILE &&
    jupyter nbextension uninstall --user --py azureml.widgets
 else
    conda env create -f $AUTOML_ENV_FILE -n $CONDA_ENV_NAME &&
@@ -31,7 +32,6 @@ else
    conda install lightgbm -c conda-forge -y &&
    python -m ipykernel install --user --name $CONDA_ENV_NAME --display-name "Python ($CONDA_ENV_NAME)" &&
    jupyter nbextension uninstall --user --py azureml.widgets &&
-   pip install numpy==1.15.3 &&
    echo "" &&
    echo "" &&
    echo "***************************************" &&

how-to-use-azureml/automated-machine-learning/classification-bank-marketing-all-features/auto-ml-classification-bank-marketing-all-features.ipynb

@@ -0,0 +1,894 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
+        "\n",
+        "Licensed under the MIT License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/automated-machine-learning/classification-bank-marketing-all-features/auto-ml-classification-bank-marketing.png)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# Automated Machine Learning\n",
+        "_**Classification with Deployment using a Bank Marketing Dataset**_\n",
+        "\n",
+        "## Contents\n",
+        "1. [Introduction](#Introduction)\n",
+        "1. [Setup](#Setup)\n",
+        "1. [Train](#Train)\n",
+        "1. [Results](#Results)\n",
+        "1. [Deploy](#Deploy)\n",
+        "1. [Test](#Test)\n",
+        "1. [Acknowledgements](#Acknowledgements)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Introduction\n",
+        "\n",
+        "In this example we use the UCI Bank Marketing dataset to showcase how you can use AutoML for a  classification problem and deploy it to an Azure Container Instance (ACI). The classification goal is to predict if the client will subscribe to a term deposit with the bank.\n",
+        "\n",
+        "If you are using an Azure Machine Learning Notebook VM, you are all set.  Otherwise, go through the [configuration](../../../configuration.ipynb)  notebook first if you haven't already to establish your connection to the AzureML Workspace. \n",
+        "\n",
+        "Please find the ONNX related documentations [here](https://github.com/onnx/onnx).\n",
+        "\n",
+        "In this notebook you will learn how to:\n",
+        "1. Create an experiment using an existing workspace.\n",
+        "2. Configure AutoML using `AutoMLConfig`.\n",
+        "3. Train the model using local compute with ONNX compatible config on.\n",
+        "4. Explore the results, featurization transparency options and save the ONNX model\n",
+        "5. Inference with the ONNX model.\n",
+        "6. Register the model.\n",
+        "7. Create a container image.\n",
+        "8. Create an Azure Container Instance (ACI) service.\n",
+        "9. Test the ACI service.\n",
+        "\n",
+        "In addition this notebook showcases the following features\n",
+        "- **Blacklisting** certain pipelines\n",
+        "- Specifying **target metrics** to indicate stopping criteria\n",
+        "- Handling **missing data** in the input"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Setup\n",
+        "\n",
+        "As part of the setup you have already created an Azure ML `Workspace` object. For AutoML you will need to create an `Experiment` object, which is a named object in a `Workspace` used to run experiments."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import logging\n",
+        "\n",
+        "from matplotlib import pyplot as plt\n",
+        "import pandas as pd\n",
+        "import os\n",
+        "\n",
+        "import azureml.core\n",
+        "from azureml.core.experiment import Experiment\n",
+        "from azureml.core.workspace import Workspace\n",
+        "from azureml.automl.core.featurization import FeaturizationConfig\n",
+        "from azureml.core.dataset import Dataset\n",
+        "from azureml.train.automl import AutoMLConfig\n",
+        "from azureml.explain.model._internal.explanation_client import ExplanationClient"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "ws = Workspace.from_config()\n",
+        "\n",
+        "# choose a name for experiment\n",
+        "experiment_name = 'automl-classification-bmarketing-all'\n",
+        "\n",
+        "experiment=Experiment(ws, experiment_name)\n",
+        "\n",
+        "output = {}\n",
+        "output['SDK version'] = azureml.core.VERSION\n",
+        "output['Subscription ID'] = ws.subscription_id\n",
+        "output['Workspace'] = ws.name\n",
+        "output['Resource Group'] = ws.resource_group\n",
+        "output['Location'] = ws.location\n",
+        "output['Experiment Name'] = experiment.name\n",
+        "pd.set_option('display.max_colwidth', -1)\n",
+        "outputDf = pd.DataFrame(data = output, index = [''])\n",
+        "outputDf.T"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Create or Attach existing AmlCompute\n",
+        "You will need to create a compute target for your AutoML run. In this tutorial, you create AmlCompute as your training compute resource.\n",
+        "#### Creation of AmlCompute takes approximately 5 minutes. \n",
+        "If the AmlCompute with that name is already in your workspace this code will skip the creation process.\n",
+        "As with other Azure services, there are limits on certain resources (e.g. AmlCompute) associated with the Azure Machine Learning service. Please read this article on the default limits and how to request more quota."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.compute import AmlCompute\n",
+        "from azureml.core.compute import ComputeTarget\n",
+        "\n",
+        "# Choose a name for your cluster.\n",
+        "amlcompute_cluster_name = \"cpu-cluster-4\"\n",
+        "\n",
+        "found = False\n",
+        "# Check if this compute target already exists in the workspace.\n",
+        "cts = ws.compute_targets\n",
+        "if amlcompute_cluster_name in cts and cts[amlcompute_cluster_name].type == 'AmlCompute':\n",
+        "    found = True\n",
+        "    print('Found existing compute target.')\n",
+        "    compute_target = cts[amlcompute_cluster_name]\n",
+        "    \n",
+        "if not found:\n",
+        "    print('Creating a new compute target...')\n",
+        "    provisioning_config = AmlCompute.provisioning_configuration(vm_size = \"STANDARD_D2_V2\", # for GPU, use \"STANDARD_NC6\"\n",
+        "                                                                #vm_priority = 'lowpriority', # optional\n",
+        "                                                                max_nodes = 6)\n",
+        "\n",
+        "    # Create the cluster.\n",
+        "    compute_target = ComputeTarget.create(ws, amlcompute_cluster_name, provisioning_config)\n",
+        "    \n",
+        "print('Checking cluster status...')\n",
+        "# Can poll for a minimum number of nodes and for a specific timeout.\n",
+        "# If no min_node_count is provided, it will use the scale settings for the cluster.\n",
+        "compute_target.wait_for_completion(show_output = True, min_node_count = None, timeout_in_minutes = 20)\n",
+        "    \n",
+        "# For a more detailed view of current AmlCompute status, use get_status()."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# Data"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Load Data\n",
+        "\n",
+        "Leverage azure compute to load the bank marketing dataset as a Tabular Dataset into the dataset variable. "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Training Data"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "data = pd.read_csv(\"https://automlsamplenotebookdata.blob.core.windows.net/automl-sample-notebook-data/bankmarketing_train.csv\")\n",
+        "data.head()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Add missing values in 75% of the lines.\n",
+        "import numpy as np\n",
+        "\n",
+        "missing_rate = 0.75\n",
+        "n_missing_samples = int(np.floor(data.shape[0] * missing_rate))\n",
+        "missing_samples = np.hstack((np.zeros(data.shape[0] - n_missing_samples, dtype=np.bool), np.ones(n_missing_samples, dtype=np.bool)))\n",
+        "rng = np.random.RandomState(0)\n",
+        "rng.shuffle(missing_samples)\n",
+        "missing_features = rng.randint(0, data.shape[1], n_missing_samples)\n",
+        "data.values[np.where(missing_samples)[0], missing_features] = np.nan"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "if not os.path.isdir('data'):\n",
+        "    os.mkdir('data')\n",
+        "    \n",
+        "# Save the train data to a csv to be uploaded to the datastore\n",
+        "pd.DataFrame(data).to_csv(\"data/train_data.csv\", index=False)\n",
+        "\n",
+        "ds = ws.get_default_datastore()\n",
+        "ds.upload(src_dir='./data', target_path='bankmarketing', overwrite=True, show_progress=True)\n",
+        "\n",
+        " \n",
+        "\n",
+        "# Upload the training data as a tabular dataset for access during training on remote compute\n",
+        "train_data = Dataset.Tabular.from_delimited_files(path=ds.path('bankmarketing/train_data.csv'))\n",
+        "label = \"y\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Validation Data"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "validation_data = \"https://automlsamplenotebookdata.blob.core.windows.net/automl-sample-notebook-data/bankmarketing_validate.csv\"\n",
+        "validation_dataset = Dataset.Tabular.from_delimited_files(validation_data)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Test Data"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "test_data = \"https://automlsamplenotebookdata.blob.core.windows.net/automl-sample-notebook-data/bankmarketing_test.csv\"\n",
+        "test_dataset = Dataset.Tabular.from_delimited_files(test_data)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Train\n",
+        "\n",
+        "Instantiate a AutoMLConfig object. This defines the settings and data used to run the experiment.\n",
+        "\n",
+        "|Property|Description|\n",
+        "|-|-|\n",
+        "|**task**|classification or regression or forecasting|\n",
+        "|**primary_metric**|This is the metric that you want to optimize. Classification supports the following primary metrics: <br><i>accuracy</i><br><i>AUC_weighted</i><br><i>average_precision_score_weighted</i><br><i>norm_macro_recall</i><br><i>precision_score_weighted</i>|\n",
+        "|**iteration_timeout_minutes**|Time limit in minutes for each iteration.|\n",
+        "|**blacklist_models** | *List* of *strings* indicating machine learning algorithms for AutoML to avoid in this run. <br><br> Allowed values for **Classification**<br><i>LogisticRegression</i><br><i>SGD</i><br><i>MultinomialNaiveBayes</i><br><i>BernoulliNaiveBayes</i><br><i>SVM</i><br><i>LinearSVM</i><br><i>KNN</i><br><i>DecisionTree</i><br><i>RandomForest</i><br><i>ExtremeRandomTrees</i><br><i>LightGBM</i><br><i>GradientBoosting</i><br><i>TensorFlowDNN</i><br><i>TensorFlowLinearClassifier</i><br><br>Allowed values for **Regression**<br><i>ElasticNet</i><br><i>GradientBoosting</i><br><i>DecisionTree</i><br><i>KNN</i><br><i>LassoLars</i><br><i>SGD</i><br><i>RandomForest</i><br><i>ExtremeRandomTrees</i><br><i>LightGBM</i><br><i>TensorFlowLinearRegressor</i><br><i>TensorFlowDNN</i><br><br>Allowed values for **Forecasting**<br><i>ElasticNet</i><br><i>GradientBoosting</i><br><i>DecisionTree</i><br><i>KNN</i><br><i>LassoLars</i><br><i>SGD</i><br><i>RandomForest</i><br><i>ExtremeRandomTrees</i><br><i>LightGBM</i><br><i>TensorFlowLinearRegressor</i><br><i>TensorFlowDNN</i><br><i>Arima</i><br><i>Prophet</i>|\n",
+        "| **whitelist_models** |  *List* of *strings* indicating machine learning algorithms for AutoML to use in this run. Same values listed above for **blacklist_models** allowed for **whitelist_models**.|\n",
+        "|**experiment_exit_score**| Value indicating the target for *primary_metric*. <br>Once the target is surpassed the run terminates.|\n",
+        "|**experiment_timeout_minutes**| Maximum amount of time in minutes that all iterations combined can take before the experiment terminates.|\n",
+        "|**enable_early_stopping**| Flag to enble early termination if the score is not improving in the short term.|\n",
+        "|**featurization**| 'auto' / 'off'  Indicator for whether featurization step should be done automatically or not. Note: If the input data is sparse, featurization cannot be turned on.|\n",
+        "|**n_cross_validations**|Number of cross validation splits.|\n",
+        "|**training_data**|Input dataset, containing both features and label column.|\n",
+        "|**label_column_name**|The name of the label column.|\n",
+        "|**model_explainability**|Indicate to explain each trained pipeline or not.|\n",
+        "\n",
+        "**_You can find more information about primary metrics_** [here](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-configure-auto-train#primary-metric)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "automl_settings = {\n",
+        "    \"experiment_timeout_minutes\" : 20,\n",
+        "    \"enable_early_stopping\" : True,\n",
+        "    \"iteration_timeout_minutes\": 5,\n",
+        "    \"max_concurrent_iterations\": 4,\n",
+        "    \"max_cores_per_iteration\": -1,\n",
+        "    #\"n_cross_validations\": 2,\n",
+        "    \"primary_metric\": 'AUC_weighted',\n",
+        "    \"featurization\": 'auto',\n",
+        "    \"verbosity\": logging.INFO,\n",
+        "}\n",
+        "\n",
+        "automl_config = AutoMLConfig(task = 'classification',\n",
+        "                             debug_log = 'automl_errors.log',\n",
+        "                             compute_target=compute_target,\n",
+        "                             experiment_exit_score = 0.9984,\n",
+        "                             blacklist_models = ['KNN','LinearSVM'],\n",
+        "                             enable_onnx_compatible_models=True,\n",
+        "                             training_data = train_data,\n",
+        "                             label_column_name = label,\n",
+        "                             validation_data = validation_dataset,\n",
+        "                             model_explainability=True,\n",
+        "                             **automl_settings\n",
+        "                            )"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Call the `submit` method on the experiment object and pass the run configuration. Execution of local runs is synchronous. Depending on the data and the number of iterations this can run for a while."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "remote_run = experiment.submit(automl_config, show_output = False)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "remote_run"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Run the following cell to access previous runs. Uncomment the cell below and update the run_id."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "#from azureml.train.automl.run import AutoMLRun\n",
+        "#experiment_name = 'automl-classification-bmarketing'\n",
+        "#experiment = Experiment(ws, experiment_name)\n",
+        "#remote_run = AutoMLRun(experiment=experiment, run_id='<run_ID_goes_here')\n",
+        "#remote_run"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Wait for the remote run to complete\n",
+        "remote_run.wait_for_completion()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "best_run_customized, fitted_model_customized = remote_run.get_output()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Transparency\n",
+        "\n",
+        "View updated featurization summary"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "custom_featurizer = fitted_model_customized.named_steps['datatransformer']\n",
+        "df = custom_featurizer.get_featurization_summary()\n",
+        "pd.DataFrame(data=df)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Set `is_user_friendly=False` to get a more detailed summary for the transforms being applied."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "df = custom_featurizer.get_featurization_summary(is_user_friendly=False)\n",
+        "pd.DataFrame(data=df)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "df = custom_featurizer.get_stats_feature_type_summary()\n",
+        "pd.DataFrame(data=df)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Results"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.widgets import RunDetails\n",
+        "RunDetails(remote_run).show() "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Retrieve the Best Model's explanation\n",
+        "Retrieve the explanation from the best_run which includes explanations for engineered features and raw features. Make sure that the run for generating explanations for the best model is completed."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Wait for the best model explanation run to complete\n",
+        "from azureml.train.automl.run import AutoMLRun\n",
+        "model_explainability_run_id = remote_run.get_properties().get('ModelExplainRunId')\n",
+        "print(model_explainability_run_id)\n",
+        "if model_explainability_run_id is not None:\n",
+        "    model_explainability_run = AutoMLRun(experiment=experiment, run_id=model_explainability_run_id)\n",
+        "    model_explainability_run.wait_for_completion()\n",
+        "\n",
+        "# Get the best run object\n",
+        "best_run, fitted_model = remote_run.get_output()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Download engineered feature importance from artifact store\n",
+        "You can use ExplanationClient to download the engineered feature explanations from the artifact store of the best_run."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "client = ExplanationClient.from_run(best_run)\n",
+        "engineered_explanations = client.download_model_explanation(raw=False)\n",
+        "exp_data = engineered_explanations.get_feature_importance_dict()\n",
+        "exp_data"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Download raw feature importance from artifact store\n",
+        "You can use ExplanationClient to download the raw feature explanations from the artifact store of the best_run."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "client = ExplanationClient.from_run(best_run)\n",
+        "engineered_explanations = client.download_model_explanation(raw=True)\n",
+        "exp_data = engineered_explanations.get_feature_importance_dict()\n",
+        "exp_data"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Retrieve the Best ONNX Model\n",
+        "\n",
+        "Below we select the best pipeline from our iterations. The `get_output` method returns the best run and the fitted model. The Model includes the pipeline and any pre-processing.  Overloads on `get_output` allow you to retrieve the best run and fitted model for *any* logged metric or for a particular *iteration*.\n",
+        "\n",
+        "Set the parameter return_onnx_model=True to retrieve the best ONNX model, instead of the Python model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "best_run, onnx_mdl = remote_run.get_output(return_onnx_model=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Save the best ONNX model"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.automl.runtime.onnx_convert import OnnxConverter\n",
+        "onnx_fl_path = \"./best_model.onnx\"\n",
+        "OnnxConverter.save_onnx_model(onnx_mdl, onnx_fl_path)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Predict with the ONNX model, using onnxruntime package"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import sys\n",
+        "import json\n",
+        "from azureml.automl.core.onnx_convert import OnnxConvertConstants\n",
+        "from azureml.train.automl import constants\n",
+        "\n",
+        "if sys.version_info < OnnxConvertConstants.OnnxIncompatiblePythonVersion:\n",
+        "    python_version_compatible = True\n",
+        "else:\n",
+        "    python_version_compatible = False\n",
+        "\n",
+        "import onnxruntime\n",
+        "from azureml.automl.runtime.onnx_convert import OnnxInferenceHelper\n",
+        "\n",
+        "def get_onnx_res(run):\n",
+        "    res_path = 'onnx_resource.json'\n",
+        "    run.download_file(name=constants.MODEL_RESOURCE_PATH_ONNX, output_file_path=res_path)\n",
+        "    with open(res_path) as f:\n",
+        "        onnx_res = json.load(f)\n",
+        "    return onnx_res\n",
+        "\n",
+        "if python_version_compatible:\n",
+        "    test_df = test_dataset.to_pandas_dataframe()\n",
+        "    mdl_bytes = onnx_mdl.SerializeToString()\n",
+        "    onnx_res = get_onnx_res(best_run)\n",
+        "\n",
+        "    onnxrt_helper = OnnxInferenceHelper(mdl_bytes, onnx_res)\n",
+        "    pred_onnx, pred_prob_onnx = onnxrt_helper.predict(test_df)\n",
+        "\n",
+        "    print(pred_onnx)\n",
+        "    print(pred_prob_onnx)\n",
+        "else:\n",
+        "    print('Please use Python version 3.6 or 3.7 to run the inference helper.')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Deploy\n",
+        "\n",
+        "### Retrieve the Best Model\n",
+        "\n",
+        "Below we select the best pipeline from our iterations. The `get_output` method on `automl_classifier` returns the best run and the fitted model for the last invocation. Overloads on `get_output` allow you to retrieve the best run and fitted model for *any* logged metric or for a particular *iteration*."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Widget for Monitoring Runs\n",
+        "\n",
+        "The widget will first report a \"loading\" status while running the first iteration. After completing the first iteration, an auto-updating graph and table will be shown. The widget will refresh once per minute, so you should see the graph update as child runs complete.\n",
+        "\n",
+        "**Note:** The widget displays a link at the bottom. Use this link to open a web interface to explore the individual run details"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "best_run, fitted_model = remote_run.get_output()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "model_name = best_run.properties['model_name']\n",
+        "\n",
+        "script_file_name = 'inference/score.py'\n",
+        "conda_env_file_name = 'inference/env.yml'\n",
+        "\n",
+        "best_run.download_file('outputs/scoring_file_v_1_0_0.py', 'inference/score.py')\n",
+        "best_run.download_file('outputs/conda_env_v_1_0_0.yml', 'inference/env.yml')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Register the Fitted Model for Deployment\n",
+        "If neither `metric` nor `iteration` are specified in the `register_model` call, the iteration with the best primary metric is registered."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "description = 'AutoML Model trained on bank marketing data to predict if a client will subscribe to a term deposit'\n",
+        "tags = None\n",
+        "model = remote_run.register_model(model_name = model_name, description = description, tags = tags)\n",
+        "\n",
+        "print(remote_run.model_id) # This will be written to the script file later in the notebook."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Deploy the model as a Web Service on Azure Container Instance"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.model import InferenceConfig\n",
+        "from azureml.core.webservice import AciWebservice\n",
+        "from azureml.core.webservice import Webservice\n",
+        "from azureml.core.model import Model\n",
+        "\n",
+        "inference_config = InferenceConfig(runtime = \"python\", \n",
+        "                                   entry_script = script_file_name,\n",
+        "                                   conda_file = conda_env_file_name)\n",
+        "\n",
+        "aciconfig = AciWebservice.deploy_configuration(cpu_cores = 1, \n",
+        "                                               memory_gb = 1, \n",
+        "                                               tags = {'area': \"bmData\", 'type': \"automl_classification\"}, \n",
+        "                                               description = 'sample service for Automl Classification')\n",
+        "\n",
+        "aci_service_name = 'automl-sample-bankmarketing-all'\n",
+        "print(aci_service_name)\n",
+        "aci_service = Model.deploy(ws, aci_service_name, [model], inference_config, aciconfig)\n",
+        "aci_service.wait_for_deployment(True)\n",
+        "print(aci_service.state)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Delete a Web Service\n",
+        "\n",
+        "Deletes the specified web service."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "#aci_service.delete()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Get Logs from a Deployed Web Service\n",
+        "\n",
+        "Gets logs from a deployed web service."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "#aci_service.get_logs()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Test\n",
+        "\n",
+        "Now that the model is trained, run the test data through the trained model to get the predicted values."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Load the bank marketing datasets.\n",
+        "from numpy import array"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "X_test = test_dataset.drop_columns(columns=['y'])\n",
+        "y_test = test_dataset.keep_columns(columns=['y'], validate=True)\n",
+        "test_dataset.take(5).to_pandas_dataframe()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "X_test = X_test.to_pandas_dataframe()\n",
+        "y_test = y_test.to_pandas_dataframe()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "y_pred  = fitted_model.predict(X_test)\n",
+        "actual = array(y_test)\n",
+        "actual = actual[:,0]\n",
+        "print(y_pred.shape, \" \", actual.shape)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Calculate metrics for the prediction\n",
+        "\n",
+        "Now visualize the data on a scatter plot to show what our truth (actual) values are compared to the predicted values \n",
+        "from the trained model that was returned."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "%matplotlib notebook\n",
+        "test_pred = plt.scatter(actual, y_pred, color='b')\n",
+        "test_test = plt.scatter(actual, actual, color='g')\n",
+        "plt.legend((test_pred, test_test), ('prediction', 'truth'), loc='upper left', fontsize=8)\n",
+        "plt.show()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Acknowledgements"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "This Bank Marketing dataset is made available under the Creative Commons (CCO: Public Domain) License: https://creativecommons.org/publicdomain/zero/1.0/. Any rights in individual contents of the database are licensed under the Database Contents License: https://creativecommons.org/publicdomain/zero/1.0/ and is available at: https://www.kaggle.com/janiobachmann/bank-marketing-dataset .\n",
+        "\n",
+        "_**Acknowledgements**_\n",
+        "This data set is originally available within the UCI Machine Learning Database: https://archive.ics.uci.edu/ml/datasets/bank+marketing\n",
+        "\n",
+        "[Moro et al., 2014] S. Moro, P. Cortez and P. Rita. A Data-Driven Approach to Predict the Success of Bank Telemarketing. Decision Support Systems, Elsevier, 62:22-31, June 2014"
+      ]
+    }
+  ],
+  "metadata": {
+    "authors": [
+      {
+        "name": "anumamah"
+      }
+    ],
+    "category": "tutorial",
+    "compute": [
+      "AML"
+    ],
+    "datasets": [
+      "Bankmarketing"
+    ],
+    "deployment": [
+      "ACI"
+    ],
+    "exclude_from_index": false,
+    "framework": [
+      "None"
+    ],
+    "friendly_name": "Automated ML run with basic edition features.",
+    "index_order": 5,
+    "kernelspec": {
+      "display_name": "Python 3.6",
+      "language": "python",
+      "name": "python36"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.6.7"
+    },
+    "tags": [
+      "featurization",
+      "explainability",
+      "remote_run",
+      "AutomatedML"
+    ],
+    "task": "Classification"
+  },
+  "nbformat": 4,
+  "nbformat_minor": 2
+}

how-to-use-azureml/automated-machine-learning/classification-bank-marketing-all-features/auto-ml-classification-bank-marketing-all-features.yml

@@ -0,0 +1,13 @@
+name: auto-ml-classification-bank-marketing-all-features
+dependencies:
+- pip:
+  - azureml-sdk
+  - interpret
+  - azureml-defaults
+  - azureml-train-automl
+  - azureml-widgets
+  - matplotlib
+  - pandas_ml
+  - onnxruntime==0.4.0
+  - azureml-explain-model
+  - azureml-contrib-interpret

how-to-use-azureml/automated-machine-learning/classification-credit-card-fraud/auto-ml-classification-credit-card-fraud.ipynb

@@ -0,0 +1,496 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
+        "\n",
+        "Licensed under the MIT License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/automated-machine-learning/classification-credit-card-fraud/auto-ml-classification-credit-card-fraud.png)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# Automated Machine Learning\n",
+        "_**Classification of credit card fraudulent transactions on remote compute **_\n",
+        "\n",
+        "## Contents\n",
+        "1. [Introduction](#Introduction)\n",
+        "1. [Setup](#Setup)\n",
+        "1. [Train](#Train)\n",
+        "1. [Results](#Results)\n",
+        "1. [Test](#Test)\n",
+        "1. [Acknowledgements](#Acknowledgements)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Introduction\n",
+        "\n",
+        "In this example we use the associated credit card dataset to showcase how you can use AutoML for a simple classification problem. The goal is to predict if a credit card transaction is considered a fraudulent charge.\n",
+        "\n",
+        "This notebook is using remote compute to train the model.\n",
+        "\n",
+        "If you are using an Azure Machine Learning [Notebook VM](https://docs.microsoft.com/en-us/azure/machine-learning/service/tutorial-1st-experiment-sdk-setup), you are all set. Otherwise, go through the [configuration](../../../configuration.ipynb) notebook first if you haven't already to establish your connection to the AzureML Workspace. \n",
+        "\n",
+        "In this notebook you will learn how to:\n",
+        "1. Create an experiment using an existing workspace.\n",
+        "2. Configure AutoML using `AutoMLConfig`.\n",
+        "3. Train the model using remote compute.\n",
+        "4. Explore the results.\n",
+        "5. Test the fitted model."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Setup\n",
+        "\n",
+        "As part of the setup you have already created an Azure ML `Workspace` object. For Automated ML you will need to create an `Experiment` object, which is a named object in a `Workspace` used to run experiments."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import logging\n",
+        "\n",
+        "from matplotlib import pyplot as plt\n",
+        "import pandas as pd\n",
+        "import os\n",
+        "\n",
+        "import azureml.core\n",
+        "from azureml.core.experiment import Experiment\n",
+        "from azureml.core.workspace import Workspace\n",
+        "from azureml.core.dataset import Dataset\n",
+        "from azureml.train.automl import AutoMLConfig"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "ws = Workspace.from_config()\n",
+        "\n",
+        "# choose a name for experiment\n",
+        "experiment_name = 'automl-classification-ccard-remote'\n",
+        "\n",
+        "experiment=Experiment(ws, experiment_name)\n",
+        "\n",
+        "output = {}\n",
+        "output['SDK version'] = azureml.core.VERSION\n",
+        "output['Subscription ID'] = ws.subscription_id\n",
+        "output['Workspace'] = ws.name\n",
+        "output['Resource Group'] = ws.resource_group\n",
+        "output['Location'] = ws.location\n",
+        "output['Experiment Name'] = experiment.name\n",
+        "pd.set_option('display.max_colwidth', -1)\n",
+        "outputDf = pd.DataFrame(data = output, index = [''])\n",
+        "outputDf.T"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Create or Attach existing AmlCompute\n",
+        "A compute target is required to execute the Automated ML run. In this tutorial, you create AmlCompute as your training compute resource.\n",
+        "#### Creation of AmlCompute takes approximately 5 minutes. \n",
+        "If the AmlCompute with that name is already in your workspace this code will skip the creation process.\n",
+        "As with other Azure services, there are limits on certain resources (e.g. AmlCompute) associated with the Azure Machine Learning service. Please read [this article](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-manage-quotas) on the default limits and how to request more quota."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.compute import AmlCompute\n",
+        "from azureml.core.compute import ComputeTarget\n",
+        "\n",
+        "# Choose a name for your AmlCompute cluster.\n",
+        "amlcompute_cluster_name = \"cpu-cluster-1\"\n",
+        "\n",
+        "found = False\n",
+        "# Check if this compute target already exists in the workspace.\n",
+        "cts = ws.compute_targets\n",
+        "if amlcompute_cluster_name in cts and cts[amlcompute_cluster_name].type == 'cpu-cluster-1':\n",
+        "    found = True\n",
+        "    print('Found existing compute target.')\n",
+        "    compute_target = cts[amlcompute_cluster_name]\n",
+        "    \n",
+        "if not found:\n",
+        "    print('Creating a new compute target...')\n",
+        "    provisioning_config = AmlCompute.provisioning_configuration(vm_size = \"STANDARD_DS12_V2\", # for GPU, use \"STANDARD_NC6\"\n",
+        "                                                                #vm_priority = 'lowpriority', # optional\n",
+        "                                                                max_nodes = 6)\n",
+        "\n",
+        "    # Create the cluster.\n",
+        "    compute_target = ComputeTarget.create(ws, amlcompute_cluster_name, provisioning_config)\n",
+        "    \n",
+        "print('Checking cluster status...')\n",
+        "# Can poll for a minimum number of nodes and for a specific timeout.\n",
+        "# If no min_node_count is provided, it will use the scale settings for the cluster.\n",
+        "compute_target.wait_for_completion(show_output = True, min_node_count = None, timeout_in_minutes = 20)\n",
+        "\n",
+        "# For a more detailed view of current AmlCompute status, use get_status()."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# Data"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Load Data\n",
+        "\n",
+        "Load the credit card dataset from a csv file containing both training features and labels. The features are inputs to the model, while the training labels represent the expected output of the model. Next, we'll split the data using random_split and extract the training data for the model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "data = \"https://automlsamplenotebookdata.blob.core.windows.net/automl-sample-notebook-data/creditcard.csv\"\n",
+        "dataset = Dataset.Tabular.from_delimited_files(data)\n",
+        "training_data, validation_data = dataset.random_split(percentage=0.8, seed=223)\n",
+        "label_column_name = 'Class'"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Train\n",
+        "\n",
+        "Instantiate a AutoMLConfig object. This defines the settings and data used to run the experiment.\n",
+        "\n",
+        "|Property|Description|\n",
+        "|-|-|\n",
+        "|**task**|classification or regression|\n",
+        "|**primary_metric**|This is the metric that you want to optimize. Classification supports the following primary metrics: <br><i>accuracy</i><br><i>AUC_weighted</i><br><i>average_precision_score_weighted</i><br><i>norm_macro_recall</i><br><i>precision_score_weighted</i>|\n",
+        "|**enable_early_stopping**|Stop the run if the metric score is not showing improvement.|\n",
+        "|**n_cross_validations**|Number of cross validation splits.|\n",
+        "|**training_data**|Input dataset, containing both features and label column.|\n",
+        "|**label_column_name**|The name of the label column.|\n",
+        "\n",
+        "**_You can find more information about primary metrics_** [here](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-configure-auto-train#primary-metric)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "automl_settings = {\n",
+        "    \"n_cross_validations\": 3,\n",
+        "    \"primary_metric\": 'average_precision_score_weighted',\n",
+        "    \"preprocess\": True,\n",
+        "    \"enable_early_stopping\": True,\n",
+        "    \"max_concurrent_iterations\": 2, # This is a limit for testing purpose, please increase it as per cluster size\n",
+        "    \"experiment_timeout_minutes\": 10, # This is a time limit for testing purposes, remove it for real use cases, this will drastically limit ablity to find the best model possible\n",
+        "    \"verbosity\": logging.INFO,\n",
+        "}\n",
+        "\n",
+        "automl_config = AutoMLConfig(task = 'classification',\n",
+        "                             debug_log = 'automl_errors.log',\n",
+        "                             compute_target = compute_target,\n",
+        "                             training_data = training_data,\n",
+        "                             label_column_name = label_column_name,\n",
+        "                             **automl_settings\n",
+        "                            )"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Call the `submit` method on the experiment object and pass the run configuration. Depending on the data and the number of iterations this can run for a while."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "remote_run = experiment.submit(automl_config, show_output = False)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# If you need to retrieve a run that already started, use the following code\n",
+        "#from azureml.train.automl.run import AutoMLRun\n",
+        "#remote_run = AutoMLRun(experiment = experiment, run_id = '<replace with your run id>')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "remote_run"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Results"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Widget for Monitoring Runs\n",
+        "\n",
+        "The widget will first report a \"loading\" status while running the first iteration. After completing the first iteration, an auto-updating graph and table will be shown. The widget will refresh once per minute, so you should see the graph update as child runs complete.\n",
+        "\n",
+        "**Note:** The widget displays a link at the bottom. Use this link to open a web interface to explore the individual run details"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.widgets import RunDetails\n",
+        "RunDetails(remote_run).show()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "remote_run.wait_for_completion(show_output=False)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Explain model\n",
+        "\n",
+        "Automated ML models can be explained and visualized using the SDK Explainability library. [Learn how to use the explainer](https://github.com/Azure/MachineLearningNotebooks/blob/master/how-to-use-azureml/automated-machine-learning/model-explanation-remote-amlcompute/auto-ml-model-explanations-remote-compute.ipynb)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Analyze results\n",
+        "\n",
+        "### Retrieve the Best Model\n",
+        "\n",
+        "Below we select the best pipeline from our iterations. The `get_output` method on `automl_classifier` returns the best run and the fitted model for the last invocation. Overloads on `get_output` allow you to retrieve the best run and fitted model for *any* logged metric or for a particular *iteration*."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "best_run, fitted_model = remote_run.get_output()\n",
+        "fitted_model"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Print the properties of the model\n",
+        "The fitted_model is a python object and you can read the different properties of the object.\n",
+        "See *Print the properties of the model* section in [this sample notebook](https://github.com/Azure/MachineLearningNotebooks/blob/master/how-to-use-azureml/automated-machine-learning/classification/auto-ml-classification.ipynb)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Deploy\n",
+        "\n",
+        "To deploy the model into a web service endpoint, see _Deploy_ section in [this sample notebook](https://github.com/Azure/MachineLearningNotebooks/blob/master/how-to-use-azureml/automated-machine-learning/classification-with-deployment/auto-ml-classification-with-deployment.ipynb)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Test the fitted model\n",
+        "\n",
+        "Now that the model is trained, split the data in the same way the data was split for training (The difference here is the data is being split locally) and then run the test data through the trained model to get the predicted values."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# convert the test data to dataframe\n",
+        "X_test_df = validation_data.drop_columns(columns=[label_column_name]).to_pandas_dataframe()\n",
+        "y_test_df = validation_data.keep_columns(columns=[label_column_name], validate=True).to_pandas_dataframe()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# call the predict functions on the model\n",
+        "y_pred = fitted_model.predict(X_test_df)\n",
+        "y_pred"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Calculate metrics for the prediction\n",
+        "\n",
+        "Now visualize the data on a scatter plot to show what our truth (actual) values are compared to the predicted values \n",
+        "from the trained model that was returned."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from sklearn.metrics import confusion_matrix\n",
+        "import numpy as np\n",
+        "import itertools\n",
+        "\n",
+        "cf =confusion_matrix(y_test_df.values,y_pred)\n",
+        "plt.imshow(cf,cmap=plt.cm.Blues,interpolation='nearest')\n",
+        "plt.colorbar()\n",
+        "plt.title('Confusion Matrix')\n",
+        "plt.xlabel('Predicted')\n",
+        "plt.ylabel('Actual')\n",
+        "class_labels = ['False','True']\n",
+        "tick_marks = np.arange(len(class_labels))\n",
+        "plt.xticks(tick_marks,class_labels)\n",
+        "plt.yticks([-0.5,0,1,1.5],['','False','True',''])\n",
+        "# plotting text value inside cells\n",
+        "thresh = cf.max() / 2.\n",
+        "for i,j in itertools.product(range(cf.shape[0]),range(cf.shape[1])):\n",
+        "    plt.text(j,i,format(cf[i,j],'d'),horizontalalignment='center',color='white' if cf[i,j] >thresh else 'black')\n",
+        "plt.show()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Acknowledgements"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "This Credit Card fraud Detection dataset is made available under the Open Database License: http://opendatacommons.org/licenses/odbl/1.0/. Any rights in individual contents of the database are licensed under the Database Contents License: http://opendatacommons.org/licenses/dbcl/1.0/ and is available at: https://www.kaggle.com/mlg-ulb/creditcardfraud\n",
+        "\n",
+        "\n",
+        "The dataset has been collected and analysed during a research collaboration of Worldline and the Machine Learning Group (http://mlg.ulb.ac.be) of ULB (Universit\u00c3\u0192\u00c2\u00a9 Libre de Bruxelles) on big data mining and fraud detection. More details on current and past projects on related topics are available on https://www.researchgate.net/project/Fraud-detection-5 and the page of the DefeatFraud project\n",
+        "Please cite the following works: \n",
+        "\u00c3\u00a2\u00e2\u201a\u00ac\u00c2\u00a2\tAndrea Dal Pozzolo, Olivier Caelen, Reid A. Johnson and Gianluca Bontempi. Calibrating Probability with Undersampling for Unbalanced Classification. In Symposium on Computational Intelligence and Data Mining (CIDM), IEEE, 2015\n",
+        "\u00c3\u00a2\u00e2\u201a\u00ac\u00c2\u00a2\tDal Pozzolo, Andrea; Caelen, Olivier; Le Borgne, Yann-Ael; Waterschoot, Serge; Bontempi, Gianluca. Learned lessons in credit card fraud detection from a practitioner perspective, Expert systems with applications,41,10,4915-4928,2014, Pergamon\n",
+        "\u00c3\u00a2\u00e2\u201a\u00ac\u00c2\u00a2\tDal Pozzolo, Andrea; Boracchi, Giacomo; Caelen, Olivier; Alippi, Cesare; Bontempi, Gianluca. Credit card fraud detection: a realistic modeling and a novel learning strategy, IEEE transactions on neural networks and learning systems,29,8,3784-3797,2018,IEEE\n",
+        "o\tDal Pozzolo, Andrea Adaptive Machine learning for credit card fraud detection ULB MLG PhD thesis (supervised by G. Bontempi)\n",
+        "\u00c3\u00a2\u00e2\u201a\u00ac\u00c2\u00a2\tCarcillo, Fabrizio; Dal Pozzolo, Andrea; Le Borgne, Yann-A\u00c3\u0192\u00c2\u00abl; Caelen, Olivier; Mazzer, Yannis; Bontempi, Gianluca. Scarff: a scalable framework for streaming credit card fraud detection with Spark, Information fusion,41, 182-194,2018,Elsevier\n",
+        "\u00c3\u00a2\u00e2\u201a\u00ac\u00c2\u00a2\tCarcillo, Fabrizio; Le Borgne, Yann-A\u00c3\u0192\u00c2\u00abl; Caelen, Olivier; Bontempi, Gianluca. Streaming active learning strategies for real-life credit card fraud detection: assessment and visualization, International Journal of Data Science and Analytics, 5,4,285-300,2018,Springer International Publishing"
+      ]
+    }
+  ],
+  "metadata": {
+    "authors": [
+      {
+        "name": "tzvikei"
+      }
+    ],
+    "category": "tutorial",
+    "compute": [
+      "AML Compute"
+    ],
+    "datasets": [
+      "Creditcard"
+    ],
+    "deployment": [
+      "None"
+    ],
+    "exclude_from_index": false,
+    "file_extension": ".py",
+    "framework": [
+      "None"
+    ],
+    "friendly_name": "Classification of credit card fraudulent transactions using Automated ML",
+    "index_order": 5,
+    "kernelspec": {
+      "display_name": "Python 3.6",
+      "language": "python",
+      "name": "python36"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.6.7"
+    },
+    "mimetype": "text/x-python",
+    "name": "python",
+    "nbconvert_exporter": "python",
+    "pygments_lexer": "ipython3",
+    "tags": [
+      "remote_run",
+      "AutomatedML"
+    ],
+    "task": "Classification",
+    "version": "3.6.7"
+  },
+  "nbformat": 4,
+  "nbformat_minor": 2
+}

how-to-use-azureml/automated-machine-learning/classification-credit-card-fraud/auto-ml-classification-credit-card-fraud.yml

@@ -0,0 +1,11 @@
+name: auto-ml-classification-credit-card-fraud
+dependencies:
+- pip:
+  - azureml-sdk
+  - interpret
+  - azureml-defaults
+  - azureml-explain-model
+  - azureml-train-automl
+  - azureml-widgets
+  - matplotlib
+  - pandas_ml

how-to-use-azureml/automated-machine-learning/classification-text-dnn/auto-ml-classification-text-dnn.ipynb

@@ -0,0 +1,561 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
+        "\n",
+        "Licensed under the MIT License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/automated-machine-learning/classification-text-dnn/auto-ml-classification-text-dnn.png)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# Automated Machine Learning\n",
+        "_**Text Classification Using Deep Learning**_\n",
+        "\n",
+        "## Contents\n",
+        "1. [Introduction](#Introduction)\n",
+        "1. [Setup](#Setup)\n",
+        "1. [Data](#Data)\n",
+        "1. [Train](#Train)\n",
+        "1. [Evaluate](#Evaluate)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Introduction\n",
+        "This notebook demonstrates classification with text data using deep learning in AutoML.\n",
+        "\n",
+        "AutoML highlights here include using deep neural networks (DNNs) to create embedded features from text data. Depending on the compute cluster the user provides, AutoML tried out Bidirectional Encoder Representations from Transformers (BERT) when a GPU compute is used, and Bidirectional Long-Short Term neural network (BiLSTM) when a CPU compute is used, thereby optimizing the choice of DNN for the uesr's setup.\n",
+        "\n",
+        "Make sure you have executed the [configuration](../../../configuration.ipynb) before running this notebook.\n",
+        "\n",
+        "An Enterprise workspace is required for this notebook. To learn more about creating an Enterprise workspace or upgrading to an Enterprise workspace from the Azure portal, please visit our [Workspace page](https://docs.microsoft.com/azure/machine-learning/service/concept-workspace#upgrade).\n",
+        "\n",
+        "Notebook synopsis:\n",
+        "1. Creating an Experiment in an existing Workspace\n",
+        "2. Configuration and remote run of AutoML for a text dataset (20 Newsgroups dataset from scikit-learn) for classification\n",
+        "3. Evaluating the final model on a test set\n",
+        "4. Deploying the model on ACI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import logging\n",
+        "import os\n",
+        "import shutil\n",
+        "\n",
+        "import pandas as pd\n",
+        "\n",
+        "import azureml.core\n",
+        "from azureml.core.experiment import Experiment\n",
+        "from azureml.core.workspace import Workspace\n",
+        "from azureml.core.dataset import Dataset\n",
+        "from azureml.core.compute import AmlCompute\n",
+        "from azureml.core.compute import ComputeTarget\n",
+        "from azureml.core.run import Run\n",
+        "from azureml.widgets import RunDetails\n",
+        "from azureml.core.model import Model \n",
+        "from helper import run_inference, get_result_df\n",
+        "from azureml.train.automl import AutoMLConfig\n",
+        "from sklearn.datasets import fetch_20newsgroups"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "As part of the setup you have already created a <b>Workspace</b>. To run AutoML, you also need to create an <b>Experiment</b>. An Experiment corresponds to a prediction problem you are trying to solve, while a Run corresponds to a specific approach to the problem."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "ws = Workspace.from_config()\n",
+        "\n",
+        "# Choose an experiment name.\n",
+        "experiment_name = 'automl-classification-text-dnn'\n",
+        "\n",
+        "experiment = Experiment(ws, experiment_name)\n",
+        "\n",
+        "output = {}\n",
+        "output['SDK version'] = azureml.core.VERSION\n",
+        "output['Subscription ID'] = ws.subscription_id\n",
+        "output['Workspace Name'] = ws.name\n",
+        "output['Resource Group'] = ws.resource_group\n",
+        "output['Location'] = ws.location\n",
+        "output['Experiment Name'] = experiment.name\n",
+        "pd.set_option('display.max_colwidth', -1)\n",
+        "outputDf = pd.DataFrame(data = output, index = [''])\n",
+        "outputDf.T"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Set up a compute cluster\n",
+        "This section uses a user-provided compute cluster (named \"cpu-cluster\" in this example). If a cluster with this name does not exist in the user's workspace, the below code will create a new cluster. You can choose the parameters of the cluster as mentioned in the comments.\n",
+        "\n",
+        "Whether you provide/select a CPU or GPU cluster, AutoML will choose the appropriate DNN for that setup - BiLSTM or BERT text featurizer will be included in the candidate featurizers on CPU and GPU respectively."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Choose a name for your cluster.\n",
+        "amlcompute_cluster_name = \"cpu-dnntext\"\n",
+        "\n",
+        "found = False\n",
+        "# Check if this compute target already exists in the workspace.\n",
+        "cts = ws.compute_targets\n",
+        "if amlcompute_cluster_name in cts and cts[amlcompute_cluster_name].type == 'AmlCompute':\n",
+        "    found = True\n",
+        "    print('Found existing compute target.')\n",
+        "    compute_target = cts[amlcompute_cluster_name]\n",
+        "\n",
+        "if not found:\n",
+        "    print('Creating a new compute target...')\n",
+        "    provisioning_config = AmlCompute.provisioning_configuration(vm_size = \"STANDARD_D2_V2\", # CPU for BiLSTM\n",
+        "                                                                # To use BERT, select a GPU such as \"STANDARD_NC6\" \n",
+        "                                                                # or similar GPU option\n",
+        "                                                                # available in your workspace\n",
+        "                                                                max_nodes = 6)\n",
+        "\n",
+        "    # Create the cluster\n",
+        "    compute_target = ComputeTarget.create(ws, amlcompute_cluster_name, provisioning_config)\n",
+        "\n",
+        "print('Checking cluster status...')\n",
+        "# Can poll for a minimum number of nodes and for a specific timeout.\n",
+        "# If no min_node_count is provided, it will use the scale settings for the cluster.\n",
+        "compute_target.wait_for_completion(show_output = True, min_node_count = None, timeout_in_minutes = 20)\n",
+        "\n",
+        "# For a more detailed view of current AmlCompute status, use get_status()."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Get data\n",
+        "For this notebook we will use 20 Newsgroups data from scikit-learn. We filter the data to contain four classes and take a sample as training data. Please note that for accuracy improvement, more data is needed. For this notebook we provide a small-data example so that you can use this template to use with your larger sized data."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "data_dir = \"text-dnn-data\" # Local directory to store data\n",
+        "blobstore_datadir = data_dir # Blob store directory to store data in\n",
+        "target_column_name = 'y'\n",
+        "feature_column_name = 'X'\n",
+        "\n",
+        "def get_20newsgroups_data():\n",
+        "    '''Fetches 20 Newsgroups data from scikit-learn\n",
+        "       Returns them in form of pandas dataframes\n",
+        "    '''\n",
+        "    remove = ('headers', 'footers', 'quotes')\n",
+        "    categories = [\n",
+        "        'alt.atheism',\n",
+        "        'talk.religion.misc',\n",
+        "        'comp.graphics',\n",
+        "        'sci.space',\n",
+        "        ]\n",
+        "\n",
+        "    data = fetch_20newsgroups(subset = 'train', categories = categories,\n",
+        "                                    shuffle = True, random_state = 42,\n",
+        "                                    remove = remove)\n",
+        "    data = pd.DataFrame({feature_column_name: data.data, target_column_name: data.target})\n",
+        "\n",
+        "    data_train = data[:200]\n",
+        "    data_test = data[200:300]    \n",
+        "\n",
+        "    data_train = remove_blanks_20news(data_train, feature_column_name, target_column_name)\n",
+        "    data_test = remove_blanks_20news(data_test, feature_column_name, target_column_name)\n",
+        "    \n",
+        "    return data_train, data_test\n",
+        "    \n",
+        "def remove_blanks_20news(data, feature_column_name, target_column_name):\n",
+        "    \n",
+        "    data[feature_column_name] = data[feature_column_name].replace(r'\\n', ' ', regex=True).apply(lambda x: x.strip())\n",
+        "    data = data[data[feature_column_name] != '']\n",
+        "    \n",
+        "    return data"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Featch data and upload to datastore for use in training"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "data_train, data_test = get_20newsgroups_data()\n",
+        "\n",
+        "if not os.path.isdir(data_dir):\n",
+        "    os.mkdir(data_dir)\n",
+        "    \n",
+        "train_data_fname = data_dir + '/train_data.csv'\n",
+        "test_data_fname = data_dir + '/test_data.csv'\n",
+        "\n",
+        "data_train.to_csv(train_data_fname, index=False)\n",
+        "data_test.to_csv(test_data_fname, index=False)\n",
+        "\n",
+        "datastore = ws.get_default_datastore()\n",
+        "datastore.upload(src_dir=data_dir, target_path=blobstore_datadir,\n",
+        "                    overwrite=True)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "train_dataset = Dataset.Tabular.from_delimited_files(path = [(datastore, blobstore_datadir + '/train_data.csv')])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Prepare AutoML run"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "This step requires an Enterprise workspace to gain access to this feature. To learn more about creating an Enterprise workspace or upgrading to an Enterprise workspace from the Azure portal, please visit our [Workspace page](https://docs.microsoft.com/azure/machine-learning/service/concept-workspace#upgrade)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "automl_settings = {\n",
+        "    \"experiment_timeout_minutes\": 20,\n",
+        "    \"primary_metric\": 'accuracy',\n",
+        "    \"preprocess\": True,\n",
+        "    \"max_concurrent_iterations\": 4, \n",
+        "    \"max_cores_per_iteration\": -1,\n",
+        "    \"enable_dnn\": True,\n",
+        "    \"enable_early_stopping\": True,\n",
+        "    \"validation_size\": 0.3,\n",
+        "    \"verbosity\": logging.INFO,\n",
+        "    \"enable_voting_ensemble\": False,\n",
+        "    \"enable_stack_ensemble\": False,\n",
+        "}\n",
+        "\n",
+        "automl_config = AutoMLConfig(task = 'classification',\n",
+        "                             debug_log = 'automl_errors.log',\n",
+        "                             compute_target=compute_target,\n",
+        "                             training_data=train_dataset,\n",
+        "                             label_column_name=target_column_name,\n",
+        "                             **automl_settings\n",
+        "                            )"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Submit AutoML Run"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "automl_run = experiment.submit(automl_config, show_output=True)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "automl_run"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Displaying the run objects gives you links to the visual tools in the Azure Portal. Go try them!"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Retrieve the Best Model\n",
+        "Below we select the best model pipeline from our iterations, use it to test on test data on the same compute cluster."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "You can test the model locally to get a feel of the input/output. This step may require additional package installations such as pytorch."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "#best_run, fitted_model = automl_run.get_output()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Deploying the model\n",
+        "We now use the best fitted model from the AutoML Run to make predictions on the test set.  "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Get results stats, extract the best model from AutoML run, download and register the resultant best model"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "summary_df = get_result_df(automl_run)\n",
+        "best_dnn_run_id = summary_df['run_id'].iloc[0]\n",
+        "best_dnn_run = Run(experiment, best_dnn_run_id)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "model_dir = 'Model' # Local folder where the model will be stored temporarily\n",
+        "if not os.path.isdir(model_dir):\n",
+        "    os.mkdir(model_dir)\n",
+        "    \n",
+        "best_dnn_run.download_file('outputs/model.pkl', model_dir + '/model.pkl')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Register the model in your Azure Machine Learning Workspace. If you previously registered a model, please make sure to delete it so as to replace it with this new model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Register the model\n",
+        "model_name = 'textDNN-20News'\n",
+        "model = Model.register(model_path = model_dir + '/model.pkl',\n",
+        "                       model_name = model_name,\n",
+        "                       tags=None,\n",
+        "                       workspace=ws)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Evaluate on Test Data"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "We now use the best fitted model from the AutoML Run to make predictions on the test set.  \n",
+        "\n",
+        "Test set schema should match that of the training set."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "test_dataset = Dataset.Tabular.from_delimited_files(path = [(datastore, blobstore_datadir + '/test_data.csv')])\n",
+        "\n",
+        "# preview the first 3 rows of the dataset\n",
+        "test_dataset.take(3).to_pandas_dataframe()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "test_experiment = Experiment(ws, experiment_name + \"_test\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "script_folder = os.path.join(os.getcwd(), 'inference')\n",
+        "os.makedirs(script_folder, exist_ok=True)\n",
+        "shutil.copy2('infer.py', script_folder)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "test_run = run_inference(test_experiment, compute_target, script_folder, best_dnn_run, test_dataset,\n",
+        "                 target_column_name, model_name)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Display computed metrics"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "test_run"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "RunDetails(test_run).show()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "test_run.wait_for_completion()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "pd.Series(test_run.get_metrics())"
+      ]
+    }
+  ],
+  "metadata": {
+    "authors": [
+      {
+        "name": "anshirga"
+      }
+    ],
+    "datasets": [
+      "None"
+    ],
+    "compute": [
+      "AML Compute"
+    ],
+    "deployment": [
+      "None"
+    ],
+    "exclude_from_index": false,
+    "framework": [
+      "None"
+    ],
+    "friendly_name": "DNN Text Featurization",
+    "index_order": 2,
+    "kernelspec": {
+      "display_name": "Python 3.6",
+      "language": "python",
+      "name": "python36"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.6.7"
+    },
+    "tags": [
+      "None"
+    ],
+    "task": "Text featurization using DNNs for classification"
+  },
+  "nbformat": 4,
+  "nbformat_minor": 2
+}

how-to-use-azureml/automated-machine-learning/classification-text-dnn/auto-ml-classification-text-dnn.yml

@@ -0,0 +1,10 @@
+name: auto-ml-classification-text-dnn
+dependencies:
+- pip:
+  - azureml-sdk
+  - azureml-train-automl
+  - azureml-train
+  - azureml-widgets
+  - matplotlib
+  - pandas_ml
+  - statsmodels

how-to-use-azureml/automated-machine-learning/classification-text-dnn/helper.py

@@ -0,0 +1,60 @@
+import pandas as pd
+from azureml.core import Environment
+from azureml.core.conda_dependencies import CondaDependencies
+from azureml.train.estimator import Estimator
+from azureml.core.run import Run
+
+
+def run_inference(test_experiment, compute_target, script_folder, train_run,
+                  test_dataset, target_column_name, model_name):
+
+    train_run.download_file('outputs/conda_env_v_1_0_0.yml',
+                            'inference/condafile.yml')
+
+    inference_env = Environment("myenv")
+    inference_env.docker.enabled = True
+    inference_env.python.conda_dependencies = CondaDependencies(
+        conda_dependencies_file_path='inference/condafile.yml')
+
+    est = Estimator(source_directory=script_folder,
+                    entry_script='infer.py',
+                    script_params={
+                        '--target_column_name': target_column_name,
+                        '--model_name': model_name
+                    },
+                    inputs=[test_dataset.as_named_input('test_data')],
+                    compute_target=compute_target,
+                    environment_definition=inference_env)
+
+    run = test_experiment.submit(
+        est, tags={
+            'training_run_id': train_run.id,
+            'run_algorithm': train_run.properties['run_algorithm'],
+            'valid_score': train_run.properties['score'],
+            'primary_metric': train_run.properties['primary_metric']
+        })
+
+    run.log("run_algorithm", run.tags['run_algorithm'])
+    return run
+
+
+def get_result_df(remote_run):
+
+    children = list(remote_run.get_children(recursive=True))
+    summary_df = pd.DataFrame(index=['run_id', 'run_algorithm',
+                                     'primary_metric', 'Score'])
+    goal_minimize = False
+    for run in children:
+        if('run_algorithm' in run.properties and 'score' in run.properties):
+            summary_df[run.id] = [run.id, run.properties['run_algorithm'],
+                                  run.properties['primary_metric'],
+                                  float(run.properties['score'])]
+            if('goal' in run.properties):
+                goal_minimize = run.properties['goal'].split('_')[-1] == 'min'
+
+    summary_df = summary_df.T.sort_values(
+        'Score',
+        ascending=goal_minimize).drop_duplicates(['run_algorithm'])
+    summary_df = summary_df.set_index('run_algorithm')
+
+    return summary_df

how-to-use-azureml/automated-machine-learning/classification-text-dnn/infer.py

@@ -0,0 +1,54 @@
+import numpy as np
+import argparse
+from azureml.core import Run
+from sklearn.externals import joblib
+from azureml.automl.core._vendor.automl.client.core.common import metrics
+from automl.client.core.common import constants
+from azureml.core.model import Model
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    '--target_column_name', type=str, dest='target_column_name',
+    help='Target Column Name')
+parser.add_argument(
+    '--model_name', type=str, dest='model_name',
+    help='Name of registered model')
+
+args = parser.parse_args()
+target_column_name = args.target_column_name
+model_name = args.model_name
+
+print('args passed are: ')
+print('Target column name: ', target_column_name)
+print('Name of registered model: ', model_name)
+
+model_path = Model.get_model_path(model_name)
+# deserialize the model file back into a sklearn model
+model = joblib.load(model_path)
+
+run = Run.get_context()
+# get input dataset by name
+test_dataset = run.input_datasets['test_data']
+
+X_test_df = test_dataset.drop_columns(columns=[target_column_name]) \
+                        .to_pandas_dataframe()
+y_test_df = test_dataset.with_timestamp_columns(None) \
+                        .keep_columns(columns=[target_column_name]) \
+                        .to_pandas_dataframe()
+
+predicted = model.predict_proba(X_test_df)
+
+# use automl metrics module
+scores = metrics.compute_metrics_classification(
+    np.array(predicted),
+    np.array(y_test_df),
+    class_labels=model.classes_,
+    metrics=list(constants.Metric.SCALAR_CLASSIFICATION_SET)
+)
+
+print("scores:")
+print(scores)
+
+for key, value in scores.items():
+    run.log(key, value)

how-to-use-azureml/automated-machine-learning/classification-with-deployment/auto-ml-classification-with-deployment.ipynb

@@ -1,502 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
-        "\n",
-        "Licensed under the MIT License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "# Automated Machine Learning\n",
-        "_**Classification with Deployment**_\n",
-        "\n",
-        "## Contents\n",
-        "1. [Introduction](#Introduction)\n",
-        "1. [Setup](#Setup)\n",
-        "1. [Train](#Train)\n",
-        "1. [Deploy](#Deploy)\n",
-        "1. [Test](#Test)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Introduction\n",
-        "\n",
-        "In this example we use the scikit learn's [digit dataset](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_digits.html) to showcase how you can use AutoML for a simple classification problem and deploy it to an Azure Container Instance (ACI).\n",
-        "\n",
-        "Make sure you have executed the [configuration](../../../configuration.ipynb) before running this notebook.\n",
-        "\n",
-        "In this notebook you will learn how to:\n",
-        "1. Create an experiment using an existing workspace.\n",
-        "2. Configure AutoML using `AutoMLConfig`.\n",
-        "3. Train the model using local compute.\n",
-        "4. Explore the results.\n",
-        "5. Register the model.\n",
-        "6. Create a container image.\n",
-        "7. Create an Azure Container Instance (ACI) service.\n",
-        "8. Test the ACI service."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Setup\n",
-        "\n",
-        "As part of the setup you have already created an Azure ML `Workspace` object. For AutoML you will need to create an `Experiment` object, which is a named object in a `Workspace` used to run experiments."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "import json\n",
-        "import logging\n",
-        "\n",
-        "from matplotlib import pyplot as plt\n",
-        "import numpy as np\n",
-        "import pandas as pd\n",
-        "from sklearn import datasets\n",
-        "\n",
-        "import azureml.core\n",
-        "from azureml.core.experiment import Experiment\n",
-        "from azureml.core.workspace import Workspace\n",
-        "from azureml.train.automl import AutoMLConfig\n",
-        "from azureml.train.automl.run import AutoMLRun"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "ws = Workspace.from_config()\n",
-        "\n",
-        "# choose a name for experiment\n",
-        "experiment_name = 'automl-classification-deployment'\n",
-        "# project folder\n",
-        "project_folder = './sample_projects/automl-classification-deployment'\n",
-        "\n",
-        "experiment=Experiment(ws, experiment_name)\n",
-        "\n",
-        "output = {}\n",
-        "output['SDK version'] = azureml.core.VERSION\n",
-        "output['Subscription ID'] = ws.subscription_id\n",
-        "output['Workspace'] = ws.name\n",
-        "output['Resource Group'] = ws.resource_group\n",
-        "output['Location'] = ws.location\n",
-        "output['Project Directory'] = project_folder\n",
-        "output['Experiment Name'] = experiment.name\n",
-        "pd.set_option('display.max_colwidth', -1)\n",
-        "outputDf = pd.DataFrame(data = output, index = [''])\n",
-        "outputDf.T"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Train\n",
-        "\n",
-        "Instantiate a AutoMLConfig object. This defines the settings and data used to run the experiment.\n",
-        "\n",
-        "|Property|Description|\n",
-        "|-|-|\n",
-        "|**task**|classification or regression|\n",
-        "|**primary_metric**|This is the metric that you want to optimize. Classification supports the following primary metrics: <br><i>accuracy</i><br><i>AUC_weighted</i><br><i>average_precision_score_weighted</i><br><i>norm_macro_recall</i><br><i>precision_score_weighted</i>|\n",
-        "|**iteration_timeout_minutes**|Time limit in minutes for each iteration.|\n",
-        "|**iterations**|Number of iterations. In each iteration AutoML trains a specific pipeline with the data.|\n",
-        "|**n_cross_validations**|Number of cross validation splits.|\n",
-        "|**X**|(sparse) array-like, shape = [n_samples, n_features]|\n",
-        "|**y**|(sparse) array-like, shape = [n_samples, ], Multi-class targets.|\n",
-        "|**path**|Relative path to the project folder. AutoML stores configuration files for the experiment under this folder. You can specify a new empty folder.|"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "digits = datasets.load_digits()\n",
-        "X_train = digits.data[10:,:]\n",
-        "y_train = digits.target[10:]\n",
-        "\n",
-        "automl_config = AutoMLConfig(task = 'classification',\n",
-        "                             name = experiment_name,\n",
-        "                             debug_log = 'automl_errors.log',\n",
-        "                             primary_metric = 'AUC_weighted',\n",
-        "                             iteration_timeout_minutes = 20,\n",
-        "                             iterations = 10,\n",
-        "                             verbosity = logging.INFO,\n",
-        "                             X = X_train, \n",
-        "                             y = y_train,\n",
-        "                             path = project_folder)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "Call the `submit` method on the experiment object and pass the run configuration. Execution of local runs is synchronous. Depending on the data and the number of iterations this can run for a while.\n",
-        "In this example, we specify `show_output = True` to print currently running iterations to the console."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "local_run = experiment.submit(automl_config, show_output = True)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "local_run"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Deploy\n",
-        "\n",
-        "### Retrieve the Best Model\n",
-        "\n",
-        "Below we select the best pipeline from our iterations. The `get_output` method on `automl_classifier` returns the best run and the fitted model for the last invocation. Overloads on `get_output` allow you to retrieve the best run and fitted model for *any* logged metric or for a particular *iteration*."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "best_run, fitted_model = local_run.get_output()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Register the Fitted Model for Deployment\n",
-        "If neither `metric` nor `iteration` are specified in the `register_model` call, the iteration with the best primary metric is registered."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "description = 'AutoML Model'\n",
-        "tags = None\n",
-        "model = local_run.register_model(description = description, tags = tags)\n",
-        "\n",
-        "print(local_run.model_id) # This will be written to the script file later in the notebook."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Create Scoring Script"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "%%writefile score.py\n",
-        "import pickle\n",
-        "import json\n",
-        "import numpy\n",
-        "import azureml.train.automl\n",
-        "from sklearn.externals import joblib\n",
-        "from azureml.core.model import Model\n",
-        "\n",
-        "\n",
-        "def init():\n",
-        "    global model\n",
-        "    model_path = Model.get_model_path(model_name = '<<modelid>>') # this name is model.id of model that we want to deploy\n",
-        "    # deserialize the model file back into a sklearn model\n",
-        "    model = joblib.load(model_path)\n",
-        "\n",
-        "def run(rawdata):\n",
-        "    try:\n",
-        "        data = json.loads(rawdata)['data']\n",
-        "        data = numpy.array(data)\n",
-        "        result = model.predict(data)\n",
-        "    except Exception as e:\n",
-        "        result = str(e)\n",
-        "        return json.dumps({\"error\": result})\n",
-        "    return json.dumps({\"result\":result.tolist()})"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Create a YAML File for the Environment"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "To ensure the fit results are consistent with the training results, the SDK dependency versions need to be the same as the environment that trains the model. The following cells create a file, myenv.yml, which specifies the dependencies from the run."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "experiment = Experiment(ws, experiment_name)\n",
-        "ml_run = AutoMLRun(experiment = experiment, run_id = local_run.id)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "dependencies = ml_run.get_run_sdk_dependencies(iteration = 7)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "for p in ['azureml-train-automl', 'azureml-sdk', 'azureml-core']:\n",
-        "    print('{}\\t{}'.format(p, dependencies[p]))"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "from azureml.core.conda_dependencies import CondaDependencies\n",
-        "\n",
-        "myenv = CondaDependencies.create(conda_packages=['numpy','scikit-learn'], pip_packages=['azureml-sdk[automl]'])\n",
-        "\n",
-        "conda_env_file_name = 'myenv.yml'\n",
-        "myenv.save_to_file('.', conda_env_file_name)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "# Substitute the actual version number in the environment file.\n",
-        "# This is not strictly needed in this notebook because the model should have been generated using the current SDK version.\n",
-        "# However, we include this in case this code is used on an experiment from a previous SDK version.\n",
-        "\n",
-        "with open(conda_env_file_name, 'r') as cefr:\n",
-        "    content = cefr.read()\n",
-        "\n",
-        "with open(conda_env_file_name, 'w') as cefw:\n",
-        "    cefw.write(content.replace(azureml.core.VERSION, dependencies['azureml-sdk']))\n",
-        "\n",
-        "# Substitute the actual model id in the script file.\n",
-        "\n",
-        "script_file_name = 'score.py'\n",
-        "\n",
-        "with open(script_file_name, 'r') as cefr:\n",
-        "    content = cefr.read()\n",
-        "\n",
-        "with open(script_file_name, 'w') as cefw:\n",
-        "    cefw.write(content.replace('<<modelid>>', local_run.model_id))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Create a Container Image"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "from azureml.core.image import Image, ContainerImage\n",
-        "\n",
-        "image_config = ContainerImage.image_configuration(runtime= \"python\",\n",
-        "                                 execution_script = script_file_name,\n",
-        "                                 conda_file = conda_env_file_name,\n",
-        "                                 tags = {'area': \"digits\", 'type': \"automl_classification\"},\n",
-        "                                 description = \"Image for automl classification sample\")\n",
-        "\n",
-        "image = Image.create(name = \"automlsampleimage\",\n",
-        "                     # this is the model object \n",
-        "                     models = [model],\n",
-        "                     image_config = image_config, \n",
-        "                     workspace = ws)\n",
-        "\n",
-        "image.wait_for_creation(show_output = True)\n",
-        "\n",
-        "if image.creation_state == 'Failed':\n",
-        "    print(\"Image build log at: \" + image.image_build_log_uri)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Deploy the Image as a Web Service on Azure Container Instance"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "from azureml.core.webservice import AciWebservice\n",
-        "\n",
-        "aciconfig = AciWebservice.deploy_configuration(cpu_cores = 1, \n",
-        "                                               memory_gb = 1, \n",
-        "                                               tags = {'area': \"digits\", 'type': \"automl_classification\"}, \n",
-        "                                               description = 'sample service for Automl Classification')"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "from azureml.core.webservice import Webservice\n",
-        "\n",
-        "aci_service_name = 'automl-sample-01'\n",
-        "print(aci_service_name)\n",
-        "aci_service = Webservice.deploy_from_image(deployment_config = aciconfig,\n",
-        "                                           image = image,\n",
-        "                                           name = aci_service_name,\n",
-        "                                           workspace = ws)\n",
-        "aci_service.wait_for_deployment(True)\n",
-        "print(aci_service.state)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Delete a Web Service"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "#aci_service.delete()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Get Logs from a Deployed Web Service"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "#aci_service.get_logs()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Test"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "#Randomly select digits and test\n",
-        "digits = datasets.load_digits()\n",
-        "X_test = digits.data[:10, :]\n",
-        "y_test = digits.target[:10]\n",
-        "images = digits.images[:10]\n",
-        "\n",
-        "for index in np.random.choice(len(y_test), 3, replace = False):\n",
-        "    print(index)\n",
-        "    test_sample = json.dumps({'data':X_test[index:index + 1].tolist()})\n",
-        "    predicted = aci_service.run(input_data = test_sample)\n",
-        "    label = y_test[index]\n",
-        "    predictedDict = json.loads(predicted)\n",
-        "    title = \"Label value = %d  Predicted value = %s \" % ( label,predictedDict['result'][0])\n",
-        "    fig = plt.figure(1, figsize = (3,3))\n",
-        "    ax1 = fig.add_axes((0,0,.8,.8))\n",
-        "    ax1.set_title(title)\n",
-        "    plt.imshow(images[index], cmap = plt.cm.gray_r, interpolation = 'nearest')\n",
-        "    plt.show()"
-      ]
-    }
-  ],
-  "metadata": {
-    "authors": [
-      {
-        "name": "savitam"
-      }
-    ],
-    "kernelspec": {
-      "display_name": "Python 3.6",
-      "language": "python",
-      "name": "python36"
-    },
-    "language_info": {
-      "codemirror_mode": {
-        "name": "ipython",
-        "version": 3
-      },
-      "file_extension": ".py",
-      "mimetype": "text/x-python",
-      "name": "python",
-      "nbconvert_exporter": "python",
-      "pygments_lexer": "ipython3",
-      "version": "3.6.6"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 2
-}

how-to-use-azureml/automated-machine-learning/classification-with-onnx/auto-ml-classification-with-onnx.ipynb

@@ -1,284 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
-        "\n",
-        "Licensed under the MIT License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "# Automated Machine Learning\n",
-        "_**Classification with Local Compute**_\n",
-        "\n",
-        "## Contents\n",
-        "1. [Introduction](#Introduction)\n",
-        "1. [Setup](#Setup)\n",
-        "1. [Data](#Data)\n",
-        "1. [Train](#Train)\n",
-        "1. [Results](#Results)\n",
-        "1. [Test](#Test)\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Introduction\n",
-        "\n",
-        "In this example we use the scikit-learn's [digit dataset](http://scikit-learn.org/stable/datasets/index.html#optical-recognition-of-handwritten-digits-dataset) to showcase how you can use AutoML for a simple classification problem.\n",
-        "\n",
-        "Make sure you have executed the [configuration](../../../configuration.ipynb) before running this notebook.\n",
-        "\n",
-        "Please find the ONNX related documentations [here](https://github.com/onnx/onnx).\n",
-        "\n",
-        "In this notebook you will learn how to:\n",
-        "1. Create an `Experiment` in an existing `Workspace`.\n",
-        "2. Configure AutoML using `AutoMLConfig`.\n",
-        "3. Train the model using local compute with ONNX compatible config on.\n",
-        "4. Explore the results and save the ONNX model."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Setup\n",
-        "\n",
-        "As part of the setup you have already created an Azure ML `Workspace` object. For AutoML you will need to create an `Experiment` object, which is a named object in a `Workspace` used to run experiments."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "import logging\n",
-        "\n",
-        "from matplotlib import pyplot as plt\n",
-        "import numpy as np\n",
-        "import pandas as pd\n",
-        "from sklearn import datasets\n",
-        "\n",
-        "import azureml.core\n",
-        "from azureml.core.experiment import Experiment\n",
-        "from azureml.core.workspace import Workspace\n",
-        "from azureml.train.automl import AutoMLConfig"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "ws = Workspace.from_config()\n",
-        "\n",
-        "# Choose a name for the experiment and specify the project folder.\n",
-        "experiment_name = 'automl-classification-onnx'\n",
-        "project_folder = './sample_projects/automl-classification-onnx'\n",
-        "\n",
-        "experiment = Experiment(ws, experiment_name)\n",
-        "\n",
-        "output = {}\n",
-        "output['SDK version'] = azureml.core.VERSION\n",
-        "output['Subscription ID'] = ws.subscription_id\n",
-        "output['Workspace Name'] = ws.name\n",
-        "output['Resource Group'] = ws.resource_group\n",
-        "output['Location'] = ws.location\n",
-        "output['Project Directory'] = project_folder\n",
-        "output['Experiment Name'] = experiment.name\n",
-        "pd.set_option('display.max_colwidth', -1)\n",
-        "outputDf = pd.DataFrame(data = output, index = [''])\n",
-        "outputDf.T"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Data\n",
-        "\n",
-        "This uses scikit-learn's [load_digits](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_digits.html) method."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "digits = datasets.load_digits()\n",
-        "\n",
-        "# Exclude the first 100 rows from training so that they can be used for test.\n",
-        "X_train = digits.data[100:,:]\n",
-        "y_train = digits.target[100:]"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Train with enable ONNX compatible models config on\n",
-        "\n",
-        "Instantiate an `AutoMLConfig` object to specify the settings and data used to run the experiment.\n",
-        "\n",
-        "Set the parameter enable_onnx_compatible_models=True, if you also want to generate the ONNX compatible models. Please note, the forecasting task and TensorFlow models are not ONNX compatible yet.\n",
-        "\n",
-        "|Property|Description|\n",
-        "|-|-|\n",
-        "|**task**|classification or regression|\n",
-        "|**primary_metric**|This is the metric that you want to optimize. Classification supports the following primary metrics: <br><i>accuracy</i><br><i>AUC_weighted</i><br><i>average_precision_score_weighted</i><br><i>norm_macro_recall</i><br><i>precision_score_weighted</i>|\n",
-        "|**iteration_timeout_minutes**|Time limit in minutes for each iteration.|\n",
-        "|**iterations**|Number of iterations. In each iteration AutoML trains a specific pipeline with the data.|\n",
-        "|**X**|(sparse) array-like, shape = [n_samples, n_features]|\n",
-        "|**y**|(sparse) array-like, shape = [n_samples, ], Multi-class targets.|\n",
-        "|**enable_onnx_compatible_models**|Enable the ONNX compatible models in the experiment.|\n",
-        "|**path**|Relative path to the project folder. AutoML stores configuration files for the experiment under this folder. You can specify a new empty folder.|"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "automl_config = AutoMLConfig(task = 'classification',\n",
-        "                             debug_log = 'automl_errors.log',\n",
-        "                             primary_metric = 'AUC_weighted',\n",
-        "                             iteration_timeout_minutes = 60,\n",
-        "                             iterations = 10,\n",
-        "                             verbosity = logging.INFO,\n",
-        "                             X = X_train, \n",
-        "                             y = y_train,\n",
-        "                             enable_onnx_compatible_models=True,\n",
-        "                             path = project_folder)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "Call the `submit` method on the experiment object and pass the run configuration. Execution of local runs is synchronous. Depending on the data and the number of iterations this can run for a while.\n",
-        "In this example, we specify `show_output = True` to print currently running iterations to the console."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "local_run = experiment.submit(automl_config, show_output = True)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "local_run"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Results"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Widget for Monitoring Runs\n",
-        "\n",
-        "The widget will first report a \"loading\" status while running the first iteration. After completing the first iteration, an auto-updating graph and table will be shown. The widget will refresh once per minute, so you should see the graph update as child runs complete.\n",
-        "\n",
-        "**Note:** The widget displays a link at the bottom. Use this link to open a web interface to explore the individual run details."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "from azureml.widgets import RunDetails\n",
-        "RunDetails(local_run).show() "
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Retrieve the Best ONNX Model\n",
-        "\n",
-        "Below we select the best pipeline from our iterations. The `get_output` method returns the best run and the fitted model. The Model includes the pipeline and any pre-processing.  Overloads on `get_output` allow you to retrieve the best run and fitted model for *any* logged metric or for a particular *iteration*.\n",
-        "\n",
-        "Set the parameter return_onnx_model=True to retrieve the best ONNX model, instead of the Python model."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "best_run, onnx_mdl = local_run.get_output(return_onnx_model=True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Save the best ONNX model"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "from azureml.train.automl._vendor.automl.client.core.common.onnx_convert import OnnxConverter\n",
-        "onnx_fl_path = \"./best_model.onnx\"\n",
-        "OnnxConverter.save_onnx_model(onnx_mdl, onnx_fl_path)"
-      ]
-    }
-  ],
-  "metadata": {
-    "authors": [
-      {
-        "name": "savitam"
-      }
-    ],
-    "kernelspec": {
-      "display_name": "Python 3.6",
-      "language": "python",
-      "name": "python36"
-    },
-    "language_info": {
-      "codemirror_mode": {
-        "name": "ipython",
-        "version": 3
-      },
-      "file_extension": ".py",
-      "mimetype": "text/x-python",
-      "name": "python",
-      "nbconvert_exporter": "python",
-      "pygments_lexer": "ipython3",
-      "version": "3.6.6"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 2
-}

how-to-use-azureml/automated-machine-learning/classification-with-whitelisting/auto-ml-classification-with-whitelisting.ipynb

@@ -1,392 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
-        "\n",
-        "Licensed under the MIT License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "# Automated Machine Learning\n",
-        "_**Classification using whitelist models**_\n",
-        "\n",
-        "## Contents\n",
-        "1. [Introduction](#Introduction)\n",
-        "1. [Setup](#Setup)\n",
-        "1. [Data](#Data)\n",
-        "1. [Train](#Train)\n",
-        "1. [Results](#Results)\n",
-        "1. [Test](#Test)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Introduction\n",
-        "\n",
-        "In this example we use the scikit-learn's [digit dataset](http://scikit-learn.org/stable/datasets/index.html#optical-recognition-of-handwritten-digits-dataset) to showcase how you can use AutoML for a simple classification problem.\n",
-        "\n",
-        "Make sure you have executed the [configuration](../../../configuration.ipynb) before running this notebook.\n",
-        "This notebooks shows how can automl can be trained on a a selected list of models,see the readme.md for the models.\n",
-        "This trains the model exclusively on tensorflow based models.\n",
-        "\n",
-        "In this notebook you will learn how to:\n",
-        "1. Create an `Experiment` in an existing `Workspace`.\n",
-        "2. Configure AutoML using `AutoMLConfig`.\n",
-        "3. Train the model on a whilelisted models using local compute. \n",
-        "4. Explore the results.\n",
-        "5. Test the best fitted model."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Setup\n",
-        "\n",
-        "As part of the setup you have already created an Azure ML `Workspace` object. For AutoML you will need to create an `Experiment` object, which is a named object in a `Workspace` used to run experiments."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "#Note: This notebook will install tensorflow if not already installed in the enviornment..\n",
-        "import logging\n",
-        "\n",
-        "from matplotlib import pyplot as plt\n",
-        "import numpy as np\n",
-        "import pandas as pd\n",
-        "from sklearn import datasets\n",
-        "\n",
-        "import azureml.core\n",
-        "from azureml.core.experiment import Experiment\n",
-        "from azureml.core.workspace import Workspace\n",
-        "import sys\n",
-        "whitelist_models=[\"LightGBM\"]\n",
-        "if \"3.7\" != sys.version[0:3]:\n",
-        "    try:\n",
-        "        import tensorflow as tf1\n",
-        "    except ImportError:\n",
-        "        from pip._internal import main\n",
-        "        main(['install', 'tensorflow>=1.10.0,<=1.12.0'])\n",
-        "        logging.getLogger().setLevel(logging.ERROR)\n",
-        "    whitelist_models=[\"TensorFlowLinearClassifier\", \"TensorFlowDNN\"]\n",
-        "\n",
-        "from azureml.train.automl import AutoMLConfig"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "ws = Workspace.from_config()\n",
-        "\n",
-        "# Choose a name for the experiment and specify the project folder.\n",
-        "experiment_name = 'automl-local-whitelist'\n",
-        "project_folder = './sample_projects/automl-local-whitelist'\n",
-        "\n",
-        "experiment = Experiment(ws, experiment_name)\n",
-        "\n",
-        "output = {}\n",
-        "output['SDK version'] = azureml.core.VERSION\n",
-        "output['Subscription ID'] = ws.subscription_id\n",
-        "output['Workspace Name'] = ws.name\n",
-        "output['Resource Group'] = ws.resource_group\n",
-        "output['Location'] = ws.location\n",
-        "output['Project Directory'] = project_folder\n",
-        "output['Experiment Name'] = experiment.name\n",
-        "pd.set_option('display.max_colwidth', -1)\n",
-        "outputDf = pd.DataFrame(data = output, index = [''])\n",
-        "outputDf.T"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Data\n",
-        "\n",
-        "This uses scikit-learn's [load_digits](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_digits.html) method."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "digits = datasets.load_digits()\n",
-        "\n",
-        "# Exclude the first 100 rows from training so that they can be used for test.\n",
-        "X_train = digits.data[100:,:]\n",
-        "y_train = digits.target[100:]"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Train\n",
-        "\n",
-        "Instantiate an `AutoMLConfig` object to specify the settings and data used to run the experiment.\n",
-        "\n",
-        "|Property|Description|\n",
-        "|-|-|\n",
-        "|**task**|classification or regression|\n",
-        "|**primary_metric**|This is the metric that you want to optimize. Classification supports the following primary metrics: <br><i>accuracy</i><br><i>AUC_weighted</i><br><i>balanced_accuracy</i><br><i>average_precision_score_weighted</i><br><i>precision_score_weighted</i>|\n",
-        "|**iteration_timeout_minutes**|Time limit in minutes for each iteration.|\n",
-        "|**iterations**|Number of iterations. In each iteration AutoML trains a specific pipeline with the data.|\n",
-        "|**n_cross_validations**|Number of cross validation splits.|\n",
-        "|**X**|(sparse) array-like, shape = [n_samples, n_features]|\n",
-        "|**y**|(sparse) array-like, shape = [n_samples, ], Multi-class targets.|\n",
-        "|**path**|Relative path to the project folder. AutoML stores configuration files for the experiment under this folder. You can specify a new empty folder.|\n",
-        "|**whitelist_models**|List of models that AutoML should use.  The possible values are listed [here](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-configure-auto-train#configure-your-experiment-settings).|"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "automl_config = AutoMLConfig(task = 'classification',\n",
-        "                             debug_log = 'automl_errors.log',\n",
-        "                             primary_metric = 'AUC_weighted',\n",
-        "                             iteration_timeout_minutes = 60,\n",
-        "                             iterations = 10,\n",
-        "                             verbosity = logging.INFO,\n",
-        "                             X = X_train, \n",
-        "                             y = y_train,\n",
-        "                             enable_tf=True,\n",
-        "                             whitelist_models=whitelist_models,\n",
-        "                             path = project_folder)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "Call the `submit` method on the experiment object and pass the run configuration. Execution of local runs is synchronous. Depending on the data and the number of iterations this can run for a while.\n",
-        "In this example, we specify `show_output = True` to print currently running iterations to the console."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "local_run = experiment.submit(automl_config, show_output = True)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "local_run"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Results"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Widget for Monitoring Runs\n",
-        "\n",
-        "The widget will first report a \"loading\" status while running the first iteration. After completing the first iteration, an auto-updating graph and table will be shown. The widget will refresh once per minute, so you should see the graph update as child runs complete.\n",
-        "\n",
-        "**Note:** The widget displays a link at the bottom. Use this link to open a web interface to explore the individual run details."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "from azureml.widgets import RunDetails\n",
-        "RunDetails(local_run).show() "
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "\n",
-        "#### Retrieve All Child Runs\n",
-        "You can also use SDK methods to fetch all the child runs and see individual metrics that we log."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "children = list(local_run.get_children())\n",
-        "metricslist = {}\n",
-        "for run in children:\n",
-        "    properties = run.get_properties()\n",
-        "    metrics = {k: v for k, v in run.get_metrics().items() if isinstance(v, float)}\n",
-        "    metricslist[int(properties['iteration'])] = metrics\n",
-        "\n",
-        "rundata = pd.DataFrame(metricslist).sort_index(1)\n",
-        "rundata"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Retrieve the Best Model\n",
-        "\n",
-        "Below we select the best pipeline from our iterations. The `get_output` method returns the best run and the fitted model. The Model includes the pipeline and any pre-processing.  Overloads on `get_output` allow you to retrieve the best run and fitted model for *any* logged metric or for a particular *iteration*."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "best_run, fitted_model = local_run.get_output()\n",
-        "print(best_run)\n",
-        "print(fitted_model)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Best Model Based on Any Other Metric\n",
-        "Show the run and the model that has the smallest `log_loss` value:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "lookup_metric = \"log_loss\"\n",
-        "best_run, fitted_model = local_run.get_output(metric = lookup_metric)\n",
-        "print(best_run)\n",
-        "print(fitted_model)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Model from a Specific Iteration\n",
-        "Show the run and the model from the third iteration:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "iteration = 3\n",
-        "third_run, third_model = local_run.get_output(iteration = iteration)\n",
-        "print(third_run)\n",
-        "print(third_model)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Test\n",
-        "\n",
-        "#### Load Test Data"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "digits = datasets.load_digits()\n",
-        "X_test = digits.data[:10, :]\n",
-        "y_test = digits.target[:10]\n",
-        "images = digits.images[:10]"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Testing Our Best Fitted Model\n",
-        "We will try to predict 2 digits and see how our model works."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "# Randomly select digits and test.\n",
-        "for index in np.random.choice(len(y_test), 2, replace = False):\n",
-        "    print(index)\n",
-        "    predicted = fitted_model.predict(X_test[index:index + 1])[0]\n",
-        "    label = y_test[index]\n",
-        "    title = \"Label value = %d  Predicted value = %d \" % (label, predicted)\n",
-        "    fig = plt.figure(1, figsize = (3,3))\n",
-        "    ax1 = fig.add_axes((0,0,.8,.8))\n",
-        "    ax1.set_title(title)\n",
-        "    plt.imshow(images[index], cmap = plt.cm.gray_r, interpolation = 'nearest')\n",
-        "    plt.show()"
-      ]
-    }
-  ],
-  "metadata": {
-    "authors": [
-      {
-        "name": "savitam"
-      }
-    ],
-    "kernelspec": {
-      "display_name": "Python 3.6",
-      "language": "python",
-      "name": "python36"
-    },
-    "language_info": {
-      "codemirror_mode": {
-        "name": "ipython",
-        "version": 3
-      },
-      "file_extension": ".py",
-      "mimetype": "text/x-python",
-      "name": "python",
-      "nbconvert_exporter": "python",
-      "pygments_lexer": "ipython3",
-      "version": "3.6.6"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 2
-}

how-to-use-azureml/automated-machine-learning/classification/auto-ml-classification.ipynb

@@ -1,441 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
-        "\n",
-        "Licensed under the MIT License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "# Automated Machine Learning\n",
-        "_**Classification with Local Compute**_\n",
-        "\n",
-        "## Contents\n",
-        "1. [Introduction](#Introduction)\n",
-        "1. [Setup](#Setup)\n",
-        "1. [Data](#Data)\n",
-        "1. [Train](#Train)\n",
-        "1. [Results](#Results)\n",
-        "1. [Test](#Test)\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Introduction\n",
-        "\n",
-        "In this example we use the scikit-learn's [digit dataset](http://scikit-learn.org/stable/datasets/index.html#optical-recognition-of-handwritten-digits-dataset) to showcase how you can use AutoML for a simple classification problem.\n",
-        "\n",
-        "Make sure you have executed the [configuration](../../../configuration.ipynb) before running this notebook.\n",
-        "\n",
-        "In this notebook you will learn how to:\n",
-        "1. Create an `Experiment` in an existing `Workspace`.\n",
-        "2. Configure AutoML using `AutoMLConfig`.\n",
-        "3. Train the model using local compute.\n",
-        "4. Explore the results.\n",
-        "5. Test the best fitted model."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Setup\n",
-        "\n",
-        "As part of the setup you have already created an Azure ML `Workspace` object. For AutoML you will need to create an `Experiment` object, which is a named object in a `Workspace` used to run experiments."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "import logging\n",
-        "\n",
-        "from matplotlib import pyplot as plt\n",
-        "import numpy as np\n",
-        "import pandas as pd\n",
-        "from sklearn import datasets\n",
-        "\n",
-        "import azureml.core\n",
-        "from azureml.core.experiment import Experiment\n",
-        "from azureml.core.workspace import Workspace\n",
-        "from azureml.train.automl import AutoMLConfig"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "ws = Workspace.from_config()\n",
-        "\n",
-        "# Choose a name for the experiment and specify the project folder.\n",
-        "experiment_name = 'automl-classification'\n",
-        "project_folder = './sample_projects/automl-classification'\n",
-        "\n",
-        "experiment = Experiment(ws, experiment_name)\n",
-        "\n",
-        "output = {}\n",
-        "output['SDK version'] = azureml.core.VERSION\n",
-        "output['Subscription ID'] = ws.subscription_id\n",
-        "output['Workspace Name'] = ws.name\n",
-        "output['Resource Group'] = ws.resource_group\n",
-        "output['Location'] = ws.location\n",
-        "output['Project Directory'] = project_folder\n",
-        "output['Experiment Name'] = experiment.name\n",
-        "pd.set_option('display.max_colwidth', -1)\n",
-        "outputDf = pd.DataFrame(data = output, index = [''])\n",
-        "outputDf.T"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Data\n",
-        "\n",
-        "This uses scikit-learn's [load_digits](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_digits.html) method."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "digits = datasets.load_digits()\n",
-        "\n",
-        "# Exclude the first 100 rows from training so that they can be used for test.\n",
-        "X_train = digits.data[100:,:]\n",
-        "y_train = digits.target[100:]"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Train\n",
-        "\n",
-        "Instantiate an `AutoMLConfig` object to specify the settings and data used to run the experiment.\n",
-        "\n",
-        "|Property|Description|\n",
-        "|-|-|\n",
-        "|**task**|classification or regression|\n",
-        "|**primary_metric**|This is the metric that you want to optimize. Classification supports the following primary metrics: <br><i>accuracy</i><br><i>AUC_weighted</i><br><i>average_precision_score_weighted</i><br><i>norm_macro_recall</i><br><i>precision_score_weighted</i>|\n",
-        "|**iteration_timeout_minutes**|Time limit in minutes for each iteration.|\n",
-        "|**iterations**|Number of iterations. In each iteration AutoML trains a specific pipeline with the data.|\n",
-        "|**X**|(sparse) array-like, shape = [n_samples, n_features]|\n",
-        "|**y**|(sparse) array-like, shape = [n_samples, ], Multi-class targets.|\n",
-        "|**path**|Relative path to the project folder. AutoML stores configuration files for the experiment under this folder. You can specify a new empty folder.|"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "automl_config = AutoMLConfig(task = 'classification',\n",
-        "                             debug_log = 'automl_errors.log',\n",
-        "                             primary_metric = 'AUC_weighted',\n",
-        "                             iteration_timeout_minutes = 60,\n",
-        "                             iterations = 25,\n",
-        "                             verbosity = logging.INFO,\n",
-        "                             X = X_train, \n",
-        "                             y = y_train,\n",
-        "                             path = project_folder)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "Call the `submit` method on the experiment object and pass the run configuration. Execution of local runs is synchronous. Depending on the data and the number of iterations this can run for a while.\n",
-        "In this example, we specify `show_output = True` to print currently running iterations to the console."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "local_run = experiment.submit(automl_config, show_output = True)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "local_run"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "Optionally, you can continue an interrupted local run by calling `continue_experiment` without the `iterations` parameter, or run more iterations for a completed run by specifying the `iterations` parameter:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "local_run = local_run.continue_experiment(X = X_train, \n",
-        "                                          y = y_train, \n",
-        "                                          show_output = True,\n",
-        "                                          iterations = 5)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Results"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Widget for Monitoring Runs\n",
-        "\n",
-        "The widget will first report a \"loading\" status while running the first iteration. After completing the first iteration, an auto-updating graph and table will be shown. The widget will refresh once per minute, so you should see the graph update as child runs complete.\n",
-        "\n",
-        "**Note:** The widget displays a link at the bottom. Use this link to open a web interface to explore the individual run details."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "from azureml.widgets import RunDetails\n",
-        "RunDetails(local_run).show() "
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "\n",
-        "#### Retrieve All Child Runs\n",
-        "You can also use SDK methods to fetch all the child runs and see individual metrics that we log."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "children = list(local_run.get_children())\n",
-        "metricslist = {}\n",
-        "for run in children:\n",
-        "    properties = run.get_properties()\n",
-        "    metrics = {k: v for k, v in run.get_metrics().items() if isinstance(v, float)}\n",
-        "    metricslist[int(properties['iteration'])] = metrics\n",
-        "\n",
-        "rundata = pd.DataFrame(metricslist).sort_index(1)\n",
-        "rundata"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Retrieve the Best Model\n",
-        "\n",
-        "Below we select the best pipeline from our iterations. The `get_output` method returns the best run and the fitted model. The Model includes the pipeline and any pre-processing.  Overloads on `get_output` allow you to retrieve the best run and fitted model for *any* logged metric or for a particular *iteration*."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "best_run, fitted_model = local_run.get_output()\n",
-        "print(best_run)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Print the properties of the model\n",
-        "The fitted_model is a python object and you can read the different properties of the object.\n",
-        "The following shows printing hyperparameters for each step in the pipeline."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "from pprint import pprint\n",
-        "\n",
-        "def print_model(model, prefix=\"\"):\n",
-        "    for step in model.steps:\n",
-        "        print(prefix + step[0])\n",
-        "        if hasattr(step[1], 'estimators') and hasattr(step[1], 'weights'):\n",
-        "            pprint({'estimators': list(e[0] for e in step[1].estimators), 'weights': step[1].weights})\n",
-        "            print()\n",
-        "            for estimator in step[1].estimators:\n",
-        "                print_model(estimator[1], estimator[0]+ ' - ')\n",
-        "        else:\n",
-        "            pprint(step[1].get_params())\n",
-        "            print()\n",
-        "            \n",
-        "print_model(fitted_model)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Best Model Based on Any Other Metric\n",
-        "Show the run and the model that has the smallest `log_loss` value:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "lookup_metric = \"log_loss\"\n",
-        "best_run, fitted_model = local_run.get_output(metric = lookup_metric)\n",
-        "print(best_run)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "print_model(fitted_model)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Model from a Specific Iteration\n",
-        "Show the run and the model from the third iteration:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "iteration = 3\n",
-        "third_run, third_model = local_run.get_output(iteration = iteration)\n",
-        "print(third_run)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "print_model(third_model)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Test \n",
-        "\n",
-        "#### Load Test Data"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "digits = datasets.load_digits()\n",
-        "X_test = digits.data[:10, :]\n",
-        "y_test = digits.target[:10]\n",
-        "images = digits.images[:10]"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Testing Our Best Fitted Model\n",
-        "We will try to predict 2 digits and see how our model works."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "# Randomly select digits and test.\n",
-        "for index in np.random.choice(len(y_test), 2, replace = False):\n",
-        "    print(index)\n",
-        "    predicted = fitted_model.predict(X_test[index:index + 1])[0]\n",
-        "    label = y_test[index]\n",
-        "    title = \"Label value = %d  Predicted value = %d \" % (label, predicted)\n",
-        "    fig = plt.figure(1, figsize = (3,3))\n",
-        "    ax1 = fig.add_axes((0,0,.8,.8))\n",
-        "    ax1.set_title(title)\n",
-        "    plt.imshow(images[index], cmap = plt.cm.gray_r, interpolation = 'nearest')\n",
-        "    plt.show()"
-      ]
-    }
-  ],
-  "metadata": {
-    "authors": [
-      {
-        "name": "savitam"
-      }
-    ],
-    "kernelspec": {
-      "display_name": "Python 3.6",
-      "language": "python",
-      "name": "python36"
-    },
-    "language_info": {
-      "codemirror_mode": {
-        "name": "ipython",
-        "version": 3
-      },
-      "file_extension": ".py",
-      "mimetype": "text/x-python",
-      "name": "python",
-      "nbconvert_exporter": "python",
-      "pygments_lexer": "ipython3",
-      "version": "3.6.6"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 2
-}

how-to-use-azureml/automated-machine-learning/continuous-retraining/auto-ml-continuous-retraining.ipynb

@@ -0,0 +1,556 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Copyright (c) Microsoft Corporation. All rights reserved.  \n",
+        "Licensed under the MIT License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/automated-machine-learning/continous-retraining/auto-ml-continuous-retraining.png)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# Automated Machine Learning \n",
+        "**Continous retraining using Pipelines and Time-Series TabularDataset**\n",
+        "## Contents\n",
+        "1. [Introduction](#Introduction)\n",
+        "2. [Setup](#Setup)\n",
+        "3. [Compute](#Compute)\n",
+        "4. [Run Configuration](#Run-Configuration)\n",
+        "5. [Data Ingestion Pipeline](#Data-Ingestion-Pipeline)\n",
+        "6. [Training Pipeline](#Training-Pipeline)\n",
+        "7. [Publish Retraining Pipeline and Schedule](#Publish-Retraining-Pipeline-and-Schedule)\n",
+        "8. [Test Retraining](#Test-Retraining)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "An Enterprise workspace is required for this notebook. To learn more about creating an Enterprise workspace or upgrading to an Enterprise workspace from the Azure portal, please visit our [Workspace page.](https://docs.microsoft.com/azure/machine-learning/service/concept-workspace#upgrade)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Introduction\n",
+        "In this example we use AutoML and Pipelines to enable contious retraining of a model based on updates to the training dataset. We will create two pipelines, the first one to demonstrate a training dataset that gets updated over time. We leverage time-series capabilities of `TabularDataset` to achieve this. The second pipeline utilizes pipeline `Schedule` to trigger continuous retraining. \n",
+        "Make sure you have executed the [configuration notebook](../../../configuration.ipynb) before running this notebook.\n",
+        "In this notebook you will learn how to:\n",
+        "* Create an Experiment in an existing Workspace.\n",
+        "* Configure AutoML using AutoMLConfig.\n",
+        "* Create data ingestion pipeline to update a time-series based TabularDataset\n",
+        "* Create training pipeline to prepare data, run AutoML, register the model and setup pipeline triggers.\n",
+        "\n",
+        "## Setup\n",
+        "As part of the setup you have already created an Azure ML `Workspace` object. For AutoML you will need to create an `Experiment` object, which is a named object in a `Workspace` used to run experiments."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import logging\n",
+        "\n",
+        "from matplotlib import pyplot as plt\n",
+        "import numpy as np\n",
+        "import pandas as pd\n",
+        "from sklearn import datasets\n",
+        "\n",
+        "import azureml.core\n",
+        "from azureml.core.experiment import Experiment\n",
+        "from azureml.core.workspace import Workspace\n",
+        "from azureml.train.automl import AutoMLConfig"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Accessing the Azure ML workspace requires authentication with Azure.\n",
+        "\n",
+        "The default authentication is interactive authentication using the default tenant. Executing the ws = Workspace.from_config() line in the cell below will prompt for authentication the first time that it is run.\n",
+        "\n",
+        "If you have multiple Azure tenants, you can specify the tenant by replacing the ws = Workspace.from_config() line in the cell below with the following:\n",
+        "```\n",
+        "from azureml.core.authentication import InteractiveLoginAuthentication\n",
+        "auth = InteractiveLoginAuthentication(tenant_id = 'mytenantid')\n",
+        "ws = Workspace.from_config(auth = auth)\n",
+        "```\n",
+        "If you need to run in an environment where interactive login is not possible, you can use Service Principal authentication by replacing the ws = Workspace.from_config() line in the cell below with the following:\n",
+        "```\n",
+        "from azureml.core.authentication import ServicePrincipalAuthentication\n",
+        "auth = auth = ServicePrincipalAuthentication('mytenantid', 'myappid', 'mypassword')\n",
+        "ws = Workspace.from_config(auth = auth)\n",
+        "```\n",
+        "For more details, see aka.ms/aml-notebook-auth"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "ws = Workspace.from_config()\n",
+        "dstor = ws.get_default_datastore()\n",
+        "\n",
+        "# Choose a name for the run history container in the workspace.\n",
+        "experiment_name = 'retrain-noaaweather'\n",
+        "experiment = Experiment(ws, experiment_name)\n",
+        "\n",
+        "output = {}\n",
+        "output['SDK version'] = azureml.core.VERSION\n",
+        "output['Subscription ID'] = ws.subscription_id\n",
+        "output['Workspace'] = ws.name\n",
+        "output['Resource Group'] = ws.resource_group\n",
+        "output['Location'] = ws.location\n",
+        "output['Run History Name'] = experiment_name\n",
+        "pd.set_option('display.max_colwidth', -1)\n",
+        "outputDf = pd.DataFrame(data = output, index = [''])\n",
+        "outputDf.T"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Compute \n",
+        "\n",
+        "#### Create or Attach existing AmlCompute\n",
+        "\n",
+        "You will need to create a compute target for your AutoML run. In this tutorial, you create AmlCompute as your training compute resource.\n",
+        "#### Creation of AmlCompute takes approximately 5 minutes. \n",
+        "If the AmlCompute with that name is already in your workspace this code will skip the creation process.\n",
+        "As with other Azure services, there are limits on certain resources (e.g. AmlCompute) associated with the Azure Machine Learning service. Please read this article on the default limits and how to request more quota."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.compute import AmlCompute, ComputeTarget\n",
+        "\n",
+        "# Choose a name for your cluster.\n",
+        "amlcompute_cluster_name = \"cpu-cluster-42\"\n",
+        "\n",
+        "found = False\n",
+        "# Check if this compute target already exists in the workspace.\n",
+        "cts = ws.compute_targets\n",
+        "if amlcompute_cluster_name in cts and cts[amlcompute_cluster_name].type == 'AmlCompute':\n",
+        "    found = True\n",
+        "    print('Found existing compute target.')\n",
+        "    compute_target = cts[amlcompute_cluster_name]\n",
+        "    \n",
+        "if not found:\n",
+        "    print('Creating a new compute target...')\n",
+        "    provisioning_config = AmlCompute.provisioning_configuration(vm_size = \"STANDARD_D2_V2\", # for GPU, use \"STANDARD_NC6\"\n",
+        "                                                                #vm_priority = 'lowpriority', # optional\n",
+        "                                                                max_nodes = 4)\n",
+        "\n",
+        "    # Create the cluster.\n",
+        "    compute_target = ComputeTarget.create(ws, amlcompute_cluster_name, provisioning_config)\n",
+        "    \n",
+        "    # Can poll for a minimum number of nodes and for a specific timeout.\n",
+        "    # If no min_node_count is provided, it will use the scale settings for the cluster.\n",
+        "    compute_target.wait_for_completion(show_output = True, min_node_count = 0, timeout_in_minutes = 10)\n",
+        "    \n",
+        "     # For a more detailed view of current AmlCompute status, use get_status()."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Run Configuration"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.runconfig import CondaDependencies, DEFAULT_CPU_IMAGE, RunConfiguration\n",
+        "\n",
+        "# create a new RunConfig object\n",
+        "conda_run_config = RunConfiguration(framework=\"python\")\n",
+        "\n",
+        "# Set compute target to AmlCompute\n",
+        "conda_run_config.target = compute_target\n",
+        "\n",
+        "conda_run_config.environment.docker.enabled = True\n",
+        "conda_run_config.environment.docker.base_image = azureml.core.runconfig.DEFAULT_CPU_IMAGE\n",
+        "\n",
+        "cd = CondaDependencies.create(pip_packages=['azureml-sdk[automl]', 'applicationinsights', 'azureml-opendatasets'], \n",
+        "                              conda_packages=['numpy', 'py-xgboost'], \n",
+        "                              pin_sdk_version=False)\n",
+        "#cd.add_pip_package('azureml-explain-model')\n",
+        "conda_run_config.environment.python.conda_dependencies = cd\n",
+        "\n",
+        "print('run config is ready')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Data Ingestion Pipeline \n",
+        "For this demo, we will use NOAA weather data from [Azure Open Datasets](https://azure.microsoft.com/services/open-datasets/). You can replace this with your own dataset, or you can skip this pipeline if you already have a time-series based `TabularDataset`.\n",
+        "\n",
+        "### Upload Data Step\n",
+        "The data ingestion pipeline has a single step with a script to query the latest weather data and upload it to the blob store. During the first run, the script will create and register a time-series based `TabularDataset` with the past one week of weather data. For each subsequent run, the script will create a partition in the blob store by querying NOAA for new weather data since the last modified time of the dataset (`dataset.data_changed_time`) and creating a data.csv file."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.pipeline.core import Pipeline, PipelineParameter\n",
+        "from azureml.pipeline.steps import PythonScriptStep\n",
+        "\n",
+        "# The name of the Dataset to create \n",
+        "dataset = \"NOAA-Weather-DS4\"\n",
+        "ds_name = PipelineParameter(name=\"ds_name\", default_value=dataset)\n",
+        "upload_data_step = PythonScriptStep(script_name=\"upload_weather_data.py\", \n",
+        "                                         allow_reuse=False,\n",
+        "                                         name=\"upload_weather_data\",\n",
+        "                                         arguments=[\"--ds_name\", ds_name],\n",
+        "                                         compute_target=compute_target, \n",
+        "                                         runconfig=conda_run_config)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Submit Pipeline Run"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "data_pipeline = Pipeline(\n",
+        "    description=\"pipeline_with_uploaddata\",\n",
+        "    workspace=ws,    \n",
+        "    steps=[upload_data_step])\n",
+        "data_pipeline_run = experiment.submit(data_pipeline, pipeline_parameters={\"ds_name\":dataset})"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "data_pipeline_run.wait_for_completion()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Training Pipeline\n",
+        "### Prepare Training Data Step\n",
+        "\n",
+        "Script to bring data into common X,y format. We need to set allow_reuse flag to False to allow the pipeline to run even when inputs don't change. We also need the name of the model to check the time the model was last trained."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.pipeline.core import PipelineData\n",
+        "\n",
+        "target_column = PipelineParameter(\"target_column\", default_value=\"y\")\n",
+        "# The model name with which to register the trained model in the workspace.\n",
+        "model_name = PipelineParameter(\"model_name\", default_value=\"y\")\n",
+        "output_x = PipelineData(\"output_x\", datastore=dstor)\n",
+        "output_y = PipelineData(\"output_y\", datastore=dstor)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "data_prep_step = PythonScriptStep(script_name=\"check_data.py\", \n",
+        "                                         allow_reuse=False,\n",
+        "                                         name=\"check_data\",\n",
+        "                                         arguments=[\"--target_column\", target_column,\n",
+        "                                                       \"--output_x\", output_x,\n",
+        "                                                       \"--output_y\", output_y,\n",
+        "                                                       \"--ds_name\", ds_name,\n",
+        "                                                       \"--model_name\", model_name],\n",
+        "                                         outputs=[output_x, output_y], \n",
+        "                                         compute_target=compute_target, \n",
+        "                                         runconfig=conda_run_config)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### AutoMLStep\n",
+        "Create an AutoMLConfig and a training step."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.train.automl import AutoMLConfig\n",
+        "from azureml.train.automl.runtime import AutoMLStep\n",
+        "\n",
+        "automl_settings = {\n",
+        "    \"iteration_timeout_minutes\": 20,\n",
+        "    \"experiment_timeout_minutes\": 30,\n",
+        "    \"n_cross_validations\": 3,\n",
+        "    \"primary_metric\": 'r2_score',\n",
+        "    \"preprocess\": True,\n",
+        "    \"max_concurrent_iterations\": 3,\n",
+        "    \"max_cores_per_iteration\": -1,\n",
+        "    \"verbosity\": logging.INFO,\n",
+        "    \"enable_early_stopping\": True\n",
+        "}\n",
+        "\n",
+        "automl_config = AutoMLConfig(task = 'regression',\n",
+        "                             debug_log = 'automl_errors.log',\n",
+        "                             path = \".\",\n",
+        "                             compute_target=compute_target,\n",
+        "                             run_configuration=conda_run_config,\n",
+        "                             data_script = \"get_data.py\",\n",
+        "                             **automl_settings\n",
+        "                            )"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.pipeline.core import PipelineData, TrainingOutput\n",
+        "\n",
+        "metrics_output_name = 'metrics_output'\n",
+        "best_model_output_name = 'best_model_output'\n",
+        "\n",
+        "metirics_data = PipelineData(name='metrics_data',\n",
+        "                           datastore=dstor,\n",
+        "                           pipeline_output_name=metrics_output_name,\n",
+        "                           training_output=TrainingOutput(type='Metrics'))\n",
+        "model_data = PipelineData(name='model_data',\n",
+        "                           datastore=dstor,\n",
+        "                           pipeline_output_name=best_model_output_name,\n",
+        "                           training_output=TrainingOutput(type='Model'))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "automl_step = AutoMLStep(\n",
+        "    name='automl_module',\n",
+        "    automl_config=automl_config,\n",
+        "    inputs=[output_x, output_y],\n",
+        "    outputs=[metirics_data, model_data],\n",
+        "    allow_reuse=False)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Register Model Step\n",
+        "Script to register the model to the workspace. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "register_model_step = PythonScriptStep(script_name=\"register_model.py\",\n",
+        "                                       name=\"register_model\",\n",
+        "                                       allow_reuse=False,\n",
+        "                                       arguments=[\"--model_name\", model_name, \"--model_path\", model_data, \"--ds_name\", ds_name],\n",
+        "                                       inputs=[model_data],\n",
+        "                                       compute_target=compute_target,\n",
+        "                                       runconfig=conda_run_config)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Submit Pipeline Run"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "training_pipeline = Pipeline(\n",
+        "    description=\"training_pipeline\",\n",
+        "    workspace=ws,    \n",
+        "    steps=[data_prep_step, automl_step, register_model_step])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "training_pipeline_run = experiment.submit(training_pipeline, pipeline_parameters={\n",
+        "        \"target_column\": \"temperature\", \"ds_name\": dataset, \"model_name\": \"noaaweatherds\"})"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "training_pipeline_run.wait_for_completion(show_output=False)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Publish Retraining Pipeline and Schedule\n",
+        "Once we are happy with the pipeline, we can publish the training pipeline to the workspace and create a schedule to trigger on blob change. The schedule polls the blob store where the data is being uploaded and runs the retraining pipeline if there is a data change. A new version of the model will be registered to the workspace once the run is complete."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "pipeline_name = \"Retraining-Pipeline-NOAAWeather\"\n",
+        "\n",
+        "published_pipeline = training_pipeline.publish(\n",
+        "    name=pipeline_name, \n",
+        "    description=\"Pipeline that retrains AutoML model\")\n",
+        "\n",
+        "published_pipeline"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.pipeline.core import Schedule\n",
+        "schedule = Schedule.create(workspace=ws, name=\"RetrainingSchedule\",\n",
+        "                           pipeline_parameters={\"target_column\": \"temperature\",\"ds_name\": dataset, \"model_name\": \"noaaweatherds\"},\n",
+        "                           pipeline_id=published_pipeline.id, \n",
+        "                           experiment_name=experiment_name, \n",
+        "                           datastore=dstor,\n",
+        "                           wait_for_provisioning=True,\n",
+        "                           polling_interval=1440)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Test Retraining\n",
+        "Here we setup the data ingestion pipeline to run on a schedule, to verify that the retraining pipeline runs as expected. \n",
+        "\n",
+        "Note: \n",
+        "* Azure NOAA Weather data is updated daily and retraining will not trigger if there is no new data available. \n",
+        "* Depending on the polling interval set in the schedule, the retraining may take some time trigger after data ingestion pipeline completes."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "pipeline_name = \"DataIngestion-Pipeline-NOAAWeather\"\n",
+        "\n",
+        "published_pipeline = training_pipeline.publish(\n",
+        "    name=pipeline_name, \n",
+        "    description=\"Pipeline that updates NOAAWeather Dataset\")\n",
+        "\n",
+        "published_pipeline"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.pipeline.core import Schedule\n",
+        "schedule = Schedule.create(workspace=ws, name=\"RetrainingSchedule-DataIngestion\",\n",
+        "                           pipeline_parameters={\"ds_name\":dataset},\n",
+        "                           pipeline_id=published_pipeline.id, \n",
+        "                           experiment_name=experiment_name, \n",
+        "                           datastore=dstor,\n",
+        "                           wait_for_provisioning=True,\n",
+        "                           polling_interval=1440)"
+      ]
+    }
+  ],
+  "metadata": {
+    "authors": [
+      {
+        "name": "vivijay"
+      }
+    ],
+    "kernelspec": {
+      "display_name": "Python 3.6",
+      "language": "python",
+      "name": "python36"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.6.6"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 2
+}

how-to-use-azureml/automated-machine-learning/continuous-retraining/auto-ml-continuous-retraining.yml

@@ -0,0 +1,9 @@
+name: auto-ml-continuous-retraining
+dependencies:
+- pip:
+  - azureml-sdk
+  - azureml-train-automl
+  - azureml-pipeline
+  - azureml-widgets
+  - matplotlib
+  - pandas_ml

how-to-use-azureml/automated-machine-learning/continuous-retraining/check_data.py

@@ -0,0 +1,75 @@
+import argparse
+import os
+import azureml.core
+from datetime import datetime
+import pandas as pd
+import pytz
+from azureml.core import Dataset, Model
+from azureml.core.run import Run, _OfflineRun
+from azureml.core import Workspace
+
+run = Run.get_context()
+ws = None
+if type(run) == _OfflineRun:
+    ws = Workspace.from_config()
+else:
+    ws = run.experiment.workspace
+
+
+def write_output(df, path):
+    os.makedirs(path, exist_ok=True)
+    print("%s created" % path)
+    df.to_csv(path + "/part-00000", index=False)
+
+
+print("Check for new data and prepare the data")
+
+parser = argparse.ArgumentParser("split")
+parser.add_argument("--target_column", type=str, help="input split features")
+parser.add_argument("--ds_name", help="input dataset name")
+parser.add_argument("--model_name", help="name of the deployed model")
+parser.add_argument("--output_x", type=str,
+                    help="output features")
+parser.add_argument("--output_y", type=str,
+                    help="output labels")
+
+
+args = parser.parse_args()
+
+print("Argument 1(ds_name): %s" % args.ds_name)
+print("Argument 2(target_column): %s" % args.target_column)
+print("Argument 3(model_name): %s" % args.model_name)
+print("Argument 4(output_x): %s" % args.output_x)
+print("Argument 5(output_y): %s" % args.output_y)
+
+# Get the latest registered model
+try:
+    model = Model(ws, args.model_name)
+    last_train_time = model.created_time
+    print("Model was last trained on {0}.".format(last_train_time))
+except Exception as e:
+    print("Could not get last model train time.")
+    last_train_time = datetime.min.replace(tzinfo=pytz.UTC)
+
+train_ds = Dataset.get_by_name(ws, args.ds_name)
+dataset_changed_time = train_ds.data_changed_time
+
+if dataset_changed_time > last_train_time:
+    # New data is available since the model was last trained
+    print("Dataset was last updated on {0}. Retraining...".format(dataset_changed_time))
+    train_ds = train_ds.drop_columns(["partition_date"])
+    X_train = train_ds.drop_columns(
+        columns=[args.target_column]).to_pandas_dataframe()
+    y_train = train_ds.keep_columns(
+        columns=[args.target_column]).to_pandas_dataframe()
+
+    non_null = y_train[args.target_column].notnull()
+    y = y_train[non_null]
+    X = X_train[non_null]
+
+    if not (args.output_x is None and args.output_y is None):
+        write_output(X, args.output_x)
+        write_output(y, args.output_y)
+else:
+    print("Cancelling run since there is no new data.")
+    run.parent.cancel()

how-to-use-azureml/automated-machine-learning/continuous-retraining/get_data.py

@@ -0,0 +1,15 @@
+import os
+import pandas as pd
+
+
+def get_data():
+    print("In get_data")
+    print(os.environ['AZUREML_DATAREFERENCE_output_x'])
+    X_train = pd.read_csv(
+        os.environ['AZUREML_DATAREFERENCE_output_x'] + "/part-00000")
+    y_train = pd.read_csv(
+        os.environ['AZUREML_DATAREFERENCE_output_y'] + "/part-00000")
+
+    print(X_train.head(3))
+
+    return {"X": X_train.values, "y": y_train.values.flatten()}

how-to-use-azureml/automated-machine-learning/continuous-retraining/register_model.py

@@ -0,0 +1,33 @@
+from azureml.core.model import Model, Dataset
+from azureml.core.run import Run, _OfflineRun
+from azureml.core import Workspace
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--model_name")
+parser.add_argument("--model_path")
+parser.add_argument("--ds_name")
+args = parser.parse_args()
+
+print("Argument 1(model_name): %s" % args.model_name)
+print("Argument 2(model_path): %s" % args.model_path)
+print("Argument 3(ds_name): %s" % args.ds_name)
+
+run = Run.get_context()
+ws = None
+if type(run) == _OfflineRun:
+    ws = Workspace.from_config()
+else:
+    ws = run.experiment.workspace
+
+train_ds = Dataset.get_by_name(ws, args.ds_name)
+datasets = [(Dataset.Scenario.TRAINING, train_ds)]
+
+# Register model with training dataset
+
+model = Model.register(workspace=ws,
+                       model_path=args.model_path,
+                       model_name=args.model_name,
+                       datasets=datasets)
+
+print("Registered version {0} of model {1}".format(model.version, model.name))

how-to-use-azureml/automated-machine-learning/continuous-retraining/upload_weather_data.py

@@ -0,0 +1,89 @@
+import argparse
+import os
+from datetime import datetime
+from dateutil.relativedelta import relativedelta
+import pandas as pd
+import traceback
+from azureml.core import Dataset
+from azureml.core.run import Run, _OfflineRun
+from azureml.core import Workspace
+from azureml.opendatasets import NoaaIsdWeather
+
+run = Run.get_context()
+ws = None
+if type(run) == _OfflineRun:
+    ws = Workspace.from_config()
+else:
+    ws = run.experiment.workspace
+
+usaf_list = ['725724', '722149', '723090', '722159', '723910', '720279',
+             '725513', '725254', '726430', '720381', '723074', '726682',
+             '725486', '727883', '723177', '722075', '723086', '724053',
+             '725070', '722073', '726060', '725224', '725260', '724520',
+             '720305', '724020', '726510', '725126', '722523', '703333',
+             '722249', '722728', '725483', '722972', '724975', '742079',
+             '727468', '722193', '725624', '722030', '726380', '720309',
+             '722071', '720326', '725415', '724504', '725665', '725424',
+             '725066']
+
+
+def get_noaa_data(start_time, end_time):
+    columns = ['usaf', 'wban', 'datetime', 'latitude', 'longitude', 'elevation',
+               'windAngle', 'windSpeed', 'temperature', 'stationName', 'p_k']
+    isd = NoaaIsdWeather(start_time, end_time, cols=columns)
+    noaa_df = isd.to_pandas_dataframe()
+    df_filtered = noaa_df[noaa_df["usaf"].isin(usaf_list)]
+    df_filtered.reset_index(drop=True)
+    print("Received {0} rows of training data between {1} and {2}".format(
+        df_filtered.shape[0], start_time, end_time))
+    return df_filtered
+
+
+print("Check for new data and prepare the data")
+
+parser = argparse.ArgumentParser("split")
+parser.add_argument("--ds_name", help="name of the Dataset to update")
+args = parser.parse_args()
+
+print("Argument 1(ds_name): %s" % args.ds_name)
+
+dstor = ws.get_default_datastore()
+register_dataset = False
+try:
+    ds = Dataset.get_by_name(ws, args.ds_name)
+    end_time_last_slice = ds.data_changed_time.replace(tzinfo=None)
+    print("Dataset {0} last updated on {1}".format(args.ds_name,
+                                                   end_time_last_slice))
+except Exception as e:
+    print(traceback.format_exc())
+    print("Dataset with name {0} not found, registering new dataset.".format(args.ds_name))
+    register_dataset = True
+    end_time_last_slice = datetime.today() - relativedelta(weeks=1)
+
+end_time = datetime.utcnow()
+train_df = get_noaa_data(end_time_last_slice, end_time)
+
+if train_df.size > 0:
+    print("Received {0} rows of new data after {0}.".format(
+        train_df.shape[0], end_time_last_slice))
+    folder_name = "{}/{:04d}/{:02d}/{:02d}/{:02d}/{:02d}/{:02d}".format(args.ds_name, end_time.year,
+                                                                        end_time.month, end_time.day,
+                                                                        end_time.hour, end_time.minute,
+                                                                        end_time.second)
+    file_path = "{0}/data.csv".format(folder_name)
+
+    # Add a new partition to the registered dataset
+    os.makedirs(folder_name, exist_ok=True)
+    train_df.to_csv(file_path, index=False)
+
+    dstor.upload_files(files=[file_path],
+                       target_path=folder_name,
+                       overwrite=True,
+                       show_progress=True)
+
+    if register_dataset:
+        ds = Dataset.Tabular.from_delimited_files(dstor.path("{}/**/*.csv".format(
+            args.ds_name)), partition_format='/{partition_date:yyyy/MM/dd/hh/mm/ss}/data.csv')
+        ds.register(ws, name=args.ds_name)
+else:
+    print("No new data since {0}.".format(end_time_last_slice))

how-to-use-azureml/automated-machine-learning/dataprep-remote-execution/auto-ml-dataprep-remote-execution.ipynb

@@ -1,498 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
-        "\n",
-        "Licensed under the MIT License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "# Automated Machine Learning\n",
-        "_**Prepare Data using `azureml.dataprep` for Remote Execution (DSVM)**_\n",
-        "\n",
-        "## Contents\n",
-        "1. [Introduction](#Introduction)\n",
-        "1. [Setup](#Setup)\n",
-        "1. [Data](#Data)\n",
-        "1. [Train](#Train)\n",
-        "1. [Results](#Results)\n",
-        "1. [Test](#Test)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Introduction\n",
-        "In this example we showcase how you can use the `azureml.dataprep` SDK to load and prepare data for AutoML. `azureml.dataprep` can also be used standalone; full documentation can be found [here](https://github.com/Microsoft/PendletonDocs).\n",
-        "\n",
-        "Make sure you have executed the [configuration](../../../configuration.ipynb) before running this notebook.\n",
-        "\n",
-        "In this notebook you will learn how to:\n",
-        "1. Define data loading and preparation steps in a `Dataflow` using `azureml.dataprep`.\n",
-        "2. Pass the `Dataflow` to AutoML for a local run.\n",
-        "3. Pass the `Dataflow` to AutoML for a remote run."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Setup\n",
-        "\n",
-        "Currently, Data Prep only supports __Ubuntu 16__ and __Red Hat Enterprise Linux 7__. We are working on supporting more linux distros."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "As part of the setup you have already created an Azure ML `Workspace` object. For AutoML you will need to create an `Experiment` object, which is a named object in a `Workspace` used to run experiments."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "import logging\n",
-        "import time\n",
-        "\n",
-        "import pandas as pd\n",
-        "\n",
-        "import azureml.core\n",
-        "from azureml.core.compute import DsvmCompute\n",
-        "from azureml.core.experiment import Experiment\n",
-        "from azureml.core.workspace import Workspace\n",
-        "import azureml.dataprep as dprep\n",
-        "from azureml.train.automl import AutoMLConfig"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "ws = Workspace.from_config()\n",
-        " \n",
-        "# choose a name for experiment\n",
-        "experiment_name = 'automl-dataprep-remote-dsvm'\n",
-        "# project folder\n",
-        "project_folder = './sample_projects/automl-dataprep-remote-dsvm'\n",
-        " \n",
-        "experiment = Experiment(ws, experiment_name)\n",
-        " \n",
-        "output = {}\n",
-        "output['SDK version'] = azureml.core.VERSION\n",
-        "output['Subscription ID'] = ws.subscription_id\n",
-        "output['Workspace Name'] = ws.name\n",
-        "output['Resource Group'] = ws.resource_group\n",
-        "output['Location'] = ws.location\n",
-        "output['Project Directory'] = project_folder\n",
-        "output['Experiment Name'] = experiment.name\n",
-        "pd.set_option('display.max_colwidth', -1)\n",
-        "outputDf = pd.DataFrame(data = output, index = [''])\n",
-        "outputDf.T"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Data"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "# You can use `auto_read_file` which intelligently figures out delimiters and datatypes of a file.\n",
-        "# The data referenced here was pulled from `sklearn.datasets.load_digits()`.\n",
-        "simple_example_data_root = 'https://dprepdata.blob.core.windows.net/automl-notebook-data/'\n",
-        "X = dprep.auto_read_file(simple_example_data_root + 'X.csv').skip(1)  # Remove the header row.\n",
-        "\n",
-        "# You can also use `read_csv` and `to_*` transformations to read (with overridable delimiter)\n",
-        "# and convert column types manually.\n",
-        "# Here we read a comma delimited file and convert all columns to integers.\n",
-        "y = dprep.read_csv(simple_example_data_root + 'y.csv').to_long(dprep.ColumnSelector(term='.*', use_regex = True))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "You can peek the result of a Dataflow at any range using `skip(i)` and `head(j)`. Doing so evaluates only `j` records for all the steps in the Dataflow, which makes it fast even against large datasets."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "X.skip(1).head(5)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Train\n",
-        "\n",
-        "This creates a general AutoML settings object applicable for both local and remote runs."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "automl_settings = {\n",
-        "    \"iteration_timeout_minutes\" : 10,\n",
-        "    \"iterations\" : 2,\n",
-        "    \"primary_metric\" : 'AUC_weighted',\n",
-        "    \"preprocess\" : False,\n",
-        "    \"verbosity\" : logging.INFO,\n",
-        "    \"n_cross_validations\": 3\n",
-        "}"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Create or Attach a Remote Linux DSVM"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "dsvm_name = 'mydsvmc'\n",
-        "\n",
-        "try:\n",
-        "    while ws.compute_targets[dsvm_name].provisioning_state == 'Creating':\n",
-        "        time.sleep(1)\n",
-        "        \n",
-        "    dsvm_compute = DsvmCompute(ws, dsvm_name)\n",
-        "    print('Found existing DVSM.')\n",
-        "except:\n",
-        "    print('Creating a new DSVM.')\n",
-        "    dsvm_config = DsvmCompute.provisioning_configuration(vm_size = \"Standard_D2_v2\")\n",
-        "    dsvm_compute = DsvmCompute.create(ws, name = dsvm_name, provisioning_configuration = dsvm_config)\n",
-        "    dsvm_compute.wait_for_completion(show_output = True)\n",
-        "    print(\"Waiting one minute for ssh to be accessible\")\n",
-        "    time.sleep(90) # Wait for ssh to be accessible"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "from azureml.core.runconfig import RunConfiguration\n",
-        "from azureml.core.conda_dependencies import CondaDependencies\n",
-        "\n",
-        "conda_run_config = RunConfiguration(framework=\"python\")\n",
-        "\n",
-        "conda_run_config.target = dsvm_compute\n",
-        "\n",
-        "cd = CondaDependencies.create(pip_packages=['azureml-sdk[automl]'], conda_packages=['numpy','py-xgboost<=0.80'])\n",
-        "conda_run_config.environment.python.conda_dependencies = cd"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Pass Data with `Dataflow` Objects\n",
-        "\n",
-        "The `Dataflow` objects captured above can also be passed to the `submit` method for a remote run. AutoML will serialize the `Dataflow` object and send it to the remote compute target. The `Dataflow` will not be evaluated locally."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "automl_config = AutoMLConfig(task = 'classification',\n",
-        "                             debug_log = 'automl_errors.log',\n",
-        "                             path = project_folder,\n",
-        "                             run_configuration=conda_run_config,\n",
-        "                             X = X,\n",
-        "                             y = y,\n",
-        "                             **automl_settings)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "remote_run = experiment.submit(automl_config, show_output = True)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "remote_run"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Results"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Widget for Monitoring Runs\n",
-        "\n",
-        "The widget will first report a \"loading\" status while running the first iteration. After completing the first iteration, an auto-updating graph and table will be shown. The widget will refresh once per minute, so you should see the graph update as child runs complete.\n",
-        "\n",
-        "**Note:** The widget displays a link at the bottom. Use this link to open a web interface to explore the individual run details."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "from azureml.widgets import RunDetails\n",
-        "RunDetails(remote_run).show()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Retrieve All Child Runs\n",
-        "You can also use SDK methods to fetch all the child runs and see individual metrics that we log."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "children = list(remote_run.get_children())\n",
-        "metricslist = {}\n",
-        "for run in children:\n",
-        "    properties = run.get_properties()\n",
-        "    metrics = {k: v for k, v in run.get_metrics().items() if isinstance(v, float)}\n",
-        "    metricslist[int(properties['iteration'])] = metrics\n",
-        "    \n",
-        "rundata = pd.DataFrame(metricslist).sort_index(1)\n",
-        "rundata"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Retrieve the Best Model\n",
-        "\n",
-        "Below we select the best pipeline from our iterations. The `get_output` method returns the best run and the fitted model. Overloads on `get_output` allow you to retrieve the best run and fitted model for *any* logged metric or for a particular *iteration*."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "best_run, fitted_model = remote_run.get_output()\n",
-        "print(best_run)\n",
-        "print(fitted_model)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Best Model Based on Any Other Metric\n",
-        "Show the run and the model that has the smallest `log_loss` value:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "lookup_metric = \"log_loss\"\n",
-        "best_run, fitted_model = remote_run.get_output(metric = lookup_metric)\n",
-        "print(best_run)\n",
-        "print(fitted_model)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Model from a Specific Iteration\n",
-        "Show the run and the model from the first iteration:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "iteration = 0\n",
-        "best_run, fitted_model = remote_run.get_output(iteration = iteration)\n",
-        "print(best_run)\n",
-        "print(fitted_model)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Test\n",
-        "\n",
-        "#### Load Test Data"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "from sklearn import datasets\n",
-        "\n",
-        "digits = datasets.load_digits()\n",
-        "X_test = digits.data[:10, :]\n",
-        "y_test = digits.target[:10]\n",
-        "images = digits.images[:10]"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Testing Our Best Fitted Model\n",
-        "We will try to predict 2 digits and see how our model works."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "#Randomly select digits and test\n",
-        "from matplotlib import pyplot as plt\n",
-        "import numpy as np\n",
-        "\n",
-        "for index in np.random.choice(len(y_test), 2, replace = False):\n",
-        "    print(index)\n",
-        "    predicted = fitted_model.predict(X_test[index:index + 1])[0]\n",
-        "    label = y_test[index]\n",
-        "    title = \"Label value = %d  Predicted value = %d \" % (label, predicted)\n",
-        "    fig = plt.figure(1, figsize=(3,3))\n",
-        "    ax1 = fig.add_axes((0,0,.8,.8))\n",
-        "    ax1.set_title(title)\n",
-        "    plt.imshow(images[index], cmap = plt.cm.gray_r, interpolation = 'nearest')\n",
-        "    plt.show()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Appendix"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Capture the `Dataflow` Objects for Later Use in AutoML\n",
-        "\n",
-        "`Dataflow` objects are immutable and are composed of a list of data preparation steps. A `Dataflow` object can be branched at any point for further usage."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "# sklearn.digits.data + target\n",
-        "digits_complete = dprep.auto_read_file('https://dprepdata.blob.core.windows.net/automl-notebook-data/digits-complete.csv')"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "`digits_complete` (sourced from `sklearn.datasets.load_digits()`) is forked into `dflow_X` to capture all the feature columns and `dflow_y` to capture the label column."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "print(digits_complete.to_pandas_dataframe().shape)\n",
-        "labels_column = 'Column64'\n",
-        "dflow_X = digits_complete.drop_columns(columns = [labels_column])\n",
-        "dflow_y = digits_complete.keep_columns(columns = [labels_column])"
-      ]
-    }
-  ],
-  "metadata": {
-    "authors": [
-      {
-        "name": "savitam"
-      }
-    ],
-    "kernelspec": {
-      "display_name": "Python 3.6",
-      "language": "python",
-      "name": "python36"
-    },
-    "language_info": {
-      "codemirror_mode": {
-        "name": "ipython",
-        "version": 3
-      },
-      "file_extension": ".py",
-      "mimetype": "text/x-python",
-      "name": "python",
-      "nbconvert_exporter": "python",
-      "pygments_lexer": "ipython3",
-      "version": "3.6.5"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 2
-}

how-to-use-azureml/automated-machine-learning/dataprep/auto-ml-dataprep.ipynb

@@ -1,448 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
-        "\n",
-        "Licensed under the MIT License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "# Automated Machine Learning\n",
-        "_**Prepare Data using `azureml.dataprep` for Local Execution**_\n",
-        "\n",
-        "## Contents\n",
-        "1. [Introduction](#Introduction)\n",
-        "1. [Setup](#Setup)\n",
-        "1. [Data](#Data)\n",
-        "1. [Train](#Train)\n",
-        "1. [Results](#Results)\n",
-        "1. [Test](#Test)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Introduction\n",
-        "In this example we showcase how you can use the `azureml.dataprep` SDK to load and prepare data for AutoML. `azureml.dataprep` can also be used standalone; full documentation can be found [here](https://github.com/Microsoft/PendletonDocs).\n",
-        "\n",
-        "Make sure you have executed the [configuration](../../../configuration.ipynb) before running this notebook.\n",
-        "\n",
-        "In this notebook you will learn how to:\n",
-        "1. Define data loading and preparation steps in a `Dataflow` using `azureml.dataprep`.\n",
-        "2. Pass the `Dataflow` to AutoML for a local run.\n",
-        "3. Pass the `Dataflow` to AutoML for a remote run."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Setup\n",
-        "\n",
-        "Currently, Data Prep only supports __Ubuntu 16__ and __Red Hat Enterprise Linux 7__. We are working on supporting more linux distros."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "As part of the setup you have already created an Azure ML `Workspace` object. For AutoML you will need to create an `Experiment` object, which is a named object in a `Workspace` used to run experiments."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "import logging\n",
-        "\n",
-        "import pandas as pd\n",
-        "\n",
-        "import azureml.core\n",
-        "from azureml.core.experiment import Experiment\n",
-        "from azureml.core.workspace import Workspace\n",
-        "import azureml.dataprep as dprep\n",
-        "from azureml.train.automl import AutoMLConfig"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "ws = Workspace.from_config()\n",
-        " \n",
-        "# choose a name for experiment\n",
-        "experiment_name = 'automl-dataprep-local'\n",
-        "# project folder\n",
-        "project_folder = './sample_projects/automl-dataprep-local'\n",
-        " \n",
-        "experiment = Experiment(ws, experiment_name)\n",
-        " \n",
-        "output = {}\n",
-        "output['SDK version'] = azureml.core.VERSION\n",
-        "output['Subscription ID'] = ws.subscription_id\n",
-        "output['Workspace Name'] = ws.name\n",
-        "output['Resource Group'] = ws.resource_group\n",
-        "output['Location'] = ws.location\n",
-        "output['Project Directory'] = project_folder\n",
-        "output['Experiment Name'] = experiment.name\n",
-        "pd.set_option('display.max_colwidth', -1)\n",
-        "outputDf = pd.DataFrame(data = output, index = [''])\n",
-        "outputDf.T"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Data"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "# You can use `auto_read_file` which intelligently figures out delimiters and datatypes of a file.\n",
-        "# The data referenced here was pulled from `sklearn.datasets.load_digits()`.\n",
-        "simple_example_data_root = 'https://dprepdata.blob.core.windows.net/automl-notebook-data/'\n",
-        "X = dprep.auto_read_file(simple_example_data_root + 'X.csv').skip(1)  # Remove the header row.\n",
-        "\n",
-        "# You can also use `read_csv` and `to_*` transformations to read (with overridable delimiter)\n",
-        "# and convert column types manually.\n",
-        "# Here we read a comma delimited file and convert all columns to integers.\n",
-        "y = dprep.read_csv(simple_example_data_root + 'y.csv').to_long(dprep.ColumnSelector(term='.*', use_regex = True))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Review the Data Preparation Result\n",
-        "\n",
-        "You can peek the result of a Dataflow at any range using `skip(i)` and `head(j)`. Doing so evaluates only `j` records for all the steps in the Dataflow, which makes it fast even against large datasets."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "X.skip(1).head(5)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Train\n",
-        "\n",
-        "This creates a general AutoML settings object applicable for both local and remote runs."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "automl_settings = {\n",
-        "    \"iteration_timeout_minutes\" : 10,\n",
-        "    \"iterations\" : 2,\n",
-        "    \"primary_metric\" : 'AUC_weighted',\n",
-        "    \"preprocess\" : False,\n",
-        "    \"verbosity\" : logging.INFO\n",
-        "}"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Pass Data with `Dataflow` Objects\n",
-        "\n",
-        "The `Dataflow` objects captured above can be passed to the `submit` method for a local run. AutoML will retrieve the results from the `Dataflow` for model training."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "automl_config = AutoMLConfig(task = 'classification',\n",
-        "                             debug_log = 'automl_errors.log',\n",
-        "                             X = X,\n",
-        "                             y = y,\n",
-        "                             **automl_settings)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "local_run = experiment.submit(automl_config, show_output = True)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "local_run"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Results"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Widget for Monitoring Runs\n",
-        "\n",
-        "The widget will first report a \"loading\" status while running the first iteration. After completing the first iteration, an auto-updating graph and table will be shown. The widget will refresh once per minute, so you should see the graph update as child runs complete.\n",
-        "\n",
-        "**Note:** The widget displays a link at the bottom. Use this link to open a web interface to explore the individual run details."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "from azureml.widgets import RunDetails\n",
-        "RunDetails(local_run).show()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Retrieve All Child Runs\n",
-        "You can also use SDK methods to fetch all the child runs and see individual metrics that we log."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "children = list(local_run.get_children())\n",
-        "metricslist = {}\n",
-        "for run in children:\n",
-        "    properties = run.get_properties()\n",
-        "    metrics = {k: v for k, v in run.get_metrics().items() if isinstance(v, float)}\n",
-        "    metricslist[int(properties['iteration'])] = metrics\n",
-        "    \n",
-        "rundata = pd.DataFrame(metricslist).sort_index(1)\n",
-        "rundata"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Retrieve the Best Model\n",
-        "\n",
-        "Below we select the best pipeline from our iterations. The `get_output` method returns the best run and the fitted model. Overloads on `get_output` allow you to retrieve the best run and fitted model for *any* logged metric or for a particular *iteration*."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "best_run, fitted_model = local_run.get_output()\n",
-        "print(best_run)\n",
-        "print(fitted_model)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Best Model Based on Any Other Metric\n",
-        "Show the run and the model that has the smallest `log_loss` value:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "lookup_metric = \"log_loss\"\n",
-        "best_run, fitted_model = local_run.get_output(metric = lookup_metric)\n",
-        "print(best_run)\n",
-        "print(fitted_model)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Model from a Specific Iteration\n",
-        "Show the run and the model from the first iteration:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "iteration = 0\n",
-        "best_run, fitted_model = local_run.get_output(iteration = iteration)\n",
-        "print(best_run)\n",
-        "print(fitted_model)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Test\n",
-        "\n",
-        "#### Load Test Data"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "from sklearn import datasets\n",
-        "\n",
-        "digits = datasets.load_digits()\n",
-        "X_test = digits.data[:10, :]\n",
-        "y_test = digits.target[:10]\n",
-        "images = digits.images[:10]"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Testing Our Best Fitted Model\n",
-        "We will try to predict 2 digits and see how our model works."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "#Randomly select digits and test\n",
-        "from matplotlib import pyplot as plt\n",
-        "import numpy as np\n",
-        "\n",
-        "for index in np.random.choice(len(y_test), 2, replace = False):\n",
-        "    print(index)\n",
-        "    predicted = fitted_model.predict(X_test[index:index + 1])[0]\n",
-        "    label = y_test[index]\n",
-        "    title = \"Label value = %d  Predicted value = %d \" % (label, predicted)\n",
-        "    fig = plt.figure(1, figsize=(3,3))\n",
-        "    ax1 = fig.add_axes((0,0,.8,.8))\n",
-        "    ax1.set_title(title)\n",
-        "    plt.imshow(images[index], cmap = plt.cm.gray_r, interpolation = 'nearest')\n",
-        "    plt.show()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Appendix"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Capture the `Dataflow` Objects for Later Use in AutoML\n",
-        "\n",
-        "`Dataflow` objects are immutable and are composed of a list of data preparation steps. A `Dataflow` object can be branched at any point for further usage."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "# sklearn.digits.data + target\n",
-        "digits_complete = dprep.auto_read_file('https://dprepdata.blob.core.windows.net/automl-notebook-data/digits-complete.csv')"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "`digits_complete` (sourced from `sklearn.datasets.load_digits()`) is forked into `dflow_X` to capture all the feature columns and `dflow_y` to capture the label column."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "print(digits_complete.to_pandas_dataframe().shape)\n",
-        "labels_column = 'Column64'\n",
-        "dflow_X = digits_complete.drop_columns(columns = [labels_column])\n",
-        "dflow_y = digits_complete.keep_columns(columns = [labels_column])"
-      ]
-    }
-  ],
-  "metadata": {
-    "authors": [
-      {
-        "name": "savitam"
-      }
-    ],
-    "kernelspec": {
-      "display_name": "Python 3.6",
-      "language": "python",
-      "name": "python36"
-    },
-    "language_info": {
-      "codemirror_mode": {
-        "name": "ipython",
-        "version": 3
-      },
-      "file_extension": ".py",
-      "mimetype": "text/x-python",
-      "name": "python",
-      "nbconvert_exporter": "python",
-      "pygments_lexer": "ipython3",
-      "version": "3.6.5"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 2
-}

how-to-use-azureml/automated-machine-learning/exploring-previous-runs/auto-ml-exploring-previous-runs.ipynb

@@ -1,342 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
-        "\n",
-        "Licensed under the MIT License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "# Automated Machine Learning\n",
-        "_**Exploring Previous Runs**_\n",
-        "\n",
-        "## Contents\n",
-        "1. [Introduction](#Introduction)\n",
-        "1. [Setup](#Setup)\n",
-        "1. [Explore](#Explore)\n",
-        "1. [Download](#Download)\n",
-        "1. [Register](#Register)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Introduction\n",
-        "In this example we present some examples on navigating previously executed runs. We also show how you can download a fitted model for any previous run.\n",
-        "\n",
-        "Make sure you have executed the [configuration](../../../configuration.ipynb) before running this notebook.\n",
-        "\n",
-        "In this notebook you will learn how to:\n",
-        "1. List all experiments in a workspace.\n",
-        "2. List all AutoML runs in an experiment.\n",
-        "3. Get details for an AutoML run, including settings, run widget, and all metrics.\n",
-        "4. Download a fitted pipeline for any iteration."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Setup"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "import pandas as pd\n",
-        "import json\n",
-        "\n",
-        "from azureml.core.experiment import Experiment\n",
-        "from azureml.core.workspace import Workspace\n",
-        "from azureml.train.automl.run import AutoMLRun"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "ws = Workspace.from_config()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Explore"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### List Experiments"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "experiment_list = Experiment.list(workspace=ws)\n",
-        "\n",
-        "summary_df = pd.DataFrame(index = ['No of Runs'])\n",
-        "for experiment in experiment_list:\n",
-        "    automl_runs = list(experiment.get_runs(type='automl'))\n",
-        "    summary_df[experiment.name] = [len(automl_runs)]\n",
-        "    \n",
-        "pd.set_option('display.max_colwidth', -1)\n",
-        "summary_df.T"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### List runs for an experiment\n",
-        "Set `experiment_name` to any experiment name from the result of the Experiment.list cell to load the AutoML runs."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "experiment_name = 'automl-local-classification' # Replace this with any project name from previous cell.\n",
-        "\n",
-        "proj = ws.experiments[experiment_name]\n",
-        "summary_df = pd.DataFrame(index = ['Type', 'Status', 'Primary Metric', 'Iterations', 'Compute', 'Name'])\n",
-        "automl_runs = list(proj.get_runs(type='automl'))\n",
-        "automl_runs_project = []\n",
-        "for run in automl_runs:\n",
-        "    properties = run.get_properties()\n",
-        "    tags = run.get_tags()\n",
-        "    amlsettings = json.loads(properties['AMLSettingsJsonString'])\n",
-        "    if 'iterations' in tags:\n",
-        "        iterations = tags['iterations']\n",
-        "    else:\n",
-        "        iterations = properties['num_iterations']\n",
-        "    summary_df[run.id] = [amlsettings['task_type'], run.get_details()['status'], properties['primary_metric'], iterations, properties['target'], amlsettings['name']]\n",
-        "    if run.get_details()['status'] == 'Completed':\n",
-        "        automl_runs_project.append(run.id)\n",
-        "    \n",
-        "from IPython.display import HTML\n",
-        "projname_html = HTML(\"<h3>{}</h3>\".format(proj.name))\n",
-        "\n",
-        "from IPython.display import display\n",
-        "display(projname_html)\n",
-        "display(summary_df.T)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Get details for a run\n",
-        "\n",
-        "Copy the project name and run id from the previous cell output to find more details on a particular run."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "run_id = automl_runs_project[0]  # Replace with your own run_id from above run ids\n",
-        "assert (run_id in summary_df.keys()), \"Run id not found! Please set run id to a value from above run ids\"\n",
-        "\n",
-        "from azureml.widgets import RunDetails\n",
-        "\n",
-        "experiment = Experiment(ws, experiment_name)\n",
-        "ml_run = AutoMLRun(experiment = experiment, run_id = run_id)\n",
-        "\n",
-        "summary_df = pd.DataFrame(index = ['Type', 'Status', 'Primary Metric', 'Iterations', 'Compute', 'Name', 'Start Time', 'End Time'])\n",
-        "properties = ml_run.get_properties()\n",
-        "tags = ml_run.get_tags()\n",
-        "status = ml_run.get_details()\n",
-        "amlsettings = json.loads(properties['AMLSettingsJsonString'])\n",
-        "if 'iterations' in tags:\n",
-        "    iterations = tags['iterations']\n",
-        "else:\n",
-        "    iterations = properties['num_iterations']\n",
-        "start_time = None\n",
-        "if 'startTimeUtc' in status:\n",
-        "    start_time = status['startTimeUtc']\n",
-        "end_time = None\n",
-        "if 'endTimeUtc' in status:\n",
-        "    end_time = status['endTimeUtc']\n",
-        "summary_df[ml_run.id] = [amlsettings['task_type'], status['status'], properties['primary_metric'], iterations, properties['target'], amlsettings['name'], start_time, end_time]\n",
-        "display(HTML('<h3>Runtime Details</h3>'))\n",
-        "display(summary_df)\n",
-        "\n",
-        "#settings_df = pd.DataFrame(data = amlsettings, index = [''])\n",
-        "display(HTML('<h3>AutoML Settings</h3>'))\n",
-        "display(amlsettings)\n",
-        "\n",
-        "display(HTML('<h3>Iterations</h3>'))\n",
-        "RunDetails(ml_run).show() \n",
-        "\n",
-        "children = list(ml_run.get_children())\n",
-        "metricslist = {}\n",
-        "for run in children:\n",
-        "    properties = run.get_properties()\n",
-        "    metrics = {k: v for k, v in run.get_metrics().items() if isinstance(v, float)}\n",
-        "    metricslist[int(properties['iteration'])] = metrics\n",
-        "\n",
-        "rundata = pd.DataFrame(metricslist).sort_index(1)\n",
-        "display(HTML('<h3>Metrics</h3>'))\n",
-        "display(rundata)\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Download"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Download the Best Model for Any Given Metric"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "metric = 'AUC_weighted' # Replace with a metric name.\n",
-        "best_run, fitted_model = ml_run.get_output(metric = metric)\n",
-        "fitted_model"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Download the Model for Any Given Iteration"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "iteration = 1 # Replace with an iteration number.\n",
-        "best_run, fitted_model = ml_run.get_output(iteration = iteration)\n",
-        "fitted_model"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Register"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Register fitted model for deployment\n",
-        "If neither `metric` nor `iteration` are specified in the `register_model` call, the iteration with the best primary metric is registered."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "description = 'AutoML Model'\n",
-        "tags = None\n",
-        "ml_run.register_model(description = description, tags = tags)\n",
-        "print(ml_run.model_id) # Use this id to deploy the model as a web service in Azure."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Register the Best Model for Any Given Metric"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "metric = 'AUC_weighted' # Replace with a metric name.\n",
-        "description = 'AutoML Model'\n",
-        "tags = None\n",
-        "ml_run.register_model(description = description, tags = tags, metric = metric)\n",
-        "print(ml_run.model_id) # Use this id to deploy the model as a web service in Azure."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Register the Model for Any Given Iteration"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "iteration = 1 # Replace with an iteration number.\n",
-        "description = 'AutoML Model'\n",
-        "tags = None\n",
-        "ml_run.register_model(description = description, tags = tags, iteration = iteration)\n",
-        "print(ml_run.model_id) # Use this id to deploy the model as a web service in Azure."
-      ]
-    }
-  ],
-  "metadata": {
-    "authors": [
-      {
-        "name": "savitam"
-      }
-    ],
-    "kernelspec": {
-      "display_name": "Python 3.6",
-      "language": "python",
-      "name": "python36"
-    },
-    "language_info": {
-      "codemirror_mode": {
-        "name": "ipython",
-        "version": 3
-      },
-      "file_extension": ".py",
-      "mimetype": "text/x-python",
-      "name": "python",
-      "nbconvert_exporter": "python",
-      "pygments_lexer": "ipython3",
-      "version": "3.6.6"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 2
-}

how-to-use-azureml/automated-machine-learning/forecasting-beer-remote/Beer_no_valid_split_test.csv

@@ -0,0 +1,20 @@
+DATE,grain,BeerProduction
+2017-01-01,grain,9049
+2017-02-01,grain,10458
+2017-03-01,grain,12489
+2017-04-01,grain,11499
+2017-05-01,grain,13553
+2017-06-01,grain,14740
+2017-07-01,grain,11424
+2017-08-01,grain,13412
+2017-09-01,grain,11917
+2017-10-01,grain,12721
+2017-11-01,grain,13272
+2017-12-01,grain,14278
+2018-01-01,grain,9572
+2018-02-01,grain,10423
+2018-03-01,grain,12667
+2018-04-01,grain,11904
+2018-05-01,grain,14120
+2018-06-01,grain,14565
+2018-07-01,grain,12622

how-to-use-azureml/automated-machine-learning/forecasting-beer-remote/Beer_no_valid_split_train.csv

@@ -0,0 +1,301 @@
+DATE,grain,BeerProduction
+1992-01-01,grain,3459
+1992-02-01,grain,3458
+1992-03-01,grain,4002
+1992-04-01,grain,4564
+1992-05-01,grain,4221
+1992-06-01,grain,4529
+1992-07-01,grain,4466
+1992-08-01,grain,4137
+1992-09-01,grain,4126
+1992-10-01,grain,4259
+1992-11-01,grain,4240
+1992-12-01,grain,4936
+1993-01-01,grain,3031
+1993-02-01,grain,3261
+1993-03-01,grain,4160
+1993-04-01,grain,4377
+1993-05-01,grain,4307
+1993-06-01,grain,4696
+1993-07-01,grain,4458
+1993-08-01,grain,4457
+1993-09-01,grain,4364
+1993-10-01,grain,4236
+1993-11-01,grain,4500
+1993-12-01,grain,4974
+1994-01-01,grain,3075
+1994-02-01,grain,3377
+1994-03-01,grain,4443
+1994-04-01,grain,4261
+1994-05-01,grain,4460
+1994-06-01,grain,4985
+1994-07-01,grain,4324
+1994-08-01,grain,4719
+1994-09-01,grain,4374
+1994-10-01,grain,4248
+1994-11-01,grain,4784
+1994-12-01,grain,4971
+1995-01-01,grain,3370
+1995-02-01,grain,3484
+1995-03-01,grain,4269
+1995-04-01,grain,3994
+1995-05-01,grain,4715
+1995-06-01,grain,4974
+1995-07-01,grain,4223
+1995-08-01,grain,5000
+1995-09-01,grain,4235
+1995-10-01,grain,4554
+1995-11-01,grain,4851
+1995-12-01,grain,4826
+1996-01-01,grain,3699
+1996-02-01,grain,3983
+1996-03-01,grain,4262
+1996-04-01,grain,4619
+1996-05-01,grain,5219
+1996-06-01,grain,4836
+1996-07-01,grain,4941
+1996-08-01,grain,5062
+1996-09-01,grain,4365
+1996-10-01,grain,5012
+1996-11-01,grain,4850
+1996-12-01,grain,5097
+1997-01-01,grain,3758
+1997-02-01,grain,3825
+1997-03-01,grain,4454
+1997-04-01,grain,4635
+1997-05-01,grain,5210
+1997-06-01,grain,5057
+1997-07-01,grain,5231
+1997-08-01,grain,5034
+1997-09-01,grain,4970
+1997-10-01,grain,5342
+1997-11-01,grain,4831
+1997-12-01,grain,5965
+1998-01-01,grain,3796
+1998-02-01,grain,4019
+1998-03-01,grain,4898
+1998-04-01,grain,5090
+1998-05-01,grain,5237
+1998-06-01,grain,5447
+1998-07-01,grain,5435
+1998-08-01,grain,5107
+1998-09-01,grain,5515
+1998-10-01,grain,5583
+1998-11-01,grain,5346
+1998-12-01,grain,6286
+1999-01-01,grain,4032
+1999-02-01,grain,4435
+1999-03-01,grain,5479
+1999-04-01,grain,5483
+1999-05-01,grain,5587
+1999-06-01,grain,6176
+1999-07-01,grain,5621
+1999-08-01,grain,5889
+1999-09-01,grain,5828
+1999-10-01,grain,5849
+1999-11-01,grain,6180
+1999-12-01,grain,6771
+2000-01-01,grain,4243
+2000-02-01,grain,4952
+2000-03-01,grain,6008
+2000-04-01,grain,5353
+2000-05-01,grain,6435
+2000-06-01,grain,6673
+2000-07-01,grain,5636
+2000-08-01,grain,6630
+2000-09-01,grain,5887
+2000-10-01,grain,6322
+2000-11-01,grain,6520
+2000-12-01,grain,6678
+2001-01-01,grain,5082
+2001-02-01,grain,5216
+2001-03-01,grain,5893
+2001-04-01,grain,5894
+2001-05-01,grain,6799
+2001-06-01,grain,6667
+2001-07-01,grain,6374
+2001-08-01,grain,6840
+2001-09-01,grain,5575
+2001-10-01,grain,6545
+2001-11-01,grain,6789
+2001-12-01,grain,7180
+2002-01-01,grain,5117
+2002-02-01,grain,5442
+2002-03-01,grain,6337
+2002-04-01,grain,6525
+2002-05-01,grain,7216
+2002-06-01,grain,6761
+2002-07-01,grain,6958
+2002-08-01,grain,7070
+2002-09-01,grain,6148
+2002-10-01,grain,6924
+2002-11-01,grain,6716
+2002-12-01,grain,7975
+2003-01-01,grain,5326
+2003-02-01,grain,5609
+2003-03-01,grain,6414
+2003-04-01,grain,6741
+2003-05-01,grain,7144
+2003-06-01,grain,7133
+2003-07-01,grain,7568
+2003-08-01,grain,7266
+2003-09-01,grain,6634
+2003-10-01,grain,7626
+2003-11-01,grain,6843
+2003-12-01,grain,8540
+2004-01-01,grain,5629
+2004-02-01,grain,5898
+2004-03-01,grain,7045
+2004-04-01,grain,7094
+2004-05-01,grain,7333
+2004-06-01,grain,7918
+2004-07-01,grain,7289
+2004-08-01,grain,7396
+2004-09-01,grain,7259
+2004-10-01,grain,7268
+2004-11-01,grain,7731
+2004-12-01,grain,9058
+2005-01-01,grain,5557
+2005-02-01,grain,6237
+2005-03-01,grain,7723
+2005-04-01,grain,7262
+2005-05-01,grain,8241
+2005-06-01,grain,8757
+2005-07-01,grain,7352
+2005-08-01,grain,8496
+2005-09-01,grain,7741
+2005-10-01,grain,7710
+2005-11-01,grain,8247
+2005-12-01,grain,8902
+2006-01-01,grain,6066
+2006-02-01,grain,6590
+2006-03-01,grain,7923
+2006-04-01,grain,7335
+2006-05-01,grain,8843
+2006-06-01,grain,9327
+2006-07-01,grain,7792
+2006-08-01,grain,9156
+2006-09-01,grain,8037
+2006-10-01,grain,8640
+2006-11-01,grain,9128
+2006-12-01,grain,9545
+2007-01-01,grain,6627
+2007-02-01,grain,6743
+2007-03-01,grain,8195
+2007-04-01,grain,7828
+2007-05-01,grain,9570
+2007-06-01,grain,9484
+2007-07-01,grain,8608
+2007-08-01,grain,9543
+2007-09-01,grain,8123
+2007-10-01,grain,9649
+2007-11-01,grain,9390
+2007-12-01,grain,10065
+2008-01-01,grain,7093
+2008-02-01,grain,7483
+2008-03-01,grain,8365
+2008-04-01,grain,8895
+2008-05-01,grain,9794
+2008-06-01,grain,9977
+2008-07-01,grain,9553
+2008-08-01,grain,9375
+2008-09-01,grain,9225
+2008-10-01,grain,9948
+2008-11-01,grain,8758
+2008-12-01,grain,10839
+2009-01-01,grain,7266
+2009-02-01,grain,7578
+2009-03-01,grain,8688
+2009-04-01,grain,9162
+2009-05-01,grain,9369
+2009-06-01,grain,10167
+2009-07-01,grain,9507
+2009-08-01,grain,8923
+2009-09-01,grain,9272
+2009-10-01,grain,9075
+2009-11-01,grain,8949
+2009-12-01,grain,10843
+2010-01-01,grain,6558
+2010-02-01,grain,7481
+2010-03-01,grain,9475
+2010-04-01,grain,9424
+2010-05-01,grain,9351
+2010-06-01,grain,10552
+2010-07-01,grain,9077
+2010-08-01,grain,9273
+2010-09-01,grain,9420
+2010-10-01,grain,9413
+2010-11-01,grain,9866
+2010-12-01,grain,11455
+2011-01-01,grain,6901
+2011-02-01,grain,8014
+2011-03-01,grain,9832
+2011-04-01,grain,9281
+2011-05-01,grain,9967
+2011-06-01,grain,11344
+2011-07-01,grain,9106
+2011-08-01,grain,10469
+2011-09-01,grain,10085
+2011-10-01,grain,9612
+2011-11-01,grain,10328
+2011-12-01,grain,11483
+2012-01-01,grain,7486
+2012-02-01,grain,8641
+2012-03-01,grain,9709
+2012-04-01,grain,9423
+2012-05-01,grain,11342
+2012-06-01,grain,11274
+2012-07-01,grain,9845
+2012-08-01,grain,11163
+2012-09-01,grain,9532
+2012-10-01,grain,10754
+2012-11-01,grain,10953
+2012-12-01,grain,11922
+2013-01-01,grain,8395
+2013-02-01,grain,8888
+2013-03-01,grain,10110
+2013-04-01,grain,10493
+2013-05-01,grain,12218
+2013-06-01,grain,11385
+2013-07-01,grain,11186
+2013-08-01,grain,11462
+2013-09-01,grain,10494
+2013-10-01,grain,11540
+2013-11-01,grain,11138
+2013-12-01,grain,12709
+2014-01-01,grain,8557
+2014-02-01,grain,9059
+2014-03-01,grain,10055
+2014-04-01,grain,10977
+2014-05-01,grain,11792
+2014-06-01,grain,11904
+2014-07-01,grain,10965
+2014-08-01,grain,10981
+2014-09-01,grain,10828
+2014-10-01,grain,11817
+2014-11-01,grain,10470
+2014-12-01,grain,13310
+2015-01-01,grain,8400
+2015-02-01,grain,9062
+2015-03-01,grain,10722
+2015-04-01,grain,11107
+2015-05-01,grain,11508
+2015-06-01,grain,12904
+2015-07-01,grain,11869
+2015-08-01,grain,11224
+2015-09-01,grain,12022
+2015-10-01,grain,11983
+2015-11-01,grain,11506
+2015-12-01,grain,14183
+2016-01-01,grain,8650
+2016-02-01,grain,10323
+2016-03-01,grain,12110
+2016-04-01,grain,11424
+2016-05-01,grain,12243
+2016-06-01,grain,13686
+2016-07-01,grain,10956
+2016-08-01,grain,12706
+2016-09-01,grain,12279
+2016-10-01,grain,11914
+2016-11-01,grain,13025
+2016-12-01,grain,14431

how-to-use-azureml/automated-machine-learning/forecasting-beer-remote/auto-ml-forecasting-beer-remote.ipynb

@@ -0,0 +1,663 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "source": [
+        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
+        "\n",
+        "Licensed under the MIT License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "source": [
+        "![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/automated-machine-learning/forecasting-beer-remote/auto-ml-forecasting-beer-remote.png)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "source": [
+        "# Automated Machine Learning\n",
+        "**Beer Production Forecasting**\n",
+        "\n",
+        "## Contents\n",
+        "1. [Introduction](#Introduction)\n",
+        "1. [Setup](#Setup)\n",
+        "1. [Data](#Data)\n",
+        "1. [Train](#Train)\n",
+        "1. [Evaluate](#Evaluate)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "source": [
+        "## Introduction\n",
+        "This notebook demonstrates demand forecasting for Beer Production Dataset using AutoML.\n",
+        "\n",
+        "AutoML highlights here include using Deep Learning forecasts, Arima, Prophet,  Remote Execution and Remote Inferencing, and working with the `forecast` function. Please also look at the additional forecasting notebooks, which document lagging, rolling windows, forecast quantiles, other ways to use the forecast function, and forecaster deployment.\n",
+        "\n",
+        "Make sure you have executed the [configuration](../../../configuration.ipynb) before running this notebook.\n",
+        "\n",
+        "An Enterprise workspace is required for this notebook. To learn more about creating an Enterprise workspace or upgrading to an Enterprise workspace from the Azure portal, please visit our [Workspace page.](https://docs.microsoft.com/azure/machine-learning/service/concept-workspace#upgrade)\n",
+        "\n",
+        "Notebook synopsis:\n",
+        "1. Creating an Experiment in an existing Workspace\n",
+        "2. Configuration and remote run of AutoML for a time-series model exploring Regression learners, Arima, Prophet and DNNs\n",
+        "4. Evaluating the fitted model using a rolling test "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "source": [
+        "## Setup\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "import azureml.core\n",
+        "import pandas as pd\n",
+        "import numpy as np\n",
+        "import logging\n",
+        "import warnings\n",
+        "\n",
+        "from pandas.tseries.frequencies import to_offset\n",
+        "\n",
+        "# Squash warning messages for cleaner output in the notebook\n",
+        "warnings.showwarning = lambda *args, **kwargs: None\n",
+        "\n",
+        "from azureml.core.workspace import Workspace\n",
+        "from azureml.core.experiment import Experiment\n",
+        "from azureml.train.automl import AutoMLConfig\n",
+        "from matplotlib import pyplot as plt\n",
+        "from sklearn.metrics import mean_absolute_error, mean_squared_error\n",
+        "from azureml.train.estimator import Estimator"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "source": [
+        "As part of the setup you have already created a <b>Workspace</b>. To run AutoML, you also need to create an <b>Experiment</b>. An Experiment corresponds to a prediction problem you are trying to solve, while a Run corresponds to a specific approach to the problem."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "outputs": [],
+      "source": [
+        "ws = Workspace.from_config()\n",
+        "\n",
+        "# choose a name for the run history container in the workspace\n",
+        "experiment_name = 'beer-remote-cpu'\n",
+        "\n",
+        "experiment = Experiment(ws, experiment_name)\n",
+        "\n",
+        "output = {}\n",
+        "output['SDK version'] = azureml.core.VERSION\n",
+        "output['Subscription ID'] = ws.subscription_id\n",
+        "output['Workspace'] = ws.name\n",
+        "output['Resource Group'] = ws.resource_group\n",
+        "output['Location'] = ws.location\n",
+        "output['Run History Name'] = experiment_name\n",
+        "pd.set_option('display.max_colwidth', -1)\n",
+        "outputDf = pd.DataFrame(data = output, index = [''])\n",
+        "outputDf.T"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "source": [
+        "### Using AmlCompute\n",
+        "You will need to create a [compute target](https://docs.microsoft.com/azure/machine-learning/service/concept-azure-machine-learning-architecture#compute-target) for your AutoML run. In this tutorial, you use `AmlCompute` as your training compute resource."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "outputs": [],
+      "source": [
+        "from azureml.core.compute import ComputeTarget, AmlCompute\n",
+        "from azureml.core.compute_target import ComputeTargetException\n",
+        "\n",
+        "# Choose a name for your CPU cluster\n",
+        "cpu_cluster_name = \"cpu-cluster\"\n",
+        "\n",
+        "# Verify that cluster does not exist already\n",
+        "try:\n",
+        "    compute_target = ComputeTarget(workspace=ws, name=cpu_cluster_name)\n",
+        "    print('Found existing cluster, use it.')\n",
+        "except ComputeTargetException:\n",
+        "    compute_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_D2_V2',\n",
+        "                                                           max_nodes=4)\n",
+        "    compute_target = ComputeTarget.create(ws, cpu_cluster_name, compute_config)\n",
+        "\n",
+        "compute_target.wait_for_completion(show_output=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "source": [
+        "## Data\n",
+        "Read Beer demand data from file, and preview data."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "source": [
+        "Let's set up what we know about the dataset. \n",
+        "\n",
+        "**Target column** is what we want to forecast.\n",
+        "\n",
+        "**Time column** is the time axis along which to predict.\n",
+        "\n",
+        "**Grain** is another word for an individual time series in your dataset. Grains are identified by values of the columns listed `grain_column_names`, for example \"store\" and \"item\" if your data has multiple time series of sales, one series for each combination of store and item sold.\n",
+        "\n",
+        "This dataset has only one time series. Please see the [orange juice notebook](https://github.com/Azure/MachineLearningNotebooks/tree/master/how-to-use-azureml/automated-machine-learning/forecasting-orange-juice-sales) for an example of a multi-time series dataset."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "outputs": [],
+      "source": [
+        "import pandas as pd\n",
+        "from pandas import DataFrame\n",
+        "from pandas import Grouper\n",
+        "from matplotlib import pyplot\n",
+        "from pandas import concat\n",
+        "from matplotlib import pyplot\n",
+        "from pandas.plotting import register_matplotlib_converters\n",
+        "register_matplotlib_converters()\n",
+        "plt.tight_layout()\n",
+        "plt.figure(figsize=(20, 10))\n",
+        "\n",
+        "plt.subplot(2, 1, 1)\n",
+        "plt.title('Beer Production By Year')\n",
+        "df = pd.read_csv(\"Beer_no_valid_split_train.csv\", parse_dates=True, index_col= 'DATE').drop(columns='grain')\n",
+        "test_df = pd.read_csv(\"Beer_no_valid_split_test.csv\", parse_dates=True, index_col= 'DATE').drop(columns='grain')\n",
+        "pyplot.plot(df)\n",
+        "\n",
+        "plt.subplot(2, 1, 2)\n",
+        "plt.title('Beer Production By Month')\n",
+        "groups = df.groupby(df.index.month)\n",
+        "months = concat([DataFrame(x[1].values) for x in groups], axis=1)\n",
+        "months = DataFrame(months)\n",
+        "months.columns = range(1,13)\n",
+        "months.boxplot()\n",
+        "pyplot.show()\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "outputs": [],
+      "source": [
+        "target_column_name = 'BeerProduction'\n",
+        "time_column_name = 'DATE'\n",
+        "grain_column_names = []\n",
+        "freq = 'M' #Monthly data"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Split Training data into Train and Validation set and Upload to Datastores"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "outputs": [],
+      "source": [
+        "from helper import split_fraction_by_grain\n",
+        "from helper import split_full_for_forecasting\n",
+        "\n",
+        "train, valid = split_full_for_forecasting(df, time_column_name)\n",
+        "train.to_csv(\"train.csv\")\n",
+        "valid.to_csv(\"valid.csv\")\n",
+        "test_df.to_csv(\"test.csv\")\n",
+        "\n",
+        "datastore = ws.get_default_datastore()\n",
+        "datastore.upload_files(files = ['./train.csv'], target_path = 'beer-dataset/tabular/', overwrite = True,show_progress = True)\n",
+        "datastore.upload_files(files = ['./valid.csv'], target_path = 'beer-dataset/tabular/', overwrite = True,show_progress = True)\n",
+        "datastore.upload_files(files = ['./test.csv'], target_path = 'beer-dataset/tabular/', overwrite = True,show_progress = True)\n",
+        "\n",
+        "from azureml.core import Dataset\n",
+        "train_dataset = Dataset.Tabular.from_delimited_files(path = [(datastore, 'beer-dataset/tabular/train.csv')])\n",
+        "valid_dataset = Dataset.Tabular.from_delimited_files(path = [(datastore, 'beer-dataset/tabular/valid.csv')])\n",
+        "test_dataset = Dataset.Tabular.from_delimited_files(path = [(datastore, 'beer-dataset/tabular/test.csv')])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "source": [
+        "### Setting forecaster maximum horizon \n",
+        "\n",
+        "The forecast horizon is the number of periods into the future that the model should predict. Here, we set the horizon to 12 periods (i.e. 12 months). Notice that this is much shorter than the number of months in the test set; we will need to use a rolling test to evaluate the performance on the whole test set. For more discussion of forecast horizons and guiding principles for setting them, please see the [energy demand notebook](https://github.com/Azure/MachineLearningNotebooks/tree/master/how-to-use-azureml/automated-machine-learning/forecasting-energy-demand).  "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "outputs": [],
+      "source": [
+        "max_horizon = 12"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "source": [
+        "## Train\n",
+        "\n",
+        "Instantiate a AutoMLConfig object. This defines the settings and data used to run the experiment.\n",
+        "\n",
+        "|Property|Description|\n",
+        "|-|-|\n",
+        "|**task**|forecasting|\n",
+        "|**primary_metric**|This is the metric that you want to optimize.<br> Forecasting supports the following primary metrics <br><i>spearman_correlation</i><br><i>normalized_root_mean_squared_error</i><br><i>r2_score</i><br><i>normalized_mean_absolute_error</i>\n",
+        "|**iteration_timeout_minutes**|Time limit in minutes for each iteration.|\n",
+        "|**training_data**|Input dataset, containing both features and label column.|\n",
+        "|**label_column_name**|The name of the label column.|\n",
+        "|**enable_dnn**|Enable Forecasting DNNs|\n",
+        "\n",
+        "This notebook uses the blacklist_models parameter to exclude some models that take a longer time to train on this dataset. You can choose to remove models from the blacklist_models list but you may need to increase the iteration_timeout_minutes parameter value to get results.\n",
+        "\n",
+        "This step requires an Enterprise workspace to gain access to this feature. To learn more about creating an Enterprise workspace or upgrading to an Enterprise workspace from the Azure portal, please visit our [Workspace page.](https://docs.microsoft.com/azure/machine-learning/service/concept-workspace#upgrade)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "outputs": [],
+      "source": [
+        "automl_settings = {\n",
+        "    'time_column_name': time_column_name,\n",
+        "    'max_horizon': max_horizon,\n",
+        "    'enable_dnn' : True,\n",
+        "}\n",
+        "\n",
+        "automl_config = AutoMLConfig(task='forecasting',                             \n",
+        "                             primary_metric='normalized_root_mean_squared_error',\n",
+        "                             experiment_timeout_minutes = 60,\n",
+        "                             training_data=train_dataset,\n",
+        "                             label_column_name=target_column_name,\n",
+        "                             validation_data=valid_dataset, \n",
+        "                             verbosity=logging.INFO,\n",
+        "                             compute_target=compute_target,\n",
+        "                             max_concurrent_iterations=4,\n",
+        "                             max_cores_per_iteration=-1,\n",
+        "                            **automl_settings)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "source": [
+        "We will now run the experiment, starting with 10 iterations of model search. The experiment can be continued for more iterations if more accurate results are required."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "outputs": [],
+      "source": [
+        "remote_run = experiment.submit(automl_config, show_output= False)\n",
+        "remote_run"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "outputs": [],
+      "source": [
+        "# If you need to retrieve a run that already started, use the following code\n",
+        "# from azureml.train.automl.run import AutoMLRun\n",
+        "# remote_run = AutoMLRun(experiment = experiment, run_id = '<replace with your run id>')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "remote_run.wait_for_completion()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "source": [
+        "Displaying the run objects gives you links to the visual tools in the Azure Portal. Go try them!"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "source": [
+        "### Retrieve the Best Model for Each Algorithm\n",
+        "Below we select the best pipeline from our iterations. The get_output method on automl_classifier returns the best run and the fitted model for the last fit invocation. There are overloads on get_output that allow you to retrieve the best run and fitted model for any logged metric or a particular iteration."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "outputs": [],
+      "source": [
+        "from helper import get_result_df\n",
+        "summary_df = get_result_df(remote_run)\n",
+        "summary_df"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "outputs": [],
+      "source": [
+        "from azureml.core.run import Run\n",
+        "from azureml.widgets import RunDetails\n",
+        "forecast_model = 'TCNForecaster'\n",
+        "if not forecast_model in summary_df['run_id']:\n",
+        "    forecast_model = 'ForecastTCN'\n",
+        "    \n",
+        "best_dnn_run_id = summary_df['run_id'][forecast_model]\n",
+        "best_dnn_run = Run(experiment, best_dnn_run_id)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "outputs": [],
+      "source": [
+        "best_dnn_run.parent\n",
+        "RunDetails(best_dnn_run.parent).show() "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "outputs": [],
+      "source": [
+        "best_dnn_run\n",
+        "RunDetails(best_dnn_run).show() "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "source": [
+        "## Evaluate on Test Data"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "source": [
+        "We now use the best fitted model from the AutoML Run to make forecasts for the test set.  \n",
+        "\n",
+        "We always score on the original dataset whose schema matches the training set schema."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "outputs": [],
+      "source": [
+        "from azureml.core import Dataset\n",
+        "test_dataset = Dataset.Tabular.from_delimited_files(path = [(datastore, 'beer-dataset/tabular/test.csv')])\n",
+        "# preview the first 3 rows of the dataset\n",
+        "test_dataset.take(5).to_pandas_dataframe()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "compute_target = ws.compute_targets['cpu-cluster']\n",
+        "test_experiment = Experiment(ws, experiment_name + \"_test\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "import shutil\n",
+        "\n",
+        "script_folder = os.path.join(os.getcwd(), 'inference')\n",
+        "os.makedirs(script_folder, exist_ok=True)\n",
+        "shutil.copy2('infer.py', script_folder)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from helper import run_inference\n",
+        "\n",
+        "test_run = run_inference(test_experiment, compute_target, script_folder, best_dnn_run, test_dataset, valid_dataset, max_horizon,\n",
+        "                 target_column_name, time_column_name, freq)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "RunDetails(test_run).show()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from helper import run_multiple_inferences\n",
+        "\n",
+        "summary_df = run_multiple_inferences(summary_df, experiment, test_experiment, compute_target, script_folder, test_dataset, \n",
+        "                  valid_dataset, max_horizon, target_column_name, time_column_name, freq)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "outputs": [],
+      "source": [
+        "for run_name, run_summary in summary_df.iterrows():\n",
+        "    print(run_name)\n",
+        "    print(run_summary)\n",
+        "    run_id = run_summary.run_id\n",
+        "    test_run_id = run_summary.test_run_id\n",
+        "    test_run = Run(test_experiment, test_run_id)\n",
+        "    test_run.wait_for_completion()\n",
+        "    test_score = test_run.get_metrics()[run_summary.primary_metric]\n",
+        "    summary_df.loc[summary_df.run_id == run_id, 'Test Score'] = test_score\n",
+        "    print(\"Test Score: \", test_score)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "hideCode": false,
+        "hidePrompt": false
+      },
+      "outputs": [],
+      "source": [
+        "summary_df"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": []
+    }
+  ],
+  "metadata": {
+    "authors": [
+      {
+        "name": "omkarm"
+      }
+    ],
+    "hide_code_all_hidden": false,
+    "kernelspec": {
+      "display_name": "Python 3.6",
+      "language": "python",
+      "name": "python36"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.6.7"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 2
+}

how-to-use-azureml/automated-machine-learning/forecasting-beer-remote/auto-ml-forecasting-beer-remote.yml

@@ -0,0 +1,12 @@
+name: auto-ml-forecasting-beer-remote
+dependencies:
+- fbprophet==0.5
+- py-xgboost<=0.80
+- pip:
+  - azureml-sdk
+  - azureml-train-automl
+  - azureml-train
+  - azureml-widgets
+  - matplotlib
+  - pandas_ml
+  - statsmodels

how-to-use-azureml/automated-machine-learning/forecasting-beer-remote/helper.py

@@ -0,0 +1,133 @@
+import pandas as pd
+from azureml.core import Environment
+from azureml.core.conda_dependencies import CondaDependencies
+from azureml.train.estimator import Estimator
+from azureml.core.run import Run
+
+
+def split_fraction_by_grain(df, fraction, time_column_name,
+                            grain_column_names=None):
+
+    if not grain_column_names:
+        df['tmp_grain_column'] = 'grain'
+        grain_column_names = ['tmp_grain_column']
+
+    """Group df by grain and split on last n rows for each group."""
+    df_grouped = (df.sort_values(time_column_name)
+                  .groupby(grain_column_names, group_keys=False))
+
+    df_head = df_grouped.apply(lambda dfg: dfg.iloc[:-int(len(dfg) *
+                               fraction)] if fraction > 0 else dfg)
+
+    df_tail = df_grouped.apply(lambda dfg: dfg.iloc[-int(len(dfg) *
+                               fraction):] if fraction > 0 else dfg[:0])
+
+    if 'tmp_grain_column' in grain_column_names:
+        for df2 in (df, df_head, df_tail):
+            df2.drop('tmp_grain_column', axis=1, inplace=True)
+
+        grain_column_names.remove('tmp_grain_column')
+
+    return df_head, df_tail
+
+
+def split_full_for_forecasting(df, time_column_name,
+                               grain_column_names=None, test_split=0.2):
+    index_name = df.index.name
+
+    # Assumes that there isn't already a column called tmpindex
+
+    df['tmpindex'] = df.index
+
+    train_df, test_df = split_fraction_by_grain(
+        df, test_split, time_column_name, grain_column_names)
+
+    train_df = train_df.set_index('tmpindex')
+    train_df.index.name = index_name
+
+    test_df = test_df.set_index('tmpindex')
+    test_df.index.name = index_name
+
+    df.drop('tmpindex', axis=1, inplace=True)
+
+    return train_df, test_df
+
+
+def get_result_df(remote_run):
+    children = list(remote_run.get_children(recursive=True))
+    summary_df = pd.DataFrame(index=['run_id', 'run_algorithm',
+                                     'primary_metric', 'Score'])
+    goal_minimize = False
+    for run in children:
+        if('run_algorithm' in run.properties and 'score' in run.properties):
+            summary_df[run.id] = [run.id, run.properties['run_algorithm'],
+                                  run.properties['primary_metric'],
+                                  float(run.properties['score'])]
+            if('goal' in run.properties):
+                goal_minimize = run.properties['goal'].split('_')[-1] == 'min'
+
+    summary_df = summary_df.T.sort_values(
+        'Score',
+        ascending=goal_minimize).drop_duplicates(['run_algorithm'])
+    summary_df = summary_df.set_index('run_algorithm')
+    return summary_df
+
+
+def run_inference(test_experiment, compute_target, script_folder, train_run,
+                  test_dataset, lookback_dataset, max_horizon,
+                  target_column_name, time_column_name, freq):
+    train_run.download_file('outputs/model.pkl', 'inference/model.pkl')
+    train_run.download_file('outputs/conda_env_v_1_0_0.yml',
+                            'inference/condafile.yml')
+
+    inference_env = Environment("myenv")
+    inference_env.docker.enabled = True
+    inference_env.python.conda_dependencies = CondaDependencies(
+        conda_dependencies_file_path='inference/condafile.yml')
+
+    est = Estimator(source_directory=script_folder,
+                    entry_script='infer.py',
+                    script_params={
+                        '--max_horizon': max_horizon,
+                        '--target_column_name': target_column_name,
+                        '--time_column_name': time_column_name,
+                        '--frequency': freq
+                    },
+                    inputs=[test_dataset.as_named_input('test_data'),
+                            lookback_dataset.as_named_input('lookback_data')],
+                    compute_target=compute_target,
+                    environment_definition=inference_env)
+
+    run = test_experiment.submit(
+        est, tags={
+            'training_run_id': train_run.id,
+            'run_algorithm': train_run.properties['run_algorithm'],
+            'valid_score': train_run.properties['score'],
+            'primary_metric': train_run.properties['primary_metric']
+        })
+
+    run.log("run_algorithm", run.tags['run_algorithm'])
+    return run
+
+
+def run_multiple_inferences(summary_df, train_experiment, test_experiment,
+                            compute_target, script_folder, test_dataset,
+                            lookback_dataset, max_horizon, target_column_name,
+                            time_column_name, freq):
+
+    for run_name, run_summary in summary_df.iterrows():
+        print(run_name)
+        print(run_summary)
+        run_id = run_summary.run_id
+        train_run = Run(train_experiment, run_id)
+
+        test_run = run_inference(
+            test_experiment, compute_target, script_folder, train_run,
+            test_dataset, lookback_dataset, max_horizon, target_column_name,
+            time_column_name, freq)
+
+        print(test_run)
+        summary_df.loc[summary_df.run_id == run_id,
+                       'test_run_id'] = test_run.id
+
+    return summary_df

how-to-use-azureml/automated-machine-learning/forecasting-beer-remote/infer.py

@@ -0,0 +1,319 @@
+import pandas as pd
+import numpy as np
+import argparse
+from azureml.core import Run
+from sklearn.externals import joblib
+from sklearn.metrics import mean_absolute_error, mean_squared_error
+from azureml.automl.core._vendor.automl.client.core.common import metrics
+from automl.client.core.common import constants
+from pandas.tseries.frequencies import to_offset
+
+
+def align_outputs(y_predicted, X_trans, X_test, y_test,
+                  predicted_column_name='predicted',
+                  horizon_colname='horizon_origin'):
+    """
+    Demonstrates how to get the output aligned to the inputs
+    using pandas indexes. Helps understand what happened if
+    the output's shape differs from the input shape, or if
+    the data got re-sorted by time and grain during forecasting.
+
+    Typical causes of misalignment are:
+    * we predicted some periods that were missing in actuals -> drop from eval
+    * model was asked to predict past max_horizon -> increase max horizon
+    * data at start of X_test was needed for lags -> provide previous periods
+    """
+    if (horizon_colname in X_trans):
+        df_fcst = pd.DataFrame({predicted_column_name: y_predicted,
+                                horizon_colname: X_trans[horizon_colname]})
+    else:
+        df_fcst = pd.DataFrame({predicted_column_name: y_predicted})
+
+    # y and X outputs are aligned by forecast() function contract
+    df_fcst.index = X_trans.index
+
+    # align original X_test to y_test
+    X_test_full = X_test.copy()
+    X_test_full[target_column_name] = y_test
+
+    # X_test_full's index does not include origin, so reset for merge
+    df_fcst.reset_index(inplace=True)
+    X_test_full = X_test_full.reset_index().drop(columns='index')
+    together = df_fcst.merge(X_test_full, how='right')
+
+    # drop rows where prediction or actuals are nan
+    # happens because of missing actuals
+    # or at edges of time due to lags/rolling windows
+    clean = together[together[[target_column_name,
+                               predicted_column_name]].notnull().all(axis=1)]
+    return(clean)
+
+
+def do_rolling_forecast_with_lookback(fitted_model, X_test, y_test,
+                                      max_horizon, X_lookback, y_lookback,
+                                      freq='D'):
+    """
+    Produce forecasts on a rolling origin over the given test set.
+
+    Each iteration makes a forecast for the next 'max_horizon' periods
+    with respect to the current origin, then advances the origin by the
+    horizon time duration. The prediction context for each forecast is set so
+    that the forecaster uses the actual target values prior to the current
+    origin time for constructing lag features.
+
+    This function returns a concatenated DataFrame of rolling forecasts.
+     """
+    print("Using lookback of size: ", y_lookback.size)
+    df_list = []
+    origin_time = X_test[time_column_name].min()
+    X = X_lookback.append(X_test)
+    y = np.concatenate((y_lookback, y_test), axis=0)
+    while origin_time <= X_test[time_column_name].max():
+        # Set the horizon time - end date of the forecast
+        horizon_time = origin_time + max_horizon * to_offset(freq)
+
+        # Extract test data from an expanding window up-to the horizon
+        expand_wind = (X[time_column_name] < horizon_time)
+        X_test_expand = X[expand_wind]
+        y_query_expand = np.zeros(len(X_test_expand)).astype(np.float)
+        y_query_expand.fill(np.NaN)
+
+        if origin_time != X[time_column_name].min():
+            # Set the context by including actuals up-to the origin time
+            test_context_expand_wind = (X[time_column_name] < origin_time)
+            context_expand_wind = (
+                X_test_expand[time_column_name] < origin_time)
+            y_query_expand[context_expand_wind] = y[test_context_expand_wind]
+
+        # Print some debug info
+        print("Horizon_time:", horizon_time,
+              " origin_time: ", origin_time,
+              " max_horizon: ", max_horizon,
+              " freq: ", freq)
+        print("expand_wind: ", expand_wind)
+        print("y_query_expand")
+        print(y_query_expand)
+        print("X_test")
+        print(X)
+        print("X_test_expand")
+        print(X_test_expand)
+        print("Type of X_test_expand: ", type(X_test_expand))
+        print("Type of y_query_expand: ", type(y_query_expand))
+
+        print("y_query_expand")
+        print(y_query_expand)
+
+        # Make a forecast out to the maximum horizon
+        # y_fcst, X_trans = y_query_expand, X_test_expand
+        y_fcst, X_trans = fitted_model.forecast(X_test_expand, y_query_expand)
+
+        print("y_fcst")
+        print(y_fcst)
+
+        # Align forecast with test set for dates within
+        # the current rolling window
+        trans_tindex = X_trans.index.get_level_values(time_column_name)
+        trans_roll_wind = (trans_tindex >= origin_time) & (
+            trans_tindex < horizon_time)
+        test_roll_wind = expand_wind & (X[time_column_name] >= origin_time)
+        df_list.append(align_outputs(
+            y_fcst[trans_roll_wind], X_trans[trans_roll_wind],
+            X[test_roll_wind], y[test_roll_wind]))
+
+        # Advance the origin time
+        origin_time = horizon_time
+
+    return pd.concat(df_list, ignore_index=True)
+
+
+def do_rolling_forecast(fitted_model, X_test, y_test, max_horizon, freq='D'):
+    """
+    Produce forecasts on a rolling origin over the given test set.
+
+    Each iteration makes a forecast for the next 'max_horizon' periods
+    with respect to the current origin, then advances the origin by the
+    horizon time duration. The prediction context for each forecast is set so
+    that the forecaster uses the actual target values prior to the current
+    origin time for constructing lag features.
+
+    This function returns a concatenated DataFrame of rolling forecasts.
+     """
+    df_list = []
+    origin_time = X_test[time_column_name].min()
+    while origin_time <= X_test[time_column_name].max():
+        # Set the horizon time - end date of the forecast
+        horizon_time = origin_time + max_horizon * to_offset(freq)
+
+        # Extract test data from an expanding window up-to the horizon
+        expand_wind = (X_test[time_column_name] < horizon_time)
+        X_test_expand = X_test[expand_wind]
+        y_query_expand = np.zeros(len(X_test_expand)).astype(np.float)
+        y_query_expand.fill(np.NaN)
+
+        if origin_time != X_test[time_column_name].min():
+            # Set the context by including actuals up-to the origin time
+            test_context_expand_wind = (X_test[time_column_name] < origin_time)
+            context_expand_wind = (
+                X_test_expand[time_column_name] < origin_time)
+            y_query_expand[context_expand_wind] = y_test[
+                test_context_expand_wind]
+
+        # Print some debug info
+        print("Horizon_time:", horizon_time,
+              " origin_time: ", origin_time,
+              " max_horizon: ", max_horizon,
+              " freq: ", freq)
+        print("expand_wind: ", expand_wind)
+        print("y_query_expand")
+        print(y_query_expand)
+        print("X_test")
+        print(X_test)
+        print("X_test_expand")
+        print(X_test_expand)
+        print("Type of X_test_expand: ", type(X_test_expand))
+        print("Type of y_query_expand: ", type(y_query_expand))
+        print("y_query_expand")
+        print(y_query_expand)
+
+        # Make a forecast out to the maximum horizon
+        y_fcst, X_trans = fitted_model.forecast(X_test_expand, y_query_expand)
+
+        print("y_fcst")
+        print(y_fcst)
+
+        # Align forecast with test set for dates within the
+        # current rolling window
+        trans_tindex = X_trans.index.get_level_values(time_column_name)
+        trans_roll_wind = (trans_tindex >= origin_time) & (
+            trans_tindex < horizon_time)
+        test_roll_wind = expand_wind & (
+            X_test[time_column_name] >= origin_time)
+        df_list.append(align_outputs(y_fcst[trans_roll_wind],
+                                     X_trans[trans_roll_wind],
+                                     X_test[test_roll_wind],
+                                     y_test[test_roll_wind]))
+
+        # Advance the origin time
+        origin_time = horizon_time
+
+    return pd.concat(df_list, ignore_index=True)
+
+
+def APE(actual, pred):
+    """
+    Calculate absolute percentage error.
+    Returns a vector of APE values with same length as actual/pred.
+    """
+    return 100 * np.abs((actual - pred) / actual)
+
+
+def MAPE(actual, pred):
+    """
+    Calculate mean absolute percentage error.
+    Remove NA and values where actual is close to zero
+    """
+    not_na = ~(np.isnan(actual) | np.isnan(pred))
+    not_zero = ~np.isclose(actual, 0.0)
+    actual_safe = actual[not_na & not_zero]
+    pred_safe = pred[not_na & not_zero]
+    return np.mean(APE(actual_safe, pred_safe))
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    '--max_horizon', type=int, dest='max_horizon',
+    default=10, help='Max Horizon for forecasting')
+parser.add_argument(
+    '--target_column_name', type=str, dest='target_column_name',
+    help='Target Column Name')
+parser.add_argument(
+    '--time_column_name', type=str, dest='time_column_name',
+    help='Time Column Name')
+parser.add_argument(
+    '--frequency', type=str, dest='freq',
+    help='Frequency of prediction')
+
+
+args = parser.parse_args()
+max_horizon = args.max_horizon
+target_column_name = args.target_column_name
+time_column_name = args.time_column_name
+freq = args.freq
+
+
+print('args passed are: ')
+print(max_horizon)
+print(target_column_name)
+print(time_column_name)
+print(freq)
+
+run = Run.get_context()
+# get input dataset by name
+test_dataset = run.input_datasets['test_data']
+lookback_dataset = run.input_datasets['lookback_data']
+
+grain_column_names = []
+
+df = test_dataset.to_pandas_dataframe()
+
+print('Read df')
+print(df)
+
+X_test_df = test_dataset.drop_columns(columns=[target_column_name])
+y_test_df = test_dataset.with_timestamp_columns(
+    None).keep_columns(columns=[target_column_name])
+
+X_lookback_df = lookback_dataset.drop_columns(columns=[target_column_name])
+y_lookback_df = lookback_dataset.with_timestamp_columns(
+    None).keep_columns(columns=[target_column_name])
+
+fitted_model = joblib.load('model.pkl')
+
+if hasattr(fitted_model, 'get_lookback'):
+    lookback = fitted_model.get_lookback()
+    df_all = do_rolling_forecast_with_lookback(
+        fitted_model,
+        X_test_df.to_pandas_dataframe(),
+        y_test_df.to_pandas_dataframe().values.T[0],
+        max_horizon,
+        X_lookback_df.to_pandas_dataframe()[-lookback:],
+        y_lookback_df.to_pandas_dataframe().values.T[0][-lookback:],
+        freq)
+else:
+    df_all = do_rolling_forecast(
+        fitted_model,
+        X_test_df.to_pandas_dataframe(),
+        y_test_df.to_pandas_dataframe().values.T[0],
+        max_horizon,
+        freq)
+
+print(df_all)
+
+print("target values:::")
+print(df_all[target_column_name])
+print("predicted values:::")
+print(df_all['predicted'])
+
+# use automl metrics module
+scores = metrics.compute_metrics_regression(
+    df_all['predicted'],
+    df_all[target_column_name],
+    list(constants.Metric.SCALAR_REGRESSION_SET),
+    None, None, None)
+
+print("scores:")
+print(scores)
+
+for key, value in scores.items():
+    run.log(key, value)
+
+print("Simple forecasting model")
+rmse = np.sqrt(mean_squared_error(
+    df_all[target_column_name], df_all['predicted']))
+print("[Test Data] \nRoot Mean squared error: %.2f" % rmse)
+mae = mean_absolute_error(df_all[target_column_name], df_all['predicted'])
+print('mean_absolute_error score: %.2f' % mae)
+print('MAPE: %.2f' % MAPE(df_all[target_column_name], df_all['predicted']))
+
+run.log('rmse', rmse)
+run.log('mae', mae)

how-to-use-azureml/automated-machine-learning/forecasting-bike-share/auto-ml-forecasting-bike-share.ipynb

@@ -0,0 +1,633 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
+        "\n",
+        "Licensed under the MIT License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/automated-machine-learning/forecasting-bike-share/auto-ml-forecasting-bike-share.png)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# Automated Machine Learning\n",
+        "**BikeShare Demand Forecasting**\n",
+        "\n",
+        "## Contents\n",
+        "1. [Introduction](#Introduction)\n",
+        "1. [Setup](#Setup)\n",
+        "1. [Compute](#Compute)\n",
+        "1. [Data](#Data)\n",
+        "1. [Train](#Train)\n",
+        "1. [Featurization](#Featurization)\n",
+        "1. [Evaluate](#Evaluate)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Introduction\n",
+        "This notebook demonstrates demand forecasting for a bike-sharing service using AutoML.\n",
+        "\n",
+        "AutoML highlights here include built-in holiday featurization, accessing engineered feature names, and working with the `forecast` function. Please also look at the additional forecasting notebooks, which document lagging, rolling windows, forecast quantiles, other ways to use the forecast function, and forecaster deployment.\n",
+        "\n",
+        "Make sure you have executed the [configuration notebook](../../../configuration.ipynb) before running this notebook.\n",
+        "\n",
+        "Notebook synopsis:\n",
+        "1. Creating an Experiment in an existing Workspace\n",
+        "2. Configuration and local run of AutoML for a time-series model with lag and holiday features \n",
+        "3. Viewing the engineered names for featurized data and featurization summary for all raw features\n",
+        "4. Evaluating the fitted model using a rolling test "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Setup\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import azureml.core\n",
+        "import pandas as pd\n",
+        "import numpy as np\n",
+        "import logging\n",
+        "\n",
+        "from azureml.core import Workspace, Experiment, Dataset\n",
+        "from azureml.train.automl import AutoMLConfig\n",
+        "from datetime import datetime"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "As part of the setup you have already created a <b>Workspace</b>. To run AutoML, you also need to create an <b>Experiment</b>. An Experiment corresponds to a prediction problem you are trying to solve, while a Run corresponds to a specific approach to the problem."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "ws = Workspace.from_config()\n",
+        "\n",
+        "# choose a name for the run history container in the workspace\n",
+        "experiment_name = 'automl-bikeshareforecasting'\n",
+        "\n",
+        "experiment = Experiment(ws, experiment_name)\n",
+        "\n",
+        "output = {}\n",
+        "output['SDK version'] = azureml.core.VERSION\n",
+        "output['Subscription ID'] = ws.subscription_id\n",
+        "output['Workspace'] = ws.name\n",
+        "output['SKU'] = ws.sku\n",
+        "output['Resource Group'] = ws.resource_group\n",
+        "output['Location'] = ws.location\n",
+        "output['Run History Name'] = experiment_name\n",
+        "pd.set_option('display.max_colwidth', -1)\n",
+        "outputDf = pd.DataFrame(data = output, index = [''])\n",
+        "outputDf.T"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Compute\n",
+        "You will need to create a [compute target](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-set-up-training-targets#amlcompute) for your AutoML run. In this tutorial, you create AmlCompute as your training compute resource.\n",
+        "#### Creation of AmlCompute takes approximately 5 minutes. \n",
+        "If the AmlCompute with that name is already in your workspace this code will skip the creation process.\n",
+        "As with other Azure services, there are limits on certain resources (e.g. AmlCompute) associated with the Azure Machine Learning service. Please read this article on the default limits and how to request more quota."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.compute import AmlCompute\n",
+        "from azureml.core.compute import ComputeTarget\n",
+        "\n",
+        "# Choose a name for your cluster.\n",
+        "amlcompute_cluster_name = \"cpu-cluster-bike\"\n",
+        "\n",
+        "found = False\n",
+        "# Check if this compute target already exists in the workspace.\n",
+        "cts = ws.compute_targets\n",
+        "if amlcompute_cluster_name in cts and cts[amlcompute_cluster_name].type == 'AmlCompute':\n",
+        "    found = True\n",
+        "    print('Found existing compute target.')\n",
+        "    compute_target = cts[amlcompute_cluster_name]\n",
+        "    \n",
+        "if not found:\n",
+        "    print('Creating a new compute target...')\n",
+        "    provisioning_config = AmlCompute.provisioning_configuration(vm_size = \"STANDARD_D2_V2\", # for GPU, use \"STANDARD_NC6\"\n",
+        "                                                                #vm_priority = 'lowpriority', # optional\n",
+        "                                                                max_nodes = 4)\n",
+        "\n",
+        "    # Create the cluster.\n",
+        "    compute_target = ComputeTarget.create(ws, amlcompute_cluster_name, provisioning_config)\n",
+        "    \n",
+        "print('Checking cluster status...')\n",
+        "# Can poll for a minimum number of nodes and for a specific timeout.\n",
+        "# If no min_node_count is provided, it will use the scale settings for the cluster.\n",
+        "compute_target.wait_for_completion(show_output = True, min_node_count = None, timeout_in_minutes = 20)\n",
+        "    \n",
+        "# For a more detailed view of current AmlCompute status, use get_status()."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Data\n",
+        "\n",
+        "The [Machine Learning service workspace](https://docs.microsoft.com/en-us/azure/machine-learning/service/concept-workspace) is paired with the storage account, which contains the default data store. We will use it to upload the bike share data and create [tabular dataset](https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.data.tabulardataset?view=azure-ml-py) for training. A tabular dataset defines a series of lazily-evaluated, immutable operations to load data from the data source into tabular representation."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "datastore = ws.get_default_datastore()\n",
+        "datastore.upload_files(files = ['./bike-no.csv'], target_path = 'dataset/', overwrite = True,show_progress = True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Let's set up what we know about the dataset. \n",
+        "\n",
+        "**Target column** is what we want to forecast.\n",
+        "\n",
+        "**Time column** is the time axis along which to predict."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "target_column_name = 'cnt'\n",
+        "time_column_name = 'date'"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "dataset = Dataset.Tabular.from_delimited_files(path = [(datastore, 'dataset/bike-no.csv')]).with_timestamp_columns(fine_grain_timestamp=time_column_name) \n",
+        "dataset.take(5).to_pandas_dataframe().reset_index(drop=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Split the data\n",
+        "\n",
+        "The first split we make is into train and test sets. Note we are splitting on time. Data before 9/1 will be used for training, and data after and including 9/1 will be used for testing."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# select data that occurs before a specified date\n",
+        "train = dataset.time_before(datetime(2012, 8, 31), include_boundary=True)\n",
+        "train.to_pandas_dataframe().tail(5).reset_index(drop=True)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "test = dataset.time_after(datetime(2012, 9, 1), include_boundary=True)\n",
+        "test.to_pandas_dataframe().head(5).reset_index(drop=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Train\n",
+        "\n",
+        "Instantiate a AutoMLConfig object. This defines the settings and data used to run the experiment.\n",
+        "\n",
+        "|Property|Description|\n",
+        "|-|-|\n",
+        "|**task**|forecasting|\n",
+        "|**primary_metric**|This is the metric that you want to optimize.<br> Forecasting supports the following primary metrics <br><i>spearman_correlation</i><br><i>normalized_root_mean_squared_error</i><br><i>r2_score</i><br><i>normalized_mean_absolute_error</i>\n",
+        "|**blacklist_models**|Models in blacklist won't be used by AutoML. All supported models can be found at [here](https://docs.microsoft.com/en-us/python/api/azureml-train-automl-client/azureml.train.automl.constants.supportedmodels.forecasting?view=azure-ml-py).|\n",
+        "|**experiment_timeout_minutes**|Experimentation timeout in minutes.|\n",
+        "|**training_data**|Input dataset, containing both features and label column.|\n",
+        "|**label_column_name**|The name of the label column.|\n",
+        "|**compute_target**|The remote compute for training.|\n",
+        "|**n_cross_validations**|Number of cross validation splits.|\n",
+        "|**enable_early_stopping**|If early stopping is on, training will stop when the primary metric is no longer improving.|\n",
+        "|**time_column_name**|Name of the datetime column in the input data|\n",
+        "|**max_horizon**|Maximum desired forecast horizon in units of time-series frequency|\n",
+        "|**country_or_region**|The country/region used to generate holiday features. These should be ISO 3166 two-letter country/region codes (i.e. 'US', 'GB').|\n",
+        "|**target_lags**|The target_lags specifies how far back we will construct the lags of the target variable.|\n",
+        "|**drop_column_names**|Name(s) of columns to drop prior to modeling|\n",
+        "\n",
+        "This notebook uses the blacklist_models parameter to exclude some models that take a longer time to train on this dataset. You can choose to remove models from the blacklist_models list but you may need to increase the experiment_timeout_minutes parameter value to get results."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Setting forecaster maximum horizon \n",
+        "\n",
+        "The forecast horizon is the number of periods into the future that the model should predict. Here, we set the horizon to 14 periods (i.e. 14 days). Notice that this is much shorter than the number of days in the test set; we will need to use a rolling test to evaluate the performance on the whole test set. For more discussion of forecast horizons and guiding principles for setting them, please see the [energy demand notebook](https://github.com/Azure/MachineLearningNotebooks/tree/master/how-to-use-azureml/automated-machine-learning/forecasting-energy-demand).  "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "max_horizon = 14"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Config AutoML"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "time_series_settings = {\n",
+        "    'time_column_name': time_column_name,\n",
+        "    'max_horizon': max_horizon,    \n",
+        "    'country_or_region': 'US', # set country_or_region will trigger holiday featurizer\n",
+        "    'target_lags': 'auto', # use heuristic based lag setting    \n",
+        "    'drop_column_names': ['casual', 'registered'] # these columns are a breakdown of the total and therefore a leak\n",
+        "}\n",
+        "\n",
+        "automl_config = AutoMLConfig(task='forecasting',                             \n",
+        "                             primary_metric='normalized_root_mean_squared_error',\n",
+        "                             blacklist_models = ['ExtremeRandomTrees'],                             \n",
+        "                             experiment_timeout_minutes=20,\n",
+        "                             training_data=train,\n",
+        "                             label_column_name=target_column_name,\n",
+        "                             compute_target=compute_target,\n",
+        "                             enable_early_stopping=True,\n",
+        "                             n_cross_validations=3, \n",
+        "                             max_concurrent_iterations=4,\n",
+        "                             max_cores_per_iteration=-1,\n",
+        "                             verbosity=logging.INFO,\n",
+        "                            **time_series_settings)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "We will now run the experiment, you can go to Azure ML portal to view the run details. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "remote_run = experiment.submit(automl_config, show_output=False)\n",
+        "remote_run"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "remote_run.wait_for_completion()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Retrieve the Best Model\n",
+        "Below we select the best model from all the training iterations using get_output method."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "best_run, fitted_model = remote_run.get_output()\n",
+        "fitted_model.steps"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Featurization\n",
+        "\n",
+        "You can access the engineered feature names generated in time-series featurization. Note that a number of named holiday periods are represented. We recommend that you have at least one year of data when using this feature to ensure that all yearly holidays are captured in the training featurization."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "fitted_model.named_steps['timeseriestransformer'].get_engineered_feature_names()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### View the featurization summary\n",
+        "\n",
+        "You can also see what featurization steps were performed on different raw features in the user data. For each raw feature in the user data, the following information is displayed:\n",
+        "\n",
+        "- Raw feature name\n",
+        "- Number of engineered features formed out of this raw feature\n",
+        "- Type detected\n",
+        "- If feature was dropped\n",
+        "- List of feature transformations for the raw feature"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Get the featurization summary as a list of JSON\n",
+        "featurization_summary = fitted_model.named_steps['timeseriestransformer'].get_featurization_summary()\n",
+        "# View the featurization summary as a pandas dataframe\n",
+        "pd.DataFrame.from_records(featurization_summary)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Evaluate"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "We now use the best fitted model from the AutoML Run to make forecasts for the test set. We will do batch scoring on the test dataset which should have the same schema as training dataset.\n",
+        "\n",
+        "The scoring will run on a remote compute. In this example, it will reuse the training compute.|"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "test_experiment = Experiment(ws, experiment_name + \"_test\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Retrieving forecasts from the model\n",
+        "To run the forecast on the remote compute we will use two helper scripts: forecasting_script and forecasting_helper. These scripts contain the utility methods which will be used by the remote estimator. We copy these scripts to the project folder to upload them to remote compute."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "import shutil\n",
+        "\n",
+        "script_folder = os.path.join(os.getcwd(), 'forecast')\n",
+        "os.makedirs(script_folder, exist_ok=True)\n",
+        "shutil.copy2('forecasting_script.py', script_folder)\n",
+        "shutil.copy2('forecasting_helper.py', script_folder)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "For brevity we have created the function called run_forecast. It submits the test data to the best model and run the estimation on the selected compute target."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from run_forecast import run_rolling_forecast\n",
+        "\n",
+        "remote_run = run_rolling_forecast(test_experiment, compute_target, best_run, test, max_horizon,\n",
+        "                 target_column_name, time_column_name)\n",
+        "remote_run"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "remote_run.wait_for_completion(show_output=False)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Download the prediction result for metrics calcuation\n",
+        "The test data with predictions are saved in artifact outputs/predictions.csv. You can download it and calculation some error metrics for the forecasts and vizualize the predictions vs. the actuals."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "remote_run.download_file('outputs/predictions.csv', 'predictions.csv')\n",
+        "df_all = pd.read_csv('predictions.csv')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.automl.core._vendor.automl.client.core.common import metrics\n",
+        "from sklearn.metrics import mean_absolute_error, mean_squared_error\n",
+        "from matplotlib import pyplot as plt\n",
+        "from automl.client.core.common import constants\n",
+        "\n",
+        "# use automl metrics module\n",
+        "scores = metrics.compute_metrics_regression(\n",
+        "    df_all['predicted'],\n",
+        "    df_all[target_column_name],\n",
+        "    list(constants.Metric.SCALAR_REGRESSION_SET),\n",
+        "    None, None, None)\n",
+        "\n",
+        "print(\"[Test data scores]\\n\")\n",
+        "for key, value in scores.items():    \n",
+        "    print('{}:   {:.3f}'.format(key, value))\n",
+        "    \n",
+        "# Plot outputs\n",
+        "%matplotlib inline\n",
+        "test_pred = plt.scatter(df_all[target_column_name], df_all['predicted'], color='b')\n",
+        "test_test = plt.scatter(df_all[target_column_name], df_all[target_column_name], color='g')\n",
+        "plt.legend((test_pred, test_test), ('prediction', 'truth'), loc='upper left', fontsize=8)\n",
+        "plt.show()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "The MAPE seems high; it is being skewed by an actual with a small absolute value. For a more informative evaluation, we can calculate the metrics by forecast horizon:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from metrics_helper import MAPE, APE\n",
+        "df_all.groupby('horizon_origin').apply(\n",
+        "    lambda df: pd.Series({'MAPE': MAPE(df[target_column_name], df['predicted']),\n",
+        "                          'RMSE': np.sqrt(mean_squared_error(df[target_column_name], df['predicted'])),\n",
+        "                          'MAE': mean_absolute_error(df[target_column_name], df['predicted'])}))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "It's also interesting to see the distributions of APE (absolute percentage error) by horizon. On a log scale, the outlying APE in the horizon-3 group is clear."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "df_all_APE = df_all.assign(APE=APE(df_all[target_column_name], df_all['predicted']))\n",
+        "APEs = [df_all_APE[df_all['horizon_origin'] == h].APE.values for h in range(1, max_horizon + 1)]\n",
+        "\n",
+        "%matplotlib inline\n",
+        "plt.boxplot(APEs)\n",
+        "plt.yscale('log')\n",
+        "plt.xlabel('horizon')\n",
+        "plt.ylabel('APE (%)')\n",
+        "plt.title('Absolute Percentage Errors by Forecast Horizon')\n",
+        "\n",
+        "plt.show()"
+      ]
+    }
+  ],
+  "metadata": {
+    "authors": [
+      {
+        "name": "erwright"
+      }
+    ],
+    "category": "tutorial",
+    "compute": [
+      "Remote"
+    ],
+    "datasets": [
+      "BikeShare"
+    ],
+    "deployment": [
+      "None"
+    ],
+    "exclude_from_index": false,
+    "file_extension": ".py",
+    "framework": [
+      "Azure ML AutoML"
+    ],
+    "friendly_name": "Forecasting BikeShare Demand",
+    "index_order": 1,
+    "kernelspec": {
+      "display_name": "Python 3.6",
+      "language": "python",
+      "name": "python36"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.6.7"
+    },
+    "mimetype": "text/x-python",
+    "name": "python",
+    "npconvert_exporter": "python",
+    "pygments_lexer": "ipython3",
+    "tags": [
+      "Forecasting"
+    ],
+    "task": "Forecasting",
+    "version": 3
+  },
+  "nbformat": 4,
+  "nbformat_minor": 2
+}

how-to-use-azureml/automated-machine-learning/forecasting-bike-share/auto-ml-forecasting-bike-share.yml

@@ -0,0 +1,11 @@
+name: auto-ml-forecasting-bike-share
+dependencies:
+- fbprophet==0.5
+- py-xgboost<=0.80
+- pip:
+  - azureml-sdk
+  - azureml-train-automl
+  - azureml-widgets
+  - matplotlib
+  - pandas_ml
+  - statsmodels

how-to-use-azureml/automated-machine-learning/forecasting-bike-share/bike-no.csv

@@ -0,0 +1,732 @@
+instant,date,season,yr,mnth,weekday,weathersit,temp,atemp,hum,windspeed,casual,registered,cnt
+1,1/1/2011,1,0,1,6,2,0.344167,0.363625,0.805833,0.160446,331,654,985
+2,1/2/2011,1,0,1,0,2,0.363478,0.353739,0.696087,0.248539,131,670,801
+3,1/3/2011,1,0,1,1,1,0.196364,0.189405,0.437273,0.248309,120,1229,1349
+4,1/4/2011,1,0,1,2,1,0.2,0.212122,0.590435,0.160296,108,1454,1562
+5,1/5/2011,1,0,1,3,1,0.226957,0.22927,0.436957,0.1869,82,1518,1600
+6,1/6/2011,1,0,1,4,1,0.204348,0.233209,0.518261,0.0895652,88,1518,1606
+7,1/7/2011,1,0,1,5,2,0.196522,0.208839,0.498696,0.168726,148,1362,1510
+8,1/8/2011,1,0,1,6,2,0.165,0.162254,0.535833,0.266804,68,891,959
+9,1/9/2011,1,0,1,0,1,0.138333,0.116175,0.434167,0.36195,54,768,822
+10,1/10/2011,1,0,1,1,1,0.150833,0.150888,0.482917,0.223267,41,1280,1321
+11,1/11/2011,1,0,1,2,2,0.169091,0.191464,0.686364,0.122132,43,1220,1263
+12,1/12/2011,1,0,1,3,1,0.172727,0.160473,0.599545,0.304627,25,1137,1162
+13,1/13/2011,1,0,1,4,1,0.165,0.150883,0.470417,0.301,38,1368,1406
+14,1/14/2011,1,0,1,5,1,0.16087,0.188413,0.537826,0.126548,54,1367,1421
+15,1/15/2011,1,0,1,6,2,0.233333,0.248112,0.49875,0.157963,222,1026,1248
+16,1/16/2011,1,0,1,0,1,0.231667,0.234217,0.48375,0.188433,251,953,1204
+17,1/17/2011,1,0,1,1,2,0.175833,0.176771,0.5375,0.194017,117,883,1000
+18,1/18/2011,1,0,1,2,2,0.216667,0.232333,0.861667,0.146775,9,674,683
+19,1/19/2011,1,0,1,3,2,0.292174,0.298422,0.741739,0.208317,78,1572,1650
+20,1/20/2011,1,0,1,4,2,0.261667,0.25505,0.538333,0.195904,83,1844,1927
+21,1/21/2011,1,0,1,5,1,0.1775,0.157833,0.457083,0.353242,75,1468,1543
+22,1/22/2011,1,0,1,6,1,0.0591304,0.0790696,0.4,0.17197,93,888,981
+23,1/23/2011,1,0,1,0,1,0.0965217,0.0988391,0.436522,0.2466,150,836,986
+24,1/24/2011,1,0,1,1,1,0.0973913,0.11793,0.491739,0.15833,86,1330,1416
+25,1/25/2011,1,0,1,2,2,0.223478,0.234526,0.616957,0.129796,186,1799,1985
+26,1/26/2011,1,0,1,3,3,0.2175,0.2036,0.8625,0.29385,34,472,506
+27,1/27/2011,1,0,1,4,1,0.195,0.2197,0.6875,0.113837,15,416,431
+28,1/28/2011,1,0,1,5,2,0.203478,0.223317,0.793043,0.1233,38,1129,1167
+29,1/29/2011,1,0,1,6,1,0.196522,0.212126,0.651739,0.145365,123,975,1098
+30,1/30/2011,1,0,1,0,1,0.216522,0.250322,0.722174,0.0739826,140,956,1096
+31,1/31/2011,1,0,1,1,2,0.180833,0.18625,0.60375,0.187192,42,1459,1501
+32,2/1/2011,1,0,2,2,2,0.192174,0.23453,0.829565,0.053213,47,1313,1360
+33,2/2/2011,1,0,2,3,2,0.26,0.254417,0.775417,0.264308,72,1454,1526
+34,2/3/2011,1,0,2,4,1,0.186957,0.177878,0.437826,0.277752,61,1489,1550
+35,2/4/2011,1,0,2,5,2,0.211304,0.228587,0.585217,0.127839,88,1620,1708
+36,2/5/2011,1,0,2,6,2,0.233333,0.243058,0.929167,0.161079,100,905,1005
+37,2/6/2011,1,0,2,0,1,0.285833,0.291671,0.568333,0.1418,354,1269,1623
+38,2/7/2011,1,0,2,1,1,0.271667,0.303658,0.738333,0.0454083,120,1592,1712
+39,2/8/2011,1,0,2,2,1,0.220833,0.198246,0.537917,0.36195,64,1466,1530
+40,2/9/2011,1,0,2,3,2,0.134783,0.144283,0.494783,0.188839,53,1552,1605
+41,2/10/2011,1,0,2,4,1,0.144348,0.149548,0.437391,0.221935,47,1491,1538
+42,2/11/2011,1,0,2,5,1,0.189091,0.213509,0.506364,0.10855,149,1597,1746
+43,2/12/2011,1,0,2,6,1,0.2225,0.232954,0.544167,0.203367,288,1184,1472
+44,2/13/2011,1,0,2,0,1,0.316522,0.324113,0.457391,0.260883,397,1192,1589
+45,2/14/2011,1,0,2,1,1,0.415,0.39835,0.375833,0.417908,208,1705,1913
+46,2/15/2011,1,0,2,2,1,0.266087,0.254274,0.314348,0.291374,140,1675,1815
+47,2/16/2011,1,0,2,3,1,0.318261,0.3162,0.423478,0.251791,218,1897,2115
+48,2/17/2011,1,0,2,4,1,0.435833,0.428658,0.505,0.230104,259,2216,2475
+49,2/18/2011,1,0,2,5,1,0.521667,0.511983,0.516667,0.264925,579,2348,2927
+50,2/19/2011,1,0,2,6,1,0.399167,0.391404,0.187917,0.507463,532,1103,1635
+51,2/20/2011,1,0,2,0,1,0.285217,0.27733,0.407826,0.223235,639,1173,1812
+52,2/21/2011,1,0,2,1,2,0.303333,0.284075,0.605,0.307846,195,912,1107
+53,2/22/2011,1,0,2,2,1,0.182222,0.186033,0.577778,0.195683,74,1376,1450
+54,2/23/2011,1,0,2,3,1,0.221739,0.245717,0.423043,0.094113,139,1778,1917
+55,2/24/2011,1,0,2,4,2,0.295652,0.289191,0.697391,0.250496,100,1707,1807
+56,2/25/2011,1,0,2,5,2,0.364348,0.350461,0.712174,0.346539,120,1341,1461
+57,2/26/2011,1,0,2,6,1,0.2825,0.282192,0.537917,0.186571,424,1545,1969
+58,2/27/2011,1,0,2,0,1,0.343478,0.351109,0.68,0.125248,694,1708,2402
+59,2/28/2011,1,0,2,1,2,0.407273,0.400118,0.876364,0.289686,81,1365,1446
+60,3/1/2011,1,0,3,2,1,0.266667,0.263879,0.535,0.216425,137,1714,1851
+61,3/2/2011,1,0,3,3,1,0.335,0.320071,0.449583,0.307833,231,1903,2134
+62,3/3/2011,1,0,3,4,1,0.198333,0.200133,0.318333,0.225754,123,1562,1685
+63,3/4/2011,1,0,3,5,2,0.261667,0.255679,0.610417,0.203346,214,1730,1944
+64,3/5/2011,1,0,3,6,2,0.384167,0.378779,0.789167,0.251871,640,1437,2077
+65,3/6/2011,1,0,3,0,2,0.376522,0.366252,0.948261,0.343287,114,491,605
+66,3/7/2011,1,0,3,1,1,0.261739,0.238461,0.551304,0.341352,244,1628,1872
+67,3/8/2011,1,0,3,2,1,0.2925,0.3024,0.420833,0.12065,316,1817,2133
+68,3/9/2011,1,0,3,3,2,0.295833,0.286608,0.775417,0.22015,191,1700,1891
+69,3/10/2011,1,0,3,4,3,0.389091,0.385668,0,0.261877,46,577,623
+70,3/11/2011,1,0,3,5,2,0.316522,0.305,0.649565,0.23297,247,1730,1977
+71,3/12/2011,1,0,3,6,1,0.329167,0.32575,0.594583,0.220775,724,1408,2132
+72,3/13/2011,1,0,3,0,1,0.384348,0.380091,0.527391,0.270604,982,1435,2417
+73,3/14/2011,1,0,3,1,1,0.325217,0.332,0.496957,0.136926,359,1687,2046
+74,3/15/2011,1,0,3,2,2,0.317391,0.318178,0.655652,0.184309,289,1767,2056
+75,3/16/2011,1,0,3,3,2,0.365217,0.36693,0.776522,0.203117,321,1871,2192
+76,3/17/2011,1,0,3,4,1,0.415,0.410333,0.602917,0.209579,424,2320,2744
+77,3/18/2011,1,0,3,5,1,0.54,0.527009,0.525217,0.231017,884,2355,3239
+78,3/19/2011,1,0,3,6,1,0.4725,0.466525,0.379167,0.368167,1424,1693,3117
+79,3/20/2011,1,0,3,0,1,0.3325,0.32575,0.47375,0.207721,1047,1424,2471
+80,3/21/2011,2,0,3,1,2,0.430435,0.409735,0.737391,0.288783,401,1676,2077
+81,3/22/2011,2,0,3,2,1,0.441667,0.440642,0.624583,0.22575,460,2243,2703
+82,3/23/2011,2,0,3,3,2,0.346957,0.337939,0.839565,0.234261,203,1918,2121
+83,3/24/2011,2,0,3,4,2,0.285,0.270833,0.805833,0.243787,166,1699,1865
+84,3/25/2011,2,0,3,5,1,0.264167,0.256312,0.495,0.230725,300,1910,2210
+85,3/26/2011,2,0,3,6,1,0.265833,0.257571,0.394167,0.209571,981,1515,2496
+86,3/27/2011,2,0,3,0,2,0.253043,0.250339,0.493913,0.1843,472,1221,1693
+87,3/28/2011,2,0,3,1,1,0.264348,0.257574,0.302174,0.212204,222,1806,2028
+88,3/29/2011,2,0,3,2,1,0.3025,0.292908,0.314167,0.226996,317,2108,2425
+89,3/30/2011,2,0,3,3,2,0.3,0.29735,0.646667,0.172888,168,1368,1536
+90,3/31/2011,2,0,3,4,3,0.268333,0.257575,0.918333,0.217646,179,1506,1685
+91,4/1/2011,2,0,4,5,2,0.3,0.283454,0.68625,0.258708,307,1920,2227
+92,4/2/2011,2,0,4,6,2,0.315,0.315637,0.65375,0.197146,898,1354,2252
+93,4/3/2011,2,0,4,0,1,0.378333,0.378767,0.48,0.182213,1651,1598,3249
+94,4/4/2011,2,0,4,1,1,0.573333,0.542929,0.42625,0.385571,734,2381,3115
+95,4/5/2011,2,0,4,2,2,0.414167,0.39835,0.642083,0.388067,167,1628,1795
+96,4/6/2011,2,0,4,3,1,0.390833,0.387608,0.470833,0.263063,413,2395,2808
+97,4/7/2011,2,0,4,4,1,0.4375,0.433696,0.602917,0.162312,571,2570,3141
+98,4/8/2011,2,0,4,5,2,0.335833,0.324479,0.83625,0.226992,172,1299,1471
+99,4/9/2011,2,0,4,6,2,0.3425,0.341529,0.8775,0.133083,879,1576,2455
+100,4/10/2011,2,0,4,0,2,0.426667,0.426737,0.8575,0.146767,1188,1707,2895
+101,4/11/2011,2,0,4,1,2,0.595652,0.565217,0.716956,0.324474,855,2493,3348
+102,4/12/2011,2,0,4,2,2,0.5025,0.493054,0.739167,0.274879,257,1777,2034
+103,4/13/2011,2,0,4,3,2,0.4125,0.417283,0.819167,0.250617,209,1953,2162
+104,4/14/2011,2,0,4,4,1,0.4675,0.462742,0.540417,0.1107,529,2738,3267
+105,4/15/2011,2,0,4,5,1,0.446667,0.441913,0.67125,0.226375,642,2484,3126
+106,4/16/2011,2,0,4,6,3,0.430833,0.425492,0.888333,0.340808,121,674,795
+107,4/17/2011,2,0,4,0,1,0.456667,0.445696,0.479583,0.303496,1558,2186,3744
+108,4/18/2011,2,0,4,1,1,0.5125,0.503146,0.5425,0.163567,669,2760,3429
+109,4/19/2011,2,0,4,2,2,0.505833,0.489258,0.665833,0.157971,409,2795,3204
+110,4/20/2011,2,0,4,3,1,0.595,0.564392,0.614167,0.241925,613,3331,3944
+111,4/21/2011,2,0,4,4,1,0.459167,0.453892,0.407083,0.325258,745,3444,4189
+112,4/22/2011,2,0,4,5,2,0.336667,0.321954,0.729583,0.219521,177,1506,1683
+113,4/23/2011,2,0,4,6,2,0.46,0.450121,0.887917,0.230725,1462,2574,4036
+114,4/24/2011,2,0,4,0,2,0.581667,0.551763,0.810833,0.192175,1710,2481,4191
+115,4/25/2011,2,0,4,1,1,0.606667,0.5745,0.776667,0.185333,773,3300,4073
+116,4/26/2011,2,0,4,2,1,0.631667,0.594083,0.729167,0.3265,678,3722,4400
+117,4/27/2011,2,0,4,3,2,0.62,0.575142,0.835417,0.3122,547,3325,3872
+118,4/28/2011,2,0,4,4,2,0.6175,0.578929,0.700833,0.320908,569,3489,4058
+119,4/29/2011,2,0,4,5,1,0.51,0.497463,0.457083,0.240063,878,3717,4595
+120,4/30/2011,2,0,4,6,1,0.4725,0.464021,0.503333,0.235075,1965,3347,5312
+121,5/1/2011,2,0,5,0,2,0.451667,0.448204,0.762083,0.106354,1138,2213,3351
+122,5/2/2011,2,0,5,1,2,0.549167,0.532833,0.73,0.183454,847,3554,4401
+123,5/3/2011,2,0,5,2,2,0.616667,0.582079,0.697083,0.342667,603,3848,4451
+124,5/4/2011,2,0,5,3,2,0.414167,0.40465,0.737083,0.328996,255,2378,2633
+125,5/5/2011,2,0,5,4,1,0.459167,0.441917,0.444167,0.295392,614,3819,4433
+126,5/6/2011,2,0,5,5,1,0.479167,0.474117,0.59,0.228246,894,3714,4608
+127,5/7/2011,2,0,5,6,1,0.52,0.512621,0.54125,0.16045,1612,3102,4714
+128,5/8/2011,2,0,5,0,1,0.528333,0.518933,0.631667,0.0746375,1401,2932,4333
+129,5/9/2011,2,0,5,1,1,0.5325,0.525246,0.58875,0.176,664,3698,4362
+130,5/10/2011,2,0,5,2,1,0.5325,0.522721,0.489167,0.115671,694,4109,4803
+131,5/11/2011,2,0,5,3,1,0.5425,0.5284,0.632917,0.120642,550,3632,4182
+132,5/12/2011,2,0,5,4,1,0.535,0.523363,0.7475,0.189667,695,4169,4864
+133,5/13/2011,2,0,5,5,2,0.5125,0.4943,0.863333,0.179725,692,3413,4105
+134,5/14/2011,2,0,5,6,2,0.520833,0.500629,0.9225,0.13495,902,2507,3409
+135,5/15/2011,2,0,5,0,2,0.5625,0.536,0.867083,0.152979,1582,2971,4553
+136,5/16/2011,2,0,5,1,1,0.5775,0.550512,0.787917,0.126871,773,3185,3958
+137,5/17/2011,2,0,5,2,2,0.561667,0.538529,0.837917,0.277354,678,3445,4123
+138,5/18/2011,2,0,5,3,2,0.55,0.527158,0.87,0.201492,536,3319,3855
+139,5/19/2011,2,0,5,4,2,0.530833,0.510742,0.829583,0.108213,735,3840,4575
+140,5/20/2011,2,0,5,5,1,0.536667,0.529042,0.719583,0.125013,909,4008,4917
+141,5/21/2011,2,0,5,6,1,0.6025,0.571975,0.626667,0.12065,2258,3547,5805
+142,5/22/2011,2,0,5,0,1,0.604167,0.5745,0.749583,0.148008,1576,3084,4660
+143,5/23/2011,2,0,5,1,2,0.631667,0.590296,0.81,0.233842,836,3438,4274
+144,5/24/2011,2,0,5,2,2,0.66,0.604813,0.740833,0.207092,659,3833,4492
+145,5/25/2011,2,0,5,3,1,0.660833,0.615542,0.69625,0.154233,740,4238,4978
+146,5/26/2011,2,0,5,4,1,0.708333,0.654688,0.6775,0.199642,758,3919,4677
+147,5/27/2011,2,0,5,5,1,0.681667,0.637008,0.65375,0.240679,871,3808,4679
+148,5/28/2011,2,0,5,6,1,0.655833,0.612379,0.729583,0.230092,2001,2757,4758
+149,5/29/2011,2,0,5,0,1,0.6675,0.61555,0.81875,0.213938,2355,2433,4788
+150,5/30/2011,2,0,5,1,1,0.733333,0.671092,0.685,0.131225,1549,2549,4098
+151,5/31/2011,2,0,5,2,1,0.775,0.725383,0.636667,0.111329,673,3309,3982
+152,6/1/2011,2,0,6,3,2,0.764167,0.720967,0.677083,0.207092,513,3461,3974
+153,6/2/2011,2,0,6,4,1,0.715,0.643942,0.305,0.292287,736,4232,4968
+154,6/3/2011,2,0,6,5,1,0.62,0.587133,0.354167,0.253121,898,4414,5312
+155,6/4/2011,2,0,6,6,1,0.635,0.594696,0.45625,0.123142,1869,3473,5342
+156,6/5/2011,2,0,6,0,2,0.648333,0.616804,0.6525,0.138692,1685,3221,4906
+157,6/6/2011,2,0,6,1,1,0.678333,0.621858,0.6,0.121896,673,3875,4548
+158,6/7/2011,2,0,6,2,1,0.7075,0.65595,0.597917,0.187808,763,4070,4833
+159,6/8/2011,2,0,6,3,1,0.775833,0.727279,0.622083,0.136817,676,3725,4401
+160,6/9/2011,2,0,6,4,2,0.808333,0.757579,0.568333,0.149883,563,3352,3915
+161,6/10/2011,2,0,6,5,1,0.755,0.703292,0.605,0.140554,815,3771,4586
+162,6/11/2011,2,0,6,6,1,0.725,0.678038,0.654583,0.15485,1729,3237,4966
+163,6/12/2011,2,0,6,0,1,0.6925,0.643325,0.747917,0.163567,1467,2993,4460
+164,6/13/2011,2,0,6,1,1,0.635,0.601654,0.494583,0.30535,863,4157,5020
+165,6/14/2011,2,0,6,2,1,0.604167,0.591546,0.507083,0.269283,727,4164,4891
+166,6/15/2011,2,0,6,3,1,0.626667,0.587754,0.471667,0.167912,769,4411,5180
+167,6/16/2011,2,0,6,4,2,0.628333,0.595346,0.688333,0.206471,545,3222,3767
+168,6/17/2011,2,0,6,5,1,0.649167,0.600383,0.735833,0.143029,863,3981,4844
+169,6/18/2011,2,0,6,6,1,0.696667,0.643954,0.670417,0.119408,1807,3312,5119
+170,6/19/2011,2,0,6,0,2,0.699167,0.645846,0.666667,0.102,1639,3105,4744
+171,6/20/2011,2,0,6,1,2,0.635,0.595346,0.74625,0.155475,699,3311,4010
+172,6/21/2011,3,0,6,2,2,0.680833,0.637646,0.770417,0.171025,774,4061,4835
+173,6/22/2011,3,0,6,3,1,0.733333,0.693829,0.7075,0.172262,661,3846,4507
+174,6/23/2011,3,0,6,4,2,0.728333,0.693833,0.703333,0.238804,746,4044,4790
+175,6/24/2011,3,0,6,5,1,0.724167,0.656583,0.573333,0.222025,969,4022,4991
+176,6/25/2011,3,0,6,6,1,0.695,0.643313,0.483333,0.209571,1782,3420,5202
+177,6/26/2011,3,0,6,0,1,0.68,0.637629,0.513333,0.0945333,1920,3385,5305
+178,6/27/2011,3,0,6,1,2,0.6825,0.637004,0.658333,0.107588,854,3854,4708
+179,6/28/2011,3,0,6,2,1,0.744167,0.692558,0.634167,0.144283,732,3916,4648
+180,6/29/2011,3,0,6,3,1,0.728333,0.654688,0.497917,0.261821,848,4377,5225
+181,6/30/2011,3,0,6,4,1,0.696667,0.637008,0.434167,0.185312,1027,4488,5515
+182,7/1/2011,3,0,7,5,1,0.7225,0.652162,0.39625,0.102608,1246,4116,5362
+183,7/2/2011,3,0,7,6,1,0.738333,0.667308,0.444583,0.115062,2204,2915,5119
+184,7/3/2011,3,0,7,0,2,0.716667,0.668575,0.6825,0.228858,2282,2367,4649
+185,7/4/2011,3,0,7,1,2,0.726667,0.665417,0.637917,0.0814792,3065,2978,6043
+186,7/5/2011,3,0,7,2,1,0.746667,0.696338,0.590417,0.126258,1031,3634,4665
+187,7/6/2011,3,0,7,3,1,0.72,0.685633,0.743333,0.149883,784,3845,4629
+188,7/7/2011,3,0,7,4,1,0.75,0.686871,0.65125,0.1592,754,3838,4592
+189,7/8/2011,3,0,7,5,2,0.709167,0.670483,0.757917,0.225129,692,3348,4040
+190,7/9/2011,3,0,7,6,1,0.733333,0.664158,0.609167,0.167912,1988,3348,5336
+191,7/10/2011,3,0,7,0,1,0.7475,0.690025,0.578333,0.183471,1743,3138,4881
+192,7/11/2011,3,0,7,1,1,0.7625,0.729804,0.635833,0.282337,723,3363,4086
+193,7/12/2011,3,0,7,2,1,0.794167,0.739275,0.559167,0.200254,662,3596,4258
+194,7/13/2011,3,0,7,3,1,0.746667,0.689404,0.631667,0.146133,748,3594,4342
+195,7/14/2011,3,0,7,4,1,0.680833,0.635104,0.47625,0.240667,888,4196,5084
+196,7/15/2011,3,0,7,5,1,0.663333,0.624371,0.59125,0.182833,1318,4220,5538
+197,7/16/2011,3,0,7,6,1,0.686667,0.638263,0.585,0.208342,2418,3505,5923
+198,7/17/2011,3,0,7,0,1,0.719167,0.669833,0.604167,0.245033,2006,3296,5302
+199,7/18/2011,3,0,7,1,1,0.746667,0.703925,0.65125,0.215804,841,3617,4458
+200,7/19/2011,3,0,7,2,1,0.776667,0.747479,0.650417,0.1306,752,3789,4541
+201,7/20/2011,3,0,7,3,1,0.768333,0.74685,0.707083,0.113817,644,3688,4332
+202,7/21/2011,3,0,7,4,2,0.815,0.826371,0.69125,0.222021,632,3152,3784
+203,7/22/2011,3,0,7,5,1,0.848333,0.840896,0.580417,0.1331,562,2825,3387
+204,7/23/2011,3,0,7,6,1,0.849167,0.804287,0.5,0.131221,987,2298,3285
+205,7/24/2011,3,0,7,0,1,0.83,0.794829,0.550833,0.169171,1050,2556,3606
+206,7/25/2011,3,0,7,1,1,0.743333,0.720958,0.757083,0.0908083,568,3272,3840
+207,7/26/2011,3,0,7,2,1,0.771667,0.696979,0.540833,0.200258,750,3840,4590
+208,7/27/2011,3,0,7,3,1,0.775,0.690667,0.402917,0.183463,755,3901,4656
+209,7/28/2011,3,0,7,4,1,0.779167,0.7399,0.583333,0.178479,606,3784,4390
+210,7/29/2011,3,0,7,5,1,0.838333,0.785967,0.5425,0.174138,670,3176,3846
+211,7/30/2011,3,0,7,6,1,0.804167,0.728537,0.465833,0.168537,1559,2916,4475
+212,7/31/2011,3,0,7,0,1,0.805833,0.729796,0.480833,0.164813,1524,2778,4302
+213,8/1/2011,3,0,8,1,1,0.771667,0.703292,0.550833,0.156717,729,3537,4266
+214,8/2/2011,3,0,8,2,1,0.783333,0.707071,0.49125,0.20585,801,4044,4845
+215,8/3/2011,3,0,8,3,2,0.731667,0.679937,0.6575,0.135583,467,3107,3574
+216,8/4/2011,3,0,8,4,2,0.71,0.664788,0.7575,0.19715,799,3777,4576
+217,8/5/2011,3,0,8,5,1,0.710833,0.656567,0.630833,0.184696,1023,3843,4866
+218,8/6/2011,3,0,8,6,2,0.716667,0.676154,0.755,0.22825,1521,2773,4294
+219,8/7/2011,3,0,8,0,1,0.7425,0.715292,0.752917,0.201487,1298,2487,3785
+220,8/8/2011,3,0,8,1,1,0.765,0.703283,0.592083,0.192175,846,3480,4326
+221,8/9/2011,3,0,8,2,1,0.775,0.724121,0.570417,0.151121,907,3695,4602
+222,8/10/2011,3,0,8,3,1,0.766667,0.684983,0.424167,0.200258,884,3896,4780
+223,8/11/2011,3,0,8,4,1,0.7175,0.651521,0.42375,0.164796,812,3980,4792
+224,8/12/2011,3,0,8,5,1,0.708333,0.654042,0.415,0.125621,1051,3854,4905
+225,8/13/2011,3,0,8,6,2,0.685833,0.645858,0.729583,0.211454,1504,2646,4150
+226,8/14/2011,3,0,8,0,2,0.676667,0.624388,0.8175,0.222633,1338,2482,3820
+227,8/15/2011,3,0,8,1,1,0.665833,0.616167,0.712083,0.208954,775,3563,4338
+228,8/16/2011,3,0,8,2,1,0.700833,0.645837,0.578333,0.236329,721,4004,4725
+229,8/17/2011,3,0,8,3,1,0.723333,0.666671,0.575417,0.143667,668,4026,4694
+230,8/18/2011,3,0,8,4,1,0.711667,0.662258,0.654583,0.233208,639,3166,3805
+231,8/19/2011,3,0,8,5,2,0.685,0.633221,0.722917,0.139308,797,3356,4153
+232,8/20/2011,3,0,8,6,1,0.6975,0.648996,0.674167,0.104467,1914,3277,5191
+233,8/21/2011,3,0,8,0,1,0.710833,0.675525,0.77,0.248754,1249,2624,3873
+234,8/22/2011,3,0,8,1,1,0.691667,0.638254,0.47,0.27675,833,3925,4758
+235,8/23/2011,3,0,8,2,1,0.640833,0.606067,0.455417,0.146763,1281,4614,5895
+236,8/24/2011,3,0,8,3,1,0.673333,0.630692,0.605,0.253108,949,4181,5130
+237,8/25/2011,3,0,8,4,2,0.684167,0.645854,0.771667,0.210833,435,3107,3542
+238,8/26/2011,3,0,8,5,1,0.7,0.659733,0.76125,0.0839625,768,3893,4661
+239,8/27/2011,3,0,8,6,2,0.68,0.635556,0.85,0.375617,226,889,1115
+240,8/28/2011,3,0,8,0,1,0.707059,0.647959,0.561765,0.304659,1415,2919,4334
+241,8/29/2011,3,0,8,1,1,0.636667,0.607958,0.554583,0.159825,729,3905,4634
+242,8/30/2011,3,0,8,2,1,0.639167,0.594704,0.548333,0.125008,775,4429,5204
+243,8/31/2011,3,0,8,3,1,0.656667,0.611121,0.597917,0.0833333,688,4370,5058
+244,9/1/2011,3,0,9,4,1,0.655,0.614921,0.639167,0.141796,783,4332,5115
+245,9/2/2011,3,0,9,5,2,0.643333,0.604808,0.727083,0.139929,875,3852,4727
+246,9/3/2011,3,0,9,6,1,0.669167,0.633213,0.716667,0.185325,1935,2549,4484
+247,9/4/2011,3,0,9,0,1,0.709167,0.665429,0.742083,0.206467,2521,2419,4940
+248,9/5/2011,3,0,9,1,2,0.673333,0.625646,0.790417,0.212696,1236,2115,3351
+249,9/6/2011,3,0,9,2,3,0.54,0.5152,0.886957,0.343943,204,2506,2710
+250,9/7/2011,3,0,9,3,3,0.599167,0.544229,0.917083,0.0970208,118,1878,1996
+251,9/8/2011,3,0,9,4,3,0.633913,0.555361,0.939565,0.192748,153,1689,1842
+252,9/9/2011,3,0,9,5,2,0.65,0.578946,0.897917,0.124379,417,3127,3544
+253,9/10/2011,3,0,9,6,1,0.66,0.607962,0.75375,0.153608,1750,3595,5345
+254,9/11/2011,3,0,9,0,1,0.653333,0.609229,0.71375,0.115054,1633,3413,5046
+255,9/12/2011,3,0,9,1,1,0.644348,0.60213,0.692174,0.088913,690,4023,4713
+256,9/13/2011,3,0,9,2,1,0.650833,0.603554,0.7125,0.141804,701,4062,4763
+257,9/14/2011,3,0,9,3,1,0.673333,0.6269,0.697083,0.1673,647,4138,4785
+258,9/15/2011,3,0,9,4,2,0.5775,0.553671,0.709167,0.271146,428,3231,3659
+259,9/16/2011,3,0,9,5,2,0.469167,0.461475,0.590417,0.164183,742,4018,4760
+260,9/17/2011,3,0,9,6,2,0.491667,0.478512,0.718333,0.189675,1434,3077,4511
+261,9/18/2011,3,0,9,0,1,0.5075,0.490537,0.695,0.178483,1353,2921,4274
+262,9/19/2011,3,0,9,1,2,0.549167,0.529675,0.69,0.151742,691,3848,4539
+263,9/20/2011,3,0,9,2,2,0.561667,0.532217,0.88125,0.134954,438,3203,3641
+264,9/21/2011,3,0,9,3,2,0.595,0.550533,0.9,0.0964042,539,3813,4352
+265,9/22/2011,3,0,9,4,2,0.628333,0.554963,0.902083,0.128125,555,4240,4795
+266,9/23/2011,4,0,9,5,2,0.609167,0.522125,0.9725,0.0783667,258,2137,2395
+267,9/24/2011,4,0,9,6,2,0.606667,0.564412,0.8625,0.0783833,1776,3647,5423
+268,9/25/2011,4,0,9,0,2,0.634167,0.572637,0.845,0.0503792,1544,3466,5010
+269,9/26/2011,4,0,9,1,2,0.649167,0.589042,0.848333,0.1107,684,3946,4630
+270,9/27/2011,4,0,9,2,2,0.636667,0.574525,0.885417,0.118171,477,3643,4120
+271,9/28/2011,4,0,9,3,2,0.635,0.575158,0.84875,0.148629,480,3427,3907
+272,9/29/2011,4,0,9,4,1,0.616667,0.574512,0.699167,0.172883,653,4186,4839
+273,9/30/2011,4,0,9,5,1,0.564167,0.544829,0.6475,0.206475,830,4372,5202
+274,10/1/2011,4,0,10,6,2,0.41,0.412863,0.75375,0.292296,480,1949,2429
+275,10/2/2011,4,0,10,0,2,0.356667,0.345317,0.791667,0.222013,616,2302,2918
+276,10/3/2011,4,0,10,1,2,0.384167,0.392046,0.760833,0.0833458,330,3240,3570
+277,10/4/2011,4,0,10,2,1,0.484167,0.472858,0.71,0.205854,486,3970,4456
+278,10/5/2011,4,0,10,3,1,0.538333,0.527138,0.647917,0.17725,559,4267,4826
+279,10/6/2011,4,0,10,4,1,0.494167,0.480425,0.620833,0.134954,639,4126,4765
+280,10/7/2011,4,0,10,5,1,0.510833,0.504404,0.684167,0.0223917,949,4036,4985
+281,10/8/2011,4,0,10,6,1,0.521667,0.513242,0.70125,0.0454042,2235,3174,5409
+282,10/9/2011,4,0,10,0,1,0.540833,0.523983,0.7275,0.06345,2397,3114,5511
+283,10/10/2011,4,0,10,1,1,0.570833,0.542925,0.73375,0.0423042,1514,3603,5117
+284,10/11/2011,4,0,10,2,2,0.566667,0.546096,0.80875,0.143042,667,3896,4563
+285,10/12/2011,4,0,10,3,3,0.543333,0.517717,0.90625,0.24815,217,2199,2416
+286,10/13/2011,4,0,10,4,2,0.589167,0.551804,0.896667,0.141787,290,2623,2913
+287,10/14/2011,4,0,10,5,2,0.550833,0.529675,0.71625,0.223883,529,3115,3644
+288,10/15/2011,4,0,10,6,1,0.506667,0.498725,0.483333,0.258083,1899,3318,5217
+289,10/16/2011,4,0,10,0,1,0.511667,0.503154,0.486667,0.281717,1748,3293,5041
+290,10/17/2011,4,0,10,1,1,0.534167,0.510725,0.579583,0.175379,713,3857,4570
+291,10/18/2011,4,0,10,2,2,0.5325,0.522721,0.701667,0.110087,637,4111,4748
+292,10/19/2011,4,0,10,3,3,0.541739,0.513848,0.895217,0.243339,254,2170,2424
+293,10/20/2011,4,0,10,4,1,0.475833,0.466525,0.63625,0.422275,471,3724,4195
+294,10/21/2011,4,0,10,5,1,0.4275,0.423596,0.574167,0.221396,676,3628,4304
+295,10/22/2011,4,0,10,6,1,0.4225,0.425492,0.629167,0.0926667,1499,2809,4308
+296,10/23/2011,4,0,10,0,1,0.421667,0.422333,0.74125,0.0995125,1619,2762,4381
+297,10/24/2011,4,0,10,1,1,0.463333,0.457067,0.772083,0.118792,699,3488,4187
+298,10/25/2011,4,0,10,2,1,0.471667,0.463375,0.622917,0.166658,695,3992,4687
+299,10/26/2011,4,0,10,3,2,0.484167,0.472846,0.720417,0.148642,404,3490,3894
+300,10/27/2011,4,0,10,4,2,0.47,0.457046,0.812917,0.197763,240,2419,2659
+301,10/28/2011,4,0,10,5,2,0.330833,0.318812,0.585833,0.229479,456,3291,3747
+302,10/29/2011,4,0,10,6,3,0.254167,0.227913,0.8825,0.351371,57,570,627
+303,10/30/2011,4,0,10,0,1,0.319167,0.321329,0.62375,0.176617,885,2446,3331
+304,10/31/2011,4,0,10,1,1,0.34,0.356063,0.703333,0.10635,362,3307,3669
+305,11/1/2011,4,0,11,2,1,0.400833,0.397088,0.68375,0.135571,410,3658,4068
+306,11/2/2011,4,0,11,3,1,0.3775,0.390133,0.71875,0.0820917,370,3816,4186
+307,11/3/2011,4,0,11,4,1,0.408333,0.405921,0.702083,0.136817,318,3656,3974
+308,11/4/2011,4,0,11,5,2,0.403333,0.403392,0.6225,0.271779,470,3576,4046
+309,11/5/2011,4,0,11,6,1,0.326667,0.323854,0.519167,0.189062,1156,2770,3926
+310,11/6/2011,4,0,11,0,1,0.348333,0.362358,0.734583,0.0920542,952,2697,3649
+311,11/7/2011,4,0,11,1,1,0.395,0.400871,0.75875,0.057225,373,3662,4035
+312,11/8/2011,4,0,11,2,1,0.408333,0.412246,0.721667,0.0690375,376,3829,4205
+313,11/9/2011,4,0,11,3,1,0.4,0.409079,0.758333,0.0621958,305,3804,4109
+314,11/10/2011,4,0,11,4,2,0.38,0.373721,0.813333,0.189067,190,2743,2933
+315,11/11/2011,4,0,11,5,1,0.324167,0.306817,0.44625,0.314675,440,2928,3368
+316,11/12/2011,4,0,11,6,1,0.356667,0.357942,0.552917,0.212062,1275,2792,4067
+317,11/13/2011,4,0,11,0,1,0.440833,0.43055,0.458333,0.281721,1004,2713,3717
+318,11/14/2011,4,0,11,1,1,0.53,0.524612,0.587083,0.306596,595,3891,4486
+319,11/15/2011,4,0,11,2,2,0.53,0.507579,0.68875,0.199633,449,3746,4195
+320,11/16/2011,4,0,11,3,3,0.456667,0.451988,0.93,0.136829,145,1672,1817
+321,11/17/2011,4,0,11,4,2,0.341667,0.323221,0.575833,0.305362,139,2914,3053
+322,11/18/2011,4,0,11,5,1,0.274167,0.272721,0.41,0.168533,245,3147,3392
+323,11/19/2011,4,0,11,6,1,0.329167,0.324483,0.502083,0.224496,943,2720,3663
+324,11/20/2011,4,0,11,0,2,0.463333,0.457058,0.684583,0.18595,787,2733,3520
+325,11/21/2011,4,0,11,1,3,0.4475,0.445062,0.91,0.138054,220,2545,2765
+326,11/22/2011,4,0,11,2,3,0.416667,0.421696,0.9625,0.118792,69,1538,1607
+327,11/23/2011,4,0,11,3,2,0.440833,0.430537,0.757917,0.335825,112,2454,2566
+328,11/24/2011,4,0,11,4,1,0.373333,0.372471,0.549167,0.167304,560,935,1495
+329,11/25/2011,4,0,11,5,1,0.375,0.380671,0.64375,0.0988958,1095,1697,2792
+330,11/26/2011,4,0,11,6,1,0.375833,0.385087,0.681667,0.0684208,1249,1819,3068
+331,11/27/2011,4,0,11,0,1,0.459167,0.4558,0.698333,0.208954,810,2261,3071
+332,11/28/2011,4,0,11,1,1,0.503478,0.490122,0.743043,0.142122,253,3614,3867
+333,11/29/2011,4,0,11,2,2,0.458333,0.451375,0.830833,0.258092,96,2818,2914
+334,11/30/2011,4,0,11,3,1,0.325,0.311221,0.613333,0.271158,188,3425,3613
+335,12/1/2011,4,0,12,4,1,0.3125,0.305554,0.524583,0.220158,182,3545,3727
+336,12/2/2011,4,0,12,5,1,0.314167,0.331433,0.625833,0.100754,268,3672,3940
+337,12/3/2011,4,0,12,6,1,0.299167,0.310604,0.612917,0.0957833,706,2908,3614
+338,12/4/2011,4,0,12,0,1,0.330833,0.3491,0.775833,0.0839583,634,2851,3485
+339,12/5/2011,4,0,12,1,2,0.385833,0.393925,0.827083,0.0622083,233,3578,3811
+340,12/6/2011,4,0,12,2,3,0.4625,0.4564,0.949583,0.232583,126,2468,2594
+341,12/7/2011,4,0,12,3,3,0.41,0.400246,0.970417,0.266175,50,655,705
+342,12/8/2011,4,0,12,4,1,0.265833,0.256938,0.58,0.240058,150,3172,3322
+343,12/9/2011,4,0,12,5,1,0.290833,0.317542,0.695833,0.0827167,261,3359,3620
+344,12/10/2011,4,0,12,6,1,0.275,0.266412,0.5075,0.233221,502,2688,3190
+345,12/11/2011,4,0,12,0,1,0.220833,0.253154,0.49,0.0665417,377,2366,2743
+346,12/12/2011,4,0,12,1,1,0.238333,0.270196,0.670833,0.06345,143,3167,3310
+347,12/13/2011,4,0,12,2,1,0.2825,0.301138,0.59,0.14055,155,3368,3523
+348,12/14/2011,4,0,12,3,2,0.3175,0.338362,0.66375,0.0609583,178,3562,3740
+349,12/15/2011,4,0,12,4,2,0.4225,0.412237,0.634167,0.268042,181,3528,3709
+350,12/16/2011,4,0,12,5,2,0.375,0.359825,0.500417,0.260575,178,3399,3577
+351,12/17/2011,4,0,12,6,2,0.258333,0.249371,0.560833,0.243167,275,2464,2739
+352,12/18/2011,4,0,12,0,1,0.238333,0.245579,0.58625,0.169779,220,2211,2431
+353,12/19/2011,4,0,12,1,1,0.276667,0.280933,0.6375,0.172896,260,3143,3403
+354,12/20/2011,4,0,12,2,2,0.385833,0.396454,0.595417,0.0615708,216,3534,3750
+355,12/21/2011,1,0,12,3,2,0.428333,0.428017,0.858333,0.2214,107,2553,2660
+356,12/22/2011,1,0,12,4,2,0.423333,0.426121,0.7575,0.047275,227,2841,3068
+357,12/23/2011,1,0,12,5,1,0.373333,0.377513,0.68625,0.274246,163,2046,2209
+358,12/24/2011,1,0,12,6,1,0.3025,0.299242,0.5425,0.190304,155,856,1011
+359,12/25/2011,1,0,12,0,1,0.274783,0.279961,0.681304,0.155091,303,451,754
+360,12/26/2011,1,0,12,1,1,0.321739,0.315535,0.506957,0.239465,430,887,1317
+361,12/27/2011,1,0,12,2,2,0.325,0.327633,0.7625,0.18845,103,1059,1162
+362,12/28/2011,1,0,12,3,1,0.29913,0.279974,0.503913,0.293961,255,2047,2302
+363,12/29/2011,1,0,12,4,1,0.248333,0.263892,0.574167,0.119412,254,2169,2423
+364,12/30/2011,1,0,12,5,1,0.311667,0.318812,0.636667,0.134337,491,2508,2999
+365,12/31/2011,1,0,12,6,1,0.41,0.414121,0.615833,0.220154,665,1820,2485
+366,1/1/2012,1,1,1,0,1,0.37,0.375621,0.6925,0.192167,686,1608,2294
+367,1/2/2012,1,1,1,1,1,0.273043,0.252304,0.381304,0.329665,244,1707,1951
+368,1/3/2012,1,1,1,2,1,0.15,0.126275,0.44125,0.365671,89,2147,2236
+369,1/4/2012,1,1,1,3,2,0.1075,0.119337,0.414583,0.1847,95,2273,2368
+370,1/5/2012,1,1,1,4,1,0.265833,0.278412,0.524167,0.129987,140,3132,3272
+371,1/6/2012,1,1,1,5,1,0.334167,0.340267,0.542083,0.167908,307,3791,4098
+372,1/7/2012,1,1,1,6,1,0.393333,0.390779,0.531667,0.174758,1070,3451,4521
+373,1/8/2012,1,1,1,0,1,0.3375,0.340258,0.465,0.191542,599,2826,3425
+374,1/9/2012,1,1,1,1,2,0.224167,0.247479,0.701667,0.0989,106,2270,2376
+375,1/10/2012,1,1,1,2,1,0.308696,0.318826,0.646522,0.187552,173,3425,3598
+376,1/11/2012,1,1,1,3,2,0.274167,0.282821,0.8475,0.131221,92,2085,2177
+377,1/12/2012,1,1,1,4,2,0.3825,0.381938,0.802917,0.180967,269,3828,4097
+378,1/13/2012,1,1,1,5,1,0.274167,0.249362,0.5075,0.378108,174,3040,3214
+379,1/14/2012,1,1,1,6,1,0.18,0.183087,0.4575,0.187183,333,2160,2493
+380,1/15/2012,1,1,1,0,1,0.166667,0.161625,0.419167,0.251258,284,2027,2311
+381,1/16/2012,1,1,1,1,1,0.19,0.190663,0.5225,0.231358,217,2081,2298
+382,1/17/2012,1,1,1,2,2,0.373043,0.364278,0.716087,0.34913,127,2808,2935
+383,1/18/2012,1,1,1,3,1,0.303333,0.275254,0.443333,0.415429,109,3267,3376
+384,1/19/2012,1,1,1,4,1,0.19,0.190038,0.4975,0.220158,130,3162,3292
+385,1/20/2012,1,1,1,5,2,0.2175,0.220958,0.45,0.20275,115,3048,3163
+386,1/21/2012,1,1,1,6,2,0.173333,0.174875,0.83125,0.222642,67,1234,1301
+387,1/22/2012,1,1,1,0,2,0.1625,0.16225,0.79625,0.199638,196,1781,1977
+388,1/23/2012,1,1,1,1,2,0.218333,0.243058,0.91125,0.110708,145,2287,2432
+389,1/24/2012,1,1,1,2,1,0.3425,0.349108,0.835833,0.123767,439,3900,4339
+390,1/25/2012,1,1,1,3,1,0.294167,0.294821,0.64375,0.161071,467,3803,4270
+391,1/26/2012,1,1,1,4,2,0.341667,0.35605,0.769583,0.0733958,244,3831,4075
+392,1/27/2012,1,1,1,5,2,0.425,0.415383,0.74125,0.342667,269,3187,3456
+393,1/28/2012,1,1,1,6,1,0.315833,0.326379,0.543333,0.210829,775,3248,4023
+394,1/29/2012,1,1,1,0,1,0.2825,0.272721,0.31125,0.24005,558,2685,3243
+395,1/30/2012,1,1,1,1,1,0.269167,0.262625,0.400833,0.215792,126,3498,3624
+396,1/31/2012,1,1,1,2,1,0.39,0.381317,0.416667,0.261817,324,4185,4509
+397,2/1/2012,1,1,2,3,1,0.469167,0.466538,0.507917,0.189067,304,4275,4579
+398,2/2/2012,1,1,2,4,2,0.399167,0.398971,0.672917,0.187187,190,3571,3761
+399,2/3/2012,1,1,2,5,1,0.313333,0.309346,0.526667,0.178496,310,3841,4151
+400,2/4/2012,1,1,2,6,2,0.264167,0.272725,0.779583,0.121896,384,2448,2832
+401,2/5/2012,1,1,2,0,2,0.265833,0.264521,0.687917,0.175996,318,2629,2947
+402,2/6/2012,1,1,2,1,1,0.282609,0.296426,0.622174,0.1538,206,3578,3784
+403,2/7/2012,1,1,2,2,1,0.354167,0.361104,0.49625,0.147379,199,4176,4375
+404,2/8/2012,1,1,2,3,2,0.256667,0.266421,0.722917,0.133721,109,2693,2802
+405,2/9/2012,1,1,2,4,1,0.265,0.261988,0.562083,0.194037,163,3667,3830
+406,2/10/2012,1,1,2,5,2,0.280833,0.293558,0.54,0.116929,227,3604,3831
+407,2/11/2012,1,1,2,6,3,0.224167,0.210867,0.73125,0.289796,192,1977,2169
+408,2/12/2012,1,1,2,0,1,0.1275,0.101658,0.464583,0.409212,73,1456,1529
+409,2/13/2012,1,1,2,1,1,0.2225,0.227913,0.41125,0.167283,94,3328,3422
+410,2/14/2012,1,1,2,2,2,0.319167,0.333946,0.50875,0.141179,135,3787,3922
+411,2/15/2012,1,1,2,3,1,0.348333,0.351629,0.53125,0.1816,141,4028,4169
+412,2/16/2012,1,1,2,4,2,0.316667,0.330162,0.752917,0.091425,74,2931,3005
+413,2/17/2012,1,1,2,5,1,0.343333,0.351629,0.634583,0.205846,349,3805,4154
+414,2/18/2012,1,1,2,6,1,0.346667,0.355425,0.534583,0.190929,1435,2883,4318
+415,2/19/2012,1,1,2,0,2,0.28,0.265788,0.515833,0.253112,618,2071,2689
+416,2/20/2012,1,1,2,1,1,0.28,0.273391,0.507826,0.229083,502,2627,3129
+417,2/21/2012,1,1,2,2,1,0.287826,0.295113,0.594348,0.205717,163,3614,3777
+418,2/22/2012,1,1,2,3,1,0.395833,0.392667,0.567917,0.234471,394,4379,4773
+419,2/23/2012,1,1,2,4,1,0.454167,0.444446,0.554583,0.190913,516,4546,5062
+420,2/24/2012,1,1,2,5,2,0.4075,0.410971,0.7375,0.237567,246,3241,3487
+421,2/25/2012,1,1,2,6,1,0.290833,0.255675,0.395833,0.421642,317,2415,2732
+422,2/26/2012,1,1,2,0,1,0.279167,0.268308,0.41,0.205229,515,2874,3389
+423,2/27/2012,1,1,2,1,1,0.366667,0.357954,0.490833,0.268033,253,4069,4322
+424,2/28/2012,1,1,2,2,1,0.359167,0.353525,0.395833,0.193417,229,4134,4363
+425,2/29/2012,1,1,2,3,2,0.344348,0.34847,0.804783,0.179117,65,1769,1834
+426,3/1/2012,1,1,3,4,1,0.485833,0.475371,0.615417,0.226987,325,4665,4990
+427,3/2/2012,1,1,3,5,2,0.353333,0.359842,0.657083,0.144904,246,2948,3194
+428,3/3/2012,1,1,3,6,2,0.414167,0.413492,0.62125,0.161079,956,3110,4066
+429,3/4/2012,1,1,3,0,1,0.325833,0.303021,0.403333,0.334571,710,2713,3423
+430,3/5/2012,1,1,3,1,1,0.243333,0.241171,0.50625,0.228858,203,3130,3333
+431,3/6/2012,1,1,3,2,1,0.258333,0.255042,0.456667,0.200875,221,3735,3956
+432,3/7/2012,1,1,3,3,1,0.404167,0.3851,0.513333,0.345779,432,4484,4916
+433,3/8/2012,1,1,3,4,1,0.5275,0.524604,0.5675,0.441563,486,4896,5382
+434,3/9/2012,1,1,3,5,2,0.410833,0.397083,0.407083,0.4148,447,4122,4569
+435,3/10/2012,1,1,3,6,1,0.2875,0.277767,0.350417,0.22575,968,3150,4118
+436,3/11/2012,1,1,3,0,1,0.361739,0.35967,0.476957,0.222587,1658,3253,4911
+437,3/12/2012,1,1,3,1,1,0.466667,0.459592,0.489167,0.207713,838,4460,5298
+438,3/13/2012,1,1,3,2,1,0.565,0.542929,0.6175,0.23695,762,5085,5847
+439,3/14/2012,1,1,3,3,1,0.5725,0.548617,0.507083,0.115062,997,5315,6312
+440,3/15/2012,1,1,3,4,1,0.5575,0.532825,0.579583,0.149883,1005,5187,6192
+441,3/16/2012,1,1,3,5,2,0.435833,0.436229,0.842083,0.113192,548,3830,4378
+442,3/17/2012,1,1,3,6,2,0.514167,0.505046,0.755833,0.110704,3155,4681,7836
+443,3/18/2012,1,1,3,0,2,0.4725,0.464,0.81,0.126883,2207,3685,5892
+444,3/19/2012,1,1,3,1,1,0.545,0.532821,0.72875,0.162317,982,5171,6153
+445,3/20/2012,1,1,3,2,1,0.560833,0.538533,0.807917,0.121271,1051,5042,6093
+446,3/21/2012,2,1,3,3,2,0.531667,0.513258,0.82125,0.0895583,1122,5108,6230
+447,3/22/2012,2,1,3,4,1,0.554167,0.531567,0.83125,0.117562,1334,5537,6871
+448,3/23/2012,2,1,3,5,2,0.601667,0.570067,0.694167,0.1163,2469,5893,8362
+449,3/24/2012,2,1,3,6,2,0.5025,0.486733,0.885417,0.192783,1033,2339,3372
+450,3/25/2012,2,1,3,0,2,0.4375,0.437488,0.880833,0.220775,1532,3464,4996
+451,3/26/2012,2,1,3,1,1,0.445833,0.43875,0.477917,0.386821,795,4763,5558
+452,3/27/2012,2,1,3,2,1,0.323333,0.315654,0.29,0.187192,531,4571,5102
+453,3/28/2012,2,1,3,3,1,0.484167,0.47095,0.48125,0.291671,674,5024,5698
+454,3/29/2012,2,1,3,4,1,0.494167,0.482304,0.439167,0.31965,834,5299,6133
+455,3/30/2012,2,1,3,5,2,0.37,0.375621,0.580833,0.138067,796,4663,5459
+456,3/31/2012,2,1,3,6,2,0.424167,0.421708,0.738333,0.250617,2301,3934,6235
+457,4/1/2012,2,1,4,0,2,0.425833,0.417287,0.67625,0.172267,2347,3694,6041
+458,4/2/2012,2,1,4,1,1,0.433913,0.427513,0.504348,0.312139,1208,4728,5936
+459,4/3/2012,2,1,4,2,1,0.466667,0.461483,0.396667,0.100133,1348,5424,6772
+460,4/4/2012,2,1,4,3,1,0.541667,0.53345,0.469583,0.180975,1058,5378,6436
+461,4/5/2012,2,1,4,4,1,0.435,0.431163,0.374167,0.219529,1192,5265,6457
+462,4/6/2012,2,1,4,5,1,0.403333,0.390767,0.377083,0.300388,1807,4653,6460
+463,4/7/2012,2,1,4,6,1,0.4375,0.426129,0.254167,0.274871,3252,3605,6857
+464,4/8/2012,2,1,4,0,1,0.5,0.492425,0.275833,0.232596,2230,2939,5169
+465,4/9/2012,2,1,4,1,1,0.489167,0.476638,0.3175,0.358196,905,4680,5585
+466,4/10/2012,2,1,4,2,1,0.446667,0.436233,0.435,0.249375,819,5099,5918
+467,4/11/2012,2,1,4,3,1,0.348696,0.337274,0.469565,0.295274,482,4380,4862
+468,4/12/2012,2,1,4,4,1,0.3975,0.387604,0.46625,0.290429,663,4746,5409
+469,4/13/2012,2,1,4,5,1,0.4425,0.431808,0.408333,0.155471,1252,5146,6398
+470,4/14/2012,2,1,4,6,1,0.495,0.487996,0.502917,0.190917,2795,4665,7460
+471,4/15/2012,2,1,4,0,1,0.606667,0.573875,0.507917,0.225129,2846,4286,7132
+472,4/16/2012,2,1,4,1,1,0.664167,0.614925,0.561667,0.284829,1198,5172,6370
+473,4/17/2012,2,1,4,2,1,0.608333,0.598487,0.390417,0.273629,989,5702,6691
+474,4/18/2012,2,1,4,3,2,0.463333,0.457038,0.569167,0.167912,347,4020,4367
+475,4/19/2012,2,1,4,4,1,0.498333,0.493046,0.6125,0.0659292,846,5719,6565
+476,4/20/2012,2,1,4,5,1,0.526667,0.515775,0.694583,0.149871,1340,5950,7290
+477,4/21/2012,2,1,4,6,1,0.57,0.542921,0.682917,0.283587,2541,4083,6624
+478,4/22/2012,2,1,4,0,3,0.396667,0.389504,0.835417,0.344546,120,907,1027
+479,4/23/2012,2,1,4,1,2,0.321667,0.301125,0.766667,0.303496,195,3019,3214
+480,4/24/2012,2,1,4,2,1,0.413333,0.405283,0.454167,0.249383,518,5115,5633
+481,4/25/2012,2,1,4,3,1,0.476667,0.470317,0.427917,0.118792,655,5541,6196
+482,4/26/2012,2,1,4,4,2,0.498333,0.483583,0.756667,0.176625,475,4551,5026
+483,4/27/2012,2,1,4,5,1,0.4575,0.452637,0.400833,0.347633,1014,5219,6233
+484,4/28/2012,2,1,4,6,2,0.376667,0.377504,0.489583,0.129975,1120,3100,4220
+485,4/29/2012,2,1,4,0,1,0.458333,0.450121,0.587083,0.116908,2229,4075,6304
+486,4/30/2012,2,1,4,1,2,0.464167,0.457696,0.57,0.171638,665,4907,5572
+487,5/1/2012,2,1,5,2,2,0.613333,0.577021,0.659583,0.156096,653,5087,5740
+488,5/2/2012,2,1,5,3,1,0.564167,0.537896,0.797083,0.138058,667,5502,6169
+489,5/3/2012,2,1,5,4,2,0.56,0.537242,0.768333,0.133696,764,5657,6421
+490,5/4/2012,2,1,5,5,1,0.6275,0.590917,0.735417,0.162938,1069,5227,6296
+491,5/5/2012,2,1,5,6,2,0.621667,0.584608,0.756667,0.152992,2496,4387,6883
+492,5/6/2012,2,1,5,0,2,0.5625,0.546737,0.74,0.149879,2135,4224,6359
+493,5/7/2012,2,1,5,1,2,0.5375,0.527142,0.664167,0.230721,1008,5265,6273
+494,5/8/2012,2,1,5,2,2,0.581667,0.557471,0.685833,0.296029,738,4990,5728
+495,5/9/2012,2,1,5,3,2,0.575,0.553025,0.744167,0.216412,620,4097,4717
+496,5/10/2012,2,1,5,4,1,0.505833,0.491783,0.552083,0.314063,1026,5546,6572
+497,5/11/2012,2,1,5,5,1,0.533333,0.520833,0.360417,0.236937,1319,5711,7030
+498,5/12/2012,2,1,5,6,1,0.564167,0.544817,0.480417,0.123133,2622,4807,7429
+499,5/13/2012,2,1,5,0,1,0.6125,0.585238,0.57625,0.225117,2172,3946,6118
+500,5/14/2012,2,1,5,1,2,0.573333,0.5499,0.789583,0.212692,342,2501,2843
+501,5/15/2012,2,1,5,2,2,0.611667,0.576404,0.794583,0.147392,625,4490,5115
+502,5/16/2012,2,1,5,3,1,0.636667,0.595975,0.697917,0.122512,991,6433,7424
+503,5/17/2012,2,1,5,4,1,0.593333,0.572613,0.52,0.229475,1242,6142,7384
+504,5/18/2012,2,1,5,5,1,0.564167,0.551121,0.523333,0.136817,1521,6118,7639
+505,5/19/2012,2,1,5,6,1,0.6,0.566908,0.45625,0.083975,3410,4884,8294
+506,5/20/2012,2,1,5,0,1,0.620833,0.583967,0.530417,0.254367,2704,4425,7129
+507,5/21/2012,2,1,5,1,2,0.598333,0.565667,0.81125,0.233204,630,3729,4359
+508,5/22/2012,2,1,5,2,2,0.615,0.580825,0.765833,0.118167,819,5254,6073
+509,5/23/2012,2,1,5,3,2,0.621667,0.584612,0.774583,0.102,766,4494,5260
+510,5/24/2012,2,1,5,4,1,0.655,0.6067,0.716667,0.172896,1059,5711,6770
+511,5/25/2012,2,1,5,5,1,0.68,0.627529,0.747083,0.14055,1417,5317,6734
+512,5/26/2012,2,1,5,6,1,0.6925,0.642696,0.7325,0.198992,2855,3681,6536
+513,5/27/2012,2,1,5,0,1,0.69,0.641425,0.697083,0.215171,3283,3308,6591
+514,5/28/2012,2,1,5,1,1,0.7125,0.6793,0.67625,0.196521,2557,3486,6043
+515,5/29/2012,2,1,5,2,1,0.7225,0.672992,0.684583,0.2954,880,4863,5743
+516,5/30/2012,2,1,5,3,2,0.656667,0.611129,0.67,0.134329,745,6110,6855
+517,5/31/2012,2,1,5,4,1,0.68,0.631329,0.492917,0.195279,1100,6238,7338
+518,6/1/2012,2,1,6,5,2,0.654167,0.607962,0.755417,0.237563,533,3594,4127
+519,6/2/2012,2,1,6,6,1,0.583333,0.566288,0.549167,0.186562,2795,5325,8120
+520,6/3/2012,2,1,6,0,1,0.6025,0.575133,0.493333,0.184087,2494,5147,7641
+521,6/4/2012,2,1,6,1,1,0.5975,0.578283,0.487083,0.284833,1071,5927,6998
+522,6/5/2012,2,1,6,2,2,0.540833,0.525892,0.613333,0.209575,968,6033,7001
+523,6/6/2012,2,1,6,3,1,0.554167,0.542292,0.61125,0.077125,1027,6028,7055
+524,6/7/2012,2,1,6,4,1,0.6025,0.569442,0.567083,0.15735,1038,6456,7494
+525,6/8/2012,2,1,6,5,1,0.649167,0.597862,0.467917,0.175383,1488,6248,7736
+526,6/9/2012,2,1,6,6,1,0.710833,0.648367,0.437083,0.144287,2708,4790,7498
+527,6/10/2012,2,1,6,0,1,0.726667,0.663517,0.538333,0.133721,2224,4374,6598
+528,6/11/2012,2,1,6,1,2,0.720833,0.659721,0.587917,0.207713,1017,5647,6664
+529,6/12/2012,2,1,6,2,2,0.653333,0.597875,0.833333,0.214546,477,4495,4972
+530,6/13/2012,2,1,6,3,1,0.655833,0.611117,0.582083,0.343279,1173,6248,7421
+531,6/14/2012,2,1,6,4,1,0.648333,0.624383,0.569583,0.253733,1180,6183,7363
+532,6/15/2012,2,1,6,5,1,0.639167,0.599754,0.589583,0.176617,1563,6102,7665
+533,6/16/2012,2,1,6,6,1,0.631667,0.594708,0.504167,0.166667,2963,4739,7702
+534,6/17/2012,2,1,6,0,1,0.5925,0.571975,0.59875,0.144904,2634,4344,6978
+535,6/18/2012,2,1,6,1,2,0.568333,0.544842,0.777917,0.174746,653,4446,5099
+536,6/19/2012,2,1,6,2,1,0.688333,0.654692,0.69,0.148017,968,5857,6825
+537,6/20/2012,2,1,6,3,1,0.7825,0.720975,0.592083,0.113812,872,5339,6211
+538,6/21/2012,3,1,6,4,1,0.805833,0.752542,0.567917,0.118787,778,5127,5905
+539,6/22/2012,3,1,6,5,1,0.7775,0.724121,0.57375,0.182842,964,4859,5823
+540,6/23/2012,3,1,6,6,1,0.731667,0.652792,0.534583,0.179721,2657,4801,7458
+541,6/24/2012,3,1,6,0,1,0.743333,0.674254,0.479167,0.145525,2551,4340,6891
+542,6/25/2012,3,1,6,1,1,0.715833,0.654042,0.504167,0.300383,1139,5640,6779
+543,6/26/2012,3,1,6,2,1,0.630833,0.594704,0.373333,0.347642,1077,6365,7442
+544,6/27/2012,3,1,6,3,1,0.6975,0.640792,0.36,0.271775,1077,6258,7335
+545,6/28/2012,3,1,6,4,1,0.749167,0.675512,0.4225,0.17165,921,5958,6879
+546,6/29/2012,3,1,6,5,1,0.834167,0.786613,0.48875,0.165417,829,4634,5463
+547,6/30/2012,3,1,6,6,1,0.765,0.687508,0.60125,0.161071,1455,4232,5687
+548,7/1/2012,3,1,7,0,1,0.815833,0.750629,0.51875,0.168529,1421,4110,5531
+549,7/2/2012,3,1,7,1,1,0.781667,0.702038,0.447083,0.195267,904,5323,6227
+550,7/3/2012,3,1,7,2,1,0.780833,0.70265,0.492083,0.126237,1052,5608,6660
+551,7/4/2012,3,1,7,3,1,0.789167,0.732337,0.53875,0.13495,2562,4841,7403
+552,7/5/2012,3,1,7,4,1,0.8275,0.761367,0.457917,0.194029,1405,4836,6241
+553,7/6/2012,3,1,7,5,1,0.828333,0.752533,0.450833,0.146142,1366,4841,6207
+554,7/7/2012,3,1,7,6,1,0.861667,0.804913,0.492083,0.163554,1448,3392,4840
+555,7/8/2012,3,1,7,0,1,0.8225,0.790396,0.57375,0.125629,1203,3469,4672
+556,7/9/2012,3,1,7,1,2,0.710833,0.654054,0.683333,0.180975,998,5571,6569
+557,7/10/2012,3,1,7,2,2,0.720833,0.664796,0.6675,0.151737,954,5336,6290
+558,7/11/2012,3,1,7,3,1,0.716667,0.650271,0.633333,0.151733,975,6289,7264
+559,7/12/2012,3,1,7,4,1,0.715833,0.654683,0.529583,0.146775,1032,6414,7446
+560,7/13/2012,3,1,7,5,2,0.731667,0.667933,0.485833,0.08085,1511,5988,7499
+561,7/14/2012,3,1,7,6,2,0.703333,0.666042,0.699167,0.143679,2355,4614,6969
+562,7/15/2012,3,1,7,0,1,0.745833,0.705196,0.717917,0.166667,1920,4111,6031
+563,7/16/2012,3,1,7,1,1,0.763333,0.724125,0.645,0.164187,1088,5742,6830
+564,7/17/2012,3,1,7,2,1,0.818333,0.755683,0.505833,0.114429,921,5865,6786
+565,7/18/2012,3,1,7,3,1,0.793333,0.745583,0.577083,0.137442,799,4914,5713
+566,7/19/2012,3,1,7,4,1,0.77,0.714642,0.600417,0.165429,888,5703,6591
+567,7/20/2012,3,1,7,5,2,0.665833,0.613025,0.844167,0.208967,747,5123,5870
+568,7/21/2012,3,1,7,6,3,0.595833,0.549912,0.865417,0.2133,1264,3195,4459
+569,7/22/2012,3,1,7,0,2,0.6675,0.623125,0.7625,0.0939208,2544,4866,7410
+570,7/23/2012,3,1,7,1,1,0.741667,0.690017,0.694167,0.138683,1135,5831,6966
+571,7/24/2012,3,1,7,2,1,0.750833,0.70645,0.655,0.211454,1140,6452,7592
+572,7/25/2012,3,1,7,3,1,0.724167,0.654054,0.45,0.1648,1383,6790,8173
+573,7/26/2012,3,1,7,4,1,0.776667,0.739263,0.596667,0.284813,1036,5825,6861
+574,7/27/2012,3,1,7,5,1,0.781667,0.734217,0.594583,0.152992,1259,5645,6904
+575,7/28/2012,3,1,7,6,1,0.755833,0.697604,0.613333,0.15735,2234,4451,6685
+576,7/29/2012,3,1,7,0,1,0.721667,0.667933,0.62375,0.170396,2153,4444,6597
+577,7/30/2012,3,1,7,1,1,0.730833,0.684987,0.66875,0.153617,1040,6065,7105
+578,7/31/2012,3,1,7,2,1,0.713333,0.662896,0.704167,0.165425,968,6248,7216
+579,8/1/2012,3,1,8,3,1,0.7175,0.667308,0.6775,0.141179,1074,6506,7580
+580,8/2/2012,3,1,8,4,1,0.7525,0.707088,0.659583,0.129354,983,6278,7261
+581,8/3/2012,3,1,8,5,2,0.765833,0.722867,0.6425,0.215792,1328,5847,7175
+582,8/4/2012,3,1,8,6,1,0.793333,0.751267,0.613333,0.257458,2345,4479,6824
+583,8/5/2012,3,1,8,0,1,0.769167,0.731079,0.6525,0.290421,1707,3757,5464
+584,8/6/2012,3,1,8,1,2,0.7525,0.710246,0.654167,0.129354,1233,5780,7013
+585,8/7/2012,3,1,8,2,2,0.735833,0.697621,0.70375,0.116908,1278,5995,7273
+586,8/8/2012,3,1,8,3,2,0.75,0.707717,0.672917,0.1107,1263,6271,7534
+587,8/9/2012,3,1,8,4,1,0.755833,0.699508,0.620417,0.1561,1196,6090,7286
+588,8/10/2012,3,1,8,5,2,0.715833,0.667942,0.715833,0.238813,1065,4721,5786
+589,8/11/2012,3,1,8,6,2,0.6925,0.638267,0.732917,0.206479,2247,4052,6299
+590,8/12/2012,3,1,8,0,1,0.700833,0.644579,0.530417,0.122512,2182,4362,6544
+591,8/13/2012,3,1,8,1,1,0.720833,0.662254,0.545417,0.136212,1207,5676,6883
+592,8/14/2012,3,1,8,2,1,0.726667,0.676779,0.686667,0.169158,1128,5656,6784
+593,8/15/2012,3,1,8,3,1,0.706667,0.654037,0.619583,0.169771,1198,6149,7347
+594,8/16/2012,3,1,8,4,1,0.719167,0.654688,0.519167,0.141796,1338,6267,7605
+595,8/17/2012,3,1,8,5,1,0.723333,0.2424,0.570833,0.231354,1483,5665,7148
+596,8/18/2012,3,1,8,6,1,0.678333,0.618071,0.603333,0.177867,2827,5038,7865
+597,8/19/2012,3,1,8,0,2,0.635833,0.603554,0.711667,0.08645,1208,3341,4549
+598,8/20/2012,3,1,8,1,2,0.635833,0.595967,0.734167,0.129979,1026,5504,6530
+599,8/21/2012,3,1,8,2,1,0.649167,0.601025,0.67375,0.0727708,1081,5925,7006
+600,8/22/2012,3,1,8,3,1,0.6675,0.621854,0.677083,0.0702833,1094,6281,7375
+601,8/23/2012,3,1,8,4,1,0.695833,0.637008,0.635833,0.0845958,1363,6402,7765
+602,8/24/2012,3,1,8,5,2,0.7025,0.6471,0.615,0.0721458,1325,6257,7582
+603,8/25/2012,3,1,8,6,2,0.661667,0.618696,0.712917,0.244408,1829,4224,6053
+604,8/26/2012,3,1,8,0,2,0.653333,0.595996,0.845833,0.228858,1483,3772,5255
+605,8/27/2012,3,1,8,1,1,0.703333,0.654688,0.730417,0.128733,989,5928,6917
+606,8/28/2012,3,1,8,2,1,0.728333,0.66605,0.62,0.190925,935,6105,7040
+607,8/29/2012,3,1,8,3,1,0.685,0.635733,0.552083,0.112562,1177,6520,7697
+608,8/30/2012,3,1,8,4,1,0.706667,0.652779,0.590417,0.0771167,1172,6541,7713
+609,8/31/2012,3,1,8,5,1,0.764167,0.6894,0.5875,0.168533,1433,5917,7350
+610,9/1/2012,3,1,9,6,2,0.753333,0.702654,0.638333,0.113187,2352,3788,6140
+611,9/2/2012,3,1,9,0,2,0.696667,0.649,0.815,0.0640708,2613,3197,5810
+612,9/3/2012,3,1,9,1,1,0.7075,0.661629,0.790833,0.151121,1965,4069,6034
+613,9/4/2012,3,1,9,2,1,0.725833,0.686888,0.755,0.236321,867,5997,6864
+614,9/5/2012,3,1,9,3,1,0.736667,0.708983,0.74125,0.187808,832,6280,7112
+615,9/6/2012,3,1,9,4,2,0.696667,0.655329,0.810417,0.142421,611,5592,6203
+616,9/7/2012,3,1,9,5,1,0.703333,0.657204,0.73625,0.171646,1045,6459,7504
+617,9/8/2012,3,1,9,6,2,0.659167,0.611121,0.799167,0.281104,1557,4419,5976
+618,9/9/2012,3,1,9,0,1,0.61,0.578925,0.5475,0.224496,2570,5657,8227
+619,9/10/2012,3,1,9,1,1,0.583333,0.565654,0.50375,0.258713,1118,6407,7525
+620,9/11/2012,3,1,9,2,1,0.5775,0.554292,0.52,0.0920542,1070,6697,7767
+621,9/12/2012,3,1,9,3,1,0.599167,0.570075,0.577083,0.131846,1050,6820,7870
+622,9/13/2012,3,1,9,4,1,0.6125,0.579558,0.637083,0.0827208,1054,6750,7804
+623,9/14/2012,3,1,9,5,1,0.633333,0.594083,0.6725,0.103863,1379,6630,8009
+624,9/15/2012,3,1,9,6,1,0.608333,0.585867,0.501667,0.247521,3160,5554,8714
+625,9/16/2012,3,1,9,0,1,0.58,0.563125,0.57,0.0901833,2166,5167,7333
+626,9/17/2012,3,1,9,1,2,0.580833,0.55305,0.734583,0.151742,1022,5847,6869
+627,9/18/2012,3,1,9,2,2,0.623333,0.565067,0.8725,0.357587,371,3702,4073
+628,9/19/2012,3,1,9,3,1,0.5525,0.540404,0.536667,0.215175,788,6803,7591
+629,9/20/2012,3,1,9,4,1,0.546667,0.532192,0.618333,0.118167,939,6781,7720
+630,9/21/2012,3,1,9,5,1,0.599167,0.571971,0.66875,0.154229,1250,6917,8167
+631,9/22/2012,3,1,9,6,1,0.65,0.610488,0.646667,0.283583,2512,5883,8395
+632,9/23/2012,4,1,9,0,1,0.529167,0.518933,0.467083,0.223258,2454,5453,7907
+633,9/24/2012,4,1,9,1,1,0.514167,0.502513,0.492917,0.142404,1001,6435,7436
+634,9/25/2012,4,1,9,2,1,0.55,0.544179,0.57,0.236321,845,6693,7538
+635,9/26/2012,4,1,9,3,1,0.635,0.596613,0.630833,0.2444,787,6946,7733
+636,9/27/2012,4,1,9,4,2,0.65,0.607975,0.690833,0.134342,751,6642,7393
+637,9/28/2012,4,1,9,5,2,0.619167,0.585863,0.69,0.164179,1045,6370,7415
+638,9/29/2012,4,1,9,6,1,0.5425,0.530296,0.542917,0.227604,2589,5966,8555
+639,9/30/2012,4,1,9,0,1,0.526667,0.517663,0.583333,0.134958,2015,4874,6889
+640,10/1/2012,4,1,10,1,2,0.520833,0.512,0.649167,0.0908042,763,6015,6778
+641,10/2/2012,4,1,10,2,3,0.590833,0.542333,0.871667,0.104475,315,4324,4639
+642,10/3/2012,4,1,10,3,2,0.6575,0.599133,0.79375,0.0665458,728,6844,7572
+643,10/4/2012,4,1,10,4,2,0.6575,0.607975,0.722917,0.117546,891,6437,7328
+644,10/5/2012,4,1,10,5,1,0.615,0.580187,0.6275,0.10635,1516,6640,8156
+645,10/6/2012,4,1,10,6,1,0.554167,0.538521,0.664167,0.268025,3031,4934,7965
+646,10/7/2012,4,1,10,0,2,0.415833,0.419813,0.708333,0.141162,781,2729,3510
+647,10/8/2012,4,1,10,1,2,0.383333,0.387608,0.709583,0.189679,874,4604,5478
+648,10/9/2012,4,1,10,2,2,0.446667,0.438112,0.761667,0.1903,601,5791,6392
+649,10/10/2012,4,1,10,3,1,0.514167,0.503142,0.630833,0.187821,780,6911,7691
+650,10/11/2012,4,1,10,4,1,0.435,0.431167,0.463333,0.181596,834,6736,7570
+651,10/12/2012,4,1,10,5,1,0.4375,0.433071,0.539167,0.235092,1060,6222,7282
+652,10/13/2012,4,1,10,6,1,0.393333,0.391396,0.494583,0.146142,2252,4857,7109
+653,10/14/2012,4,1,10,0,1,0.521667,0.508204,0.640417,0.278612,2080,4559,6639
+654,10/15/2012,4,1,10,1,2,0.561667,0.53915,0.7075,0.296037,760,5115,5875
+655,10/16/2012,4,1,10,2,1,0.468333,0.460846,0.558333,0.182221,922,6612,7534
+656,10/17/2012,4,1,10,3,1,0.455833,0.450108,0.692917,0.101371,979,6482,7461
+657,10/18/2012,4,1,10,4,2,0.5225,0.512625,0.728333,0.236937,1008,6501,7509
+658,10/19/2012,4,1,10,5,2,0.563333,0.537896,0.815,0.134954,753,4671,5424
+659,10/20/2012,4,1,10,6,1,0.484167,0.472842,0.572917,0.117537,2806,5284,8090
+660,10/21/2012,4,1,10,0,1,0.464167,0.456429,0.51,0.166054,2132,4692,6824
+661,10/22/2012,4,1,10,1,1,0.4875,0.482942,0.568333,0.0814833,830,6228,7058
+662,10/23/2012,4,1,10,2,1,0.544167,0.530304,0.641667,0.0945458,841,6625,7466
+663,10/24/2012,4,1,10,3,1,0.5875,0.558721,0.63625,0.0727792,795,6898,7693
+664,10/25/2012,4,1,10,4,2,0.55,0.529688,0.800417,0.124375,875,6484,7359
+665,10/26/2012,4,1,10,5,2,0.545833,0.52275,0.807083,0.132467,1182,6262,7444
+666,10/27/2012,4,1,10,6,2,0.53,0.515133,0.72,0.235692,2643,5209,7852
+667,10/28/2012,4,1,10,0,2,0.4775,0.467771,0.694583,0.398008,998,3461,4459
+668,10/29/2012,4,1,10,1,3,0.44,0.4394,0.88,0.3582,2,20,22
+669,10/30/2012,4,1,10,2,2,0.318182,0.309909,0.825455,0.213009,87,1009,1096
+670,10/31/2012,4,1,10,3,2,0.3575,0.3611,0.666667,0.166667,419,5147,5566
+671,11/1/2012,4,1,11,4,2,0.365833,0.369942,0.581667,0.157346,466,5520,5986
+672,11/2/2012,4,1,11,5,1,0.355,0.356042,0.522083,0.266175,618,5229,5847
+673,11/3/2012,4,1,11,6,2,0.343333,0.323846,0.49125,0.270529,1029,4109,5138
+674,11/4/2012,4,1,11,0,1,0.325833,0.329538,0.532917,0.179108,1201,3906,5107
+675,11/5/2012,4,1,11,1,1,0.319167,0.308075,0.494167,0.236325,378,4881,5259
+676,11/6/2012,4,1,11,2,1,0.280833,0.281567,0.567083,0.173513,466,5220,5686
+677,11/7/2012,4,1,11,3,2,0.295833,0.274621,0.5475,0.304108,326,4709,5035
+678,11/8/2012,4,1,11,4,1,0.352174,0.341891,0.333478,0.347835,340,4975,5315
+679,11/9/2012,4,1,11,5,1,0.361667,0.355413,0.540833,0.214558,709,5283,5992
+680,11/10/2012,4,1,11,6,1,0.389167,0.393937,0.645417,0.0578458,2090,4446,6536
+681,11/11/2012,4,1,11,0,1,0.420833,0.421713,0.659167,0.1275,2290,4562,6852
+682,11/12/2012,4,1,11,1,1,0.485,0.475383,0.741667,0.173517,1097,5172,6269
+683,11/13/2012,4,1,11,2,2,0.343333,0.323225,0.662917,0.342046,327,3767,4094
+684,11/14/2012,4,1,11,3,1,0.289167,0.281563,0.552083,0.199625,373,5122,5495
+685,11/15/2012,4,1,11,4,2,0.321667,0.324492,0.620417,0.152987,320,5125,5445
+686,11/16/2012,4,1,11,5,1,0.345,0.347204,0.524583,0.171025,484,5214,5698
+687,11/17/2012,4,1,11,6,1,0.325,0.326383,0.545417,0.179729,1313,4316,5629
+688,11/18/2012,4,1,11,0,1,0.3425,0.337746,0.692917,0.227612,922,3747,4669
+689,11/19/2012,4,1,11,1,2,0.380833,0.375621,0.623333,0.235067,449,5050,5499
+690,11/20/2012,4,1,11,2,2,0.374167,0.380667,0.685,0.082725,534,5100,5634
+691,11/21/2012,4,1,11,3,1,0.353333,0.364892,0.61375,0.103246,615,4531,5146
+692,11/22/2012,4,1,11,4,1,0.34,0.350371,0.580417,0.0528708,955,1470,2425
+693,11/23/2012,4,1,11,5,1,0.368333,0.378779,0.56875,0.148021,1603,2307,3910
+694,11/24/2012,4,1,11,6,1,0.278333,0.248742,0.404583,0.376871,532,1745,2277
+695,11/25/2012,4,1,11,0,1,0.245833,0.257583,0.468333,0.1505,309,2115,2424
+696,11/26/2012,4,1,11,1,1,0.313333,0.339004,0.535417,0.04665,337,4750,5087
+697,11/27/2012,4,1,11,2,2,0.291667,0.281558,0.786667,0.237562,123,3836,3959
+698,11/28/2012,4,1,11,3,1,0.296667,0.289762,0.50625,0.210821,198,5062,5260
+699,11/29/2012,4,1,11,4,1,0.28087,0.298422,0.555652,0.115522,243,5080,5323
+700,11/30/2012,4,1,11,5,1,0.298333,0.323867,0.649583,0.0584708,362,5306,5668
+701,12/1/2012,4,1,12,6,2,0.298333,0.316904,0.806667,0.0597042,951,4240,5191
+702,12/2/2012,4,1,12,0,2,0.3475,0.359208,0.823333,0.124379,892,3757,4649
+703,12/3/2012,4,1,12,1,1,0.4525,0.455796,0.7675,0.0827208,555,5679,6234
+704,12/4/2012,4,1,12,2,1,0.475833,0.469054,0.73375,0.174129,551,6055,6606
+705,12/5/2012,4,1,12,3,1,0.438333,0.428012,0.485,0.324021,331,5398,5729
+706,12/6/2012,4,1,12,4,1,0.255833,0.258204,0.50875,0.174754,340,5035,5375
+707,12/7/2012,4,1,12,5,2,0.320833,0.321958,0.764167,0.1306,349,4659,5008
+708,12/8/2012,4,1,12,6,2,0.381667,0.389508,0.91125,0.101379,1153,4429,5582
+709,12/9/2012,4,1,12,0,2,0.384167,0.390146,0.905417,0.157975,441,2787,3228
+710,12/10/2012,4,1,12,1,2,0.435833,0.435575,0.925,0.190308,329,4841,5170
+711,12/11/2012,4,1,12,2,2,0.353333,0.338363,0.596667,0.296037,282,5219,5501
+712,12/12/2012,4,1,12,3,2,0.2975,0.297338,0.538333,0.162937,310,5009,5319
+713,12/13/2012,4,1,12,4,1,0.295833,0.294188,0.485833,0.174129,425,5107,5532
+714,12/14/2012,4,1,12,5,1,0.281667,0.294192,0.642917,0.131229,429,5182,5611
+715,12/15/2012,4,1,12,6,1,0.324167,0.338383,0.650417,0.10635,767,4280,5047
+716,12/16/2012,4,1,12,0,2,0.3625,0.369938,0.83875,0.100742,538,3248,3786
+717,12/17/2012,4,1,12,1,2,0.393333,0.4015,0.907083,0.0982583,212,4373,4585
+718,12/18/2012,4,1,12,2,1,0.410833,0.409708,0.66625,0.221404,433,5124,5557
+719,12/19/2012,4,1,12,3,1,0.3325,0.342162,0.625417,0.184092,333,4934,5267
+720,12/20/2012,4,1,12,4,2,0.33,0.335217,0.667917,0.132463,314,3814,4128
+721,12/21/2012,1,1,12,5,2,0.326667,0.301767,0.556667,0.374383,221,3402,3623
+722,12/22/2012,1,1,12,6,1,0.265833,0.236113,0.44125,0.407346,205,1544,1749
+723,12/23/2012,1,1,12,0,1,0.245833,0.259471,0.515417,0.133083,408,1379,1787
+724,12/24/2012,1,1,12,1,2,0.231304,0.2589,0.791304,0.0772304,174,746,920
+725,12/25/2012,1,1,12,2,2,0.291304,0.294465,0.734783,0.168726,440,573,1013
+726,12/26/2012,1,1,12,3,3,0.243333,0.220333,0.823333,0.316546,9,432,441
+727,12/27/2012,1,1,12,4,2,0.254167,0.226642,0.652917,0.350133,247,1867,2114
+728,12/28/2012,1,1,12,5,2,0.253333,0.255046,0.59,0.155471,644,2451,3095
+729,12/29/2012,1,1,12,6,2,0.253333,0.2424,0.752917,0.124383,159,1182,1341
+730,12/30/2012,1,1,12,0,1,0.255833,0.2317,0.483333,0.350754,364,1432,1796
+731,12/31/2012,1,1,12,1,2,0.215833,0.223487,0.5775,0.154846,439,2290,2729

how-to-use-azureml/automated-machine-learning/forecasting-bike-share/forecasting_helper.py

@@ -0,0 +1,99 @@
+import pandas as pd
+import numpy as np
+from pandas.tseries.frequencies import to_offset
+
+
+def align_outputs(y_predicted, X_trans, X_test, y_test, target_column_name,
+                  predicted_column_name='predicted',
+                  horizon_colname='horizon_origin'):
+    """
+    Demonstrates how to get the output aligned to the inputs
+    using pandas indexes. Helps understand what happened if
+    the output's shape differs from the input shape, or if
+    the data got re-sorted by time and grain during forecasting.
+
+    Typical causes of misalignment are:
+    * we predicted some periods that were missing in actuals -> drop from eval
+    * model was asked to predict past max_horizon -> increase max horizon
+    * data at start of X_test was needed for lags -> provide previous periods
+    """
+
+    if (horizon_colname in X_trans):
+        df_fcst = pd.DataFrame({predicted_column_name: y_predicted,
+                                horizon_colname: X_trans[horizon_colname]})
+    else:
+        df_fcst = pd.DataFrame({predicted_column_name: y_predicted})
+
+    # y and X outputs are aligned by forecast() function contract
+    df_fcst.index = X_trans.index
+
+    # align original X_test to y_test
+    X_test_full = X_test.copy()
+    X_test_full[target_column_name] = y_test
+
+    # X_test_full's index does not include origin, so reset for merge
+    df_fcst.reset_index(inplace=True)
+    X_test_full = X_test_full.reset_index().drop(columns='index')
+    together = df_fcst.merge(X_test_full, how='right')
+
+    # drop rows where prediction or actuals are nan
+    # happens because of missing actuals
+    # or at edges of time due to lags/rolling windows
+    clean = together[together[[target_column_name,
+                               predicted_column_name]].notnull().all(axis=1)]
+    return(clean)
+
+
+def do_rolling_forecast(fitted_model, X_test, y_test, target_column_name,
+                        time_column_name, max_horizon, freq='D'):
+    """
+    Produce forecasts on a rolling origin over the given test set.
+
+    Each iteration makes a forecast for the next 'max_horizon' periods
+    with respect to the current origin, then advances the origin by the
+    horizon time duration. The prediction context for each forecast is set so
+    that the forecaster uses the actual target values prior to the current
+    origin time for constructing lag features.
+
+    This function returns a concatenated DataFrame of rolling forecasts.
+     """
+    df_list = []
+    origin_time = X_test[time_column_name].min()
+    while origin_time <= X_test[time_column_name].max():
+        # Set the horizon time - end date of the forecast
+        horizon_time = origin_time + max_horizon * to_offset(freq)
+
+        # Extract test data from an expanding window up-to the horizon
+        expand_wind = (X_test[time_column_name] < horizon_time)
+        X_test_expand = X_test[expand_wind]
+        y_query_expand = np.zeros(len(X_test_expand)).astype(np.float)
+        y_query_expand.fill(np.NaN)
+
+        if origin_time != X_test[time_column_name].min():
+            # Set the context by including actuals up-to the origin time
+            test_context_expand_wind = (X_test[time_column_name] < origin_time)
+            context_expand_wind = (
+                X_test_expand[time_column_name] < origin_time)
+            y_query_expand[context_expand_wind] = y_test[
+                test_context_expand_wind]
+
+        # Make a forecast out to the maximum horizon
+        y_fcst, X_trans = fitted_model.forecast(X_test_expand, y_query_expand)
+
+        # Align forecast with test set for dates within the
+        # current rolling window
+        trans_tindex = X_trans.index.get_level_values(time_column_name)
+        trans_roll_wind = (trans_tindex >= origin_time) & (
+            trans_tindex < horizon_time)
+        test_roll_wind = expand_wind & (
+            X_test[time_column_name] >= origin_time)
+        df_list.append(align_outputs(y_fcst[trans_roll_wind],
+                                     X_trans[trans_roll_wind],
+                                     X_test[test_roll_wind],
+                                     y_test[test_roll_wind],
+                                     target_column_name))
+
+        # Advance the origin time
+        origin_time = horizon_time
+
+    return pd.concat(df_list, ignore_index=True)

how-to-use-azureml/automated-machine-learning/forecasting-bike-share/forecasting_script.py

@@ -0,0 +1,55 @@
+import argparse
+import azureml.train.automl
+from azureml.automl.runtime._vendor.automl.client.core.runtime import forecasting_models
+from azureml.core import Run
+from sklearn.externals import joblib
+import forecasting_helper
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    '--max_horizon', type=int, dest='max_horizon',
+    default=10, help='Max Horizon for forecasting')
+parser.add_argument(
+    '--target_column_name', type=str, dest='target_column_name',
+    help='Target Column Name')
+parser.add_argument(
+    '--time_column_name', type=str, dest='time_column_name',
+    help='Time Column Name')
+parser.add_argument(
+    '--frequency', type=str, dest='freq',
+    help='Frequency of prediction')
+
+args = parser.parse_args()
+max_horizon = args.max_horizon
+target_column_name = args.target_column_name
+time_column_name = args.time_column_name
+freq = args.freq
+
+run = Run.get_context()
+# get input dataset by name
+test_dataset = run.input_datasets['test_data']
+
+grain_column_names = []
+
+df = test_dataset.to_pandas_dataframe().reset_index(drop=True)
+
+X_test_df = test_dataset.drop_columns(columns=[target_column_name]).to_pandas_dataframe().reset_index(drop=True)
+y_test_df = test_dataset.with_timestamp_columns(None).keep_columns(columns=[target_column_name]).to_pandas_dataframe()
+
+fitted_model = joblib.load('model.pkl')
+
+df_all = forecasting_helper.do_rolling_forecast(
+    fitted_model,
+    X_test_df,
+    y_test_df.values.T[0],
+    target_column_name,
+    time_column_name,
+    max_horizon,
+    freq)
+
+file_name = 'outputs/predictions.csv'
+export_csv = df_all.to_csv(file_name, header=True)
+
+# Upload the predictions into artifacts
+run.upload_file(name=file_name, path_or_stream=file_name)

how-to-use-azureml/automated-machine-learning/forecasting-bike-share/metrics_helper.py

@@ -0,0 +1,22 @@
+import pandas as pd
+import numpy as np
+
+
+def APE(actual, pred):
+    """
+    Calculate absolute percentage error.
+    Returns a vector of APE values with same length as actual/pred.
+    """
+    return 100 * np.abs((actual - pred) / actual)
+
+
+def MAPE(actual, pred):
+    """
+    Calculate mean absolute percentage error.
+    Remove NA and values where actual is close to zero
+    """
+    not_na = ~(np.isnan(actual) | np.isnan(pred))
+    not_zero = ~np.isclose(actual, 0.0)
+    actual_safe = actual[not_na & not_zero]
+    pred_safe = pred[not_na & not_zero]
+    return np.mean(APE(actual_safe, pred_safe))

how-to-use-azureml/automated-machine-learning/forecasting-bike-share/run_forecast.py

@@ -0,0 +1,41 @@
+from azureml.core import Environment
+from azureml.core.conda_dependencies import CondaDependencies
+from azureml.train.estimator import Estimator
+from azureml.core.run import Run
+
+
+def run_rolling_forecast(test_experiment, compute_target, train_run, test_dataset,
+                         max_horizon, target_column_name, time_column_name,
+                         freq='D', inference_folder='./forecast'):
+    condafile = inference_folder + '/condafile.yml'
+    train_run.download_file('outputs/model.pkl',
+                            inference_folder + '/model.pkl')
+    train_run.download_file('outputs/conda_env_v_1_0_0.yml', condafile)
+
+    inference_env = Environment("myenv")
+    inference_env.docker.enabled = True
+    inference_env.python.conda_dependencies = CondaDependencies(
+        conda_dependencies_file_path=condafile)
+
+    est = Estimator(source_directory=inference_folder,
+                    entry_script='forecasting_script.py',
+                    script_params={
+                        '--max_horizon': max_horizon,
+                        '--target_column_name': target_column_name,
+                        '--time_column_name': time_column_name,
+                        '--frequency': freq
+                    },
+                    inputs=[test_dataset.as_named_input('test_data')],
+                    compute_target=compute_target,
+                    environment_definition=inference_env)
+
+    run = test_experiment.submit(est,
+                                 tags={
+                                     'training_run_id': train_run.id,
+                                     'run_algorithm': train_run.properties['run_algorithm'],
+                                     'valid_score': train_run.properties['score'],
+                                     'primary_metric': train_run.properties['primary_metric']
+                                 })
+
+    run.log("run_algorithm", run.tags['run_algorithm'])
+    return run

how-to-use-azureml/automated-machine-learning/forecasting-energy-demand/auto-ml-forecasting-energy-demand.ipynb

@@ -9,18 +9,30 @@
         "Licensed under the MIT License."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/automated-machine-learning/forecasting-energy-demand/auto-ml-forecasting-energy-demand.png)"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
         "# Automated Machine Learning\n",
-        "_**Energy Demand Forecasting**_\n",
+        "_**Forecasting using the Energy Demand Dataset**_\n",
         "\n",
         "## Contents\n",
         "1. [Introduction](#Introduction)\n",
         "1. [Setup](#Setup)\n",
-        "1. [Data](#Data)\n",
-        "1. [Train](#Train)"
+        "1. [Data and Forecasting Configurations](#Data)\n",
+        "1. [Train](#Train)\n",
+        "1. [Results](#Results)\n",
+        "\n",
+        "Advanced Forecasting\n",
+        "1. [Advanced Training](#advanced_training)\n",
+        "1. [Advanced Results](#advanced_results)"
       ]
     },
     {
@@ -28,23 +40,25 @@
       "metadata": {},
       "source": [
         "## Introduction\n",
-        "In this example, we show how AutoML can be used for energy demand forecasting.\n",
         "\n",
-        "Make sure you have executed the [configuration](../../../configuration.ipynb) before running this notebook.\n",
+        "In this example we use the associated New York City energy demand dataset to showcase how you can use AutoML for a simple forecasting problem and explore the results. The goal is predict the energy demand for the next 48 hours based on historic time-series data.\n",
         "\n",
-        "In this notebook you would see\n",
-        "1. Creating an Experiment in an existing Workspace\n",
-        "2. Instantiating AutoMLConfig with new task type \"forecasting\" for timeseries data training, and other timeseries related settings: for this dataset we use the basic one: \"time_column_name\" \n",
-        "3. Training the Model using local compute\n",
-        "4. Exploring the results\n",
-        "5. Testing the fitted model"
+        "If you are using an Azure Machine Learning [Notebook VM](https://docs.microsoft.com/en-us/azure/machine-learning/service/tutorial-1st-experiment-sdk-setup), you are all set. Otherwise, go through the [configuration notebook](../../../configuration.ipynb) first, if you haven't already, to establish your connection to the AzureML Workspace.\n",
+        "\n",
+        "In this notebook you will learn how to:\n",
+        "1. Creating an Experiment using an existing Workspace\n",
+        "1. Configure AutoML using 'AutoMLConfig'\n",
+        "1. Train the model using AmlCompute\n",
+        "1. Explore the engineered features and results\n",
+        "1. Configuration and remote run of AutoML for a time-series model with lag and rolling window features\n",
+        "1. Run and explore the forecast"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "## Setup\n"
+        "## Setup"
       ]
     },
     {
@@ -53,27 +67,29 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "import azureml.core\n",
+        "import logging\n",
+        "\n",
+        "from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\n",
+        "from matplotlib import pyplot as plt\n",
         "import pandas as pd\n",
         "import numpy as np\n",
-        "import logging\n",
         "import warnings\n",
+        "import os\n",
+        "\n",
         "# Squash warning messages for cleaner output in the notebook\n",
         "warnings.showwarning = lambda *args, **kwargs: None\n",
         "\n",
-        "\n",
-        "from azureml.core.workspace import Workspace\n",
-        "from azureml.core.experiment import Experiment\n",
+        "import azureml.core\n",
+        "from azureml.core import Experiment, Workspace, Dataset\n",
         "from azureml.train.automl import AutoMLConfig\n",
-        "from matplotlib import pyplot as plt\n",
-        "from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score"
+        "from datetime import datetime"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "As part of the setup you have already created a <b>Workspace</b>. For AutoML you would need to create an <b>Experiment</b>. An <b>Experiment</b> is a named object in a <b>Workspace</b>, which is used to run experiments."
+        "As part of the setup you have already created an Azure ML `Workspace` object. For Automated ML you will need to create an `Experiment` object, which is a named object in a `Workspace` used to run experiments."
       ]
     },
     {
@@ -85,9 +101,10 @@
         "ws = Workspace.from_config()\n",
         "\n",
         "# choose a name for the run history container in the workspace\n",
-        "experiment_name = 'automl-energydemandforecasting'\n",
-        "# project folder\n",
-        "project_folder = './sample_projects/automl-local-energydemandforecasting'\n",
+        "experiment_name = 'automl-forecasting-energydemand'\n",
+        "\n",
+        "# # project folder\n",
+        "# project_folder = './sample_projects/automl-forecasting-energy-demand'\n",
         "\n",
         "experiment = Experiment(ws, experiment_name)\n",
         "\n",
@@ -97,7 +114,6 @@
         "output['Workspace'] = ws.name\n",
         "output['Resource Group'] = ws.resource_group\n",
         "output['Location'] = ws.location\n",
-        "output['Project Directory'] = project_folder\n",
         "output['Run History Name'] = experiment_name\n",
         "pd.set_option('display.max_colwidth', -1)\n",
         "outputDf = pd.DataFrame(data = output, index = [''])\n",
@@ -108,62 +124,14 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "## Data\n",
-        "Read energy demanding data from file, and preview data."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "data = pd.read_csv(\"nyc_energy.csv\", parse_dates=['timeStamp'])\n",
-        "data.head()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Split the data to train and test\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "train = data[data['timeStamp'] < '2017-02-01']\n",
-        "test = data[data['timeStamp'] >= '2017-02-01']\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Prepare the test data, we will feed X_test to the fitted model and get prediction"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "y_test = test.pop('demand').values\n",
-        "X_test = test"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Split the train data to train and valid\n",
+        "## Create or Attach existing AmlCompute\n",
+        "A compute target is required to execute a remote Automated ML run. \n",
         "\n",
-        "Use one month's data as valid data\n"
+        "[Azure Machine Learning Compute](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-set-up-training-targets#amlcompute) is a managed-compute infrastructure that allows the user to easily create a single or multi-node compute. In this tutorial, you create AmlCompute as your training compute resource.\n",
+        "\n",
+        "#### Creation of AmlCompute takes approximately 5 minutes. \n",
+        "If the AmlCompute with that name is already in your workspace this code will skip the creation process.\n",
+        "As with other Azure services, there are limits on certain resources (e.g. AmlCompute) associated with the Azure Machine Learning service. Please read [this article](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-manage-quotas) on the default limits and how to request more quota."
       ]
     },
     {
@@ -172,14 +140,153 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "X_train = train[train['timeStamp'] < '2017-01-01']\n",
-        "X_valid = train[train['timeStamp'] >= '2017-01-01']\n",
-        "y_train = X_train.pop('demand').values\n",
-        "y_valid = X_valid.pop('demand').values\n",
-        "print(X_train.shape)\n",
-        "print(y_train.shape)\n",
-        "print(X_valid.shape)\n",
-        "print(y_valid.shape)"
+        "from azureml.core.compute import AmlCompute\n",
+        "from azureml.core.compute import ComputeTarget\n",
+        "\n",
+        "# Choose a name for your cluster.\n",
+        "amlcompute_cluster_name = \"aml-compute\"\n",
+        "\n",
+        "found = False\n",
+        "# Check if this compute target already exists in the workspace.\n",
+        "cts = ws.compute_targets\n",
+        "if amlcompute_cluster_name in cts and cts[amlcompute_cluster_name].type == 'AmlCompute':\n",
+        "    found = True\n",
+        "    print('Found existing compute target.')\n",
+        "    compute_target = cts[amlcompute_cluster_name]\n",
+        "\n",
+        "if not found:\n",
+        "    print('Creating a new compute target...')\n",
+        "    provisioning_config = AmlCompute.provisioning_configuration(vm_size = \"STANDARD_DS12_V2\", # for GPU, use \"STANDARD_NC6\"\n",
+        "                                                                #vm_priority = 'lowpriority', # optional\n",
+        "                                                                max_nodes = 6)\n",
+        "\n",
+        "    # Create the cluster.\\n\",\n",
+        "    compute_target = ComputeTarget.create(ws, amlcompute_cluster_name, provisioning_config)\n",
+        "\n",
+        "print('Checking cluster status...')\n",
+        "# Can poll for a minimum number of nodes and for a specific timeout.\n",
+        "# If no min_node_count is provided, it will use the scale settings for the cluster.\n",
+        "compute_target.wait_for_completion(show_output = True, min_node_count = None, timeout_in_minutes = 20)\n",
+        "\n",
+        "# For a more detailed view of current AmlCompute status, use get_status()."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# Data\n",
+        "\n",
+        "We will use energy consumption [data from New York City](http://mis.nyiso.com/public/P-58Blist.htm) for model training. The data is stored in a tabular format and includes energy demand and basic weather data at an hourly frequency. \n",
+        "\n",
+        "With Azure Machine Learning datasets you can keep a single copy of data in your storage, easily access data during model training, share data and collaborate with other users. Below, we will upload the datatset and create a [tabular dataset](https://docs.microsoft.com/bs-latn-ba/azure/machine-learning/service/how-to-create-register-datasets#dataset-types) to be used training and prediction."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Let's set up what we know about the dataset.\n",
+        "\n",
+        "<b>Target column</b> is what we want to forecast.<br></br>\n",
+        "<b>Time column</b> is the time axis along which to predict.\n",
+        "\n",
+        "The other columns, \"temp\" and \"precip\", are implicitly designated as features."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "target_column_name = 'demand'\n",
+        "time_column_name = 'timeStamp'"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "dataset = Dataset.Tabular.from_delimited_files(path = \"https://automlsamplenotebookdata.blob.core.windows.net/automl-sample-notebook-data/nyc_energy.csv\").with_timestamp_columns(fine_grain_timestamp=time_column_name) \n",
+        "dataset.take(5).to_pandas_dataframe().reset_index(drop=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "The NYC Energy dataset is missing energy demand values for all datetimes later than August 10th, 2017 5AM. Below, we trim the rows containing these missing values from the end of the dataset."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Cut off the end of the dataset due to large number of nan values\n",
+        "dataset = dataset.time_before(datetime(2017, 10, 10, 5))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Split the data into train and test sets"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "The first split we make is into train and test sets. Note that we are splitting on time. Data before and including August 8th, 2017 5AM will be used for training, and data after will be used for testing."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# split into train based on time\n",
+        "train = dataset.time_before(datetime(2017, 8, 8, 5), include_boundary=True)\n",
+        "train.to_pandas_dataframe().sort_values(time_column_name).tail(5).reset_index(drop=True)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# split into test based on time\n",
+        "test = dataset.time_between(datetime(2017, 8, 8, 6), datetime(2017, 8, 10, 5))\n",
+        "test.to_pandas_dataframe().head(5).reset_index(drop=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Setting the maximum forecast horizon\n",
+        "\n",
+        "The forecast horizon is the number of periods into the future that the model should predict. It is generally recommend that users set forecast horizons to less than 100 time periods (i.e. less than 100 hours in the NYC energy example). Furthermore, **AutoML's memory use and computation time increase in proportion to the length of the horizon**, so consider carefully how this value is set. If a long horizon forecast really is necessary, consider aggregating the series to a coarser time scale. \n",
+        "\n",
+        "Learn more about forecast horizons in our [Auto-train a time-series forecast model](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-auto-train-forecast#configure-and-run-experiment) guide.\n",
+        "\n",
+        "In this example, we set the horizon to 48 hours."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "max_horizon = 48"
       ]
     },
     {
@@ -188,19 +295,28 @@
       "source": [
         "## Train\n",
         "\n",
-        "Instantiate a AutoMLConfig object. This defines the settings and data used to run the experiment.\n",
+        "Instantiate an AutoMLConfig object. This config defines the settings and data used to run the experiment. We can provide extra configurations within 'automl_settings', for this forecasting task we add the name of the time column and the maximum forecast horizon.\n",
         "\n",
         "|Property|Description|\n",
         "|-|-|\n",
         "|**task**|forecasting|\n",
-        "|**primary_metric**|This is the metric that you want to optimize.<br> Forecasting supports the following primary metrics <br><i>spearman_correlation</i><br><i>normalized_root_mean_squared_error</i><br><i>r2_score</i><br><i>normalized_mean_absolute_error</i>\n",
-        "|**iterations**|Number of iterations. In each iteration, Auto ML trains a specific pipeline on the given data|\n",
-        "|**iteration_timeout_minutes**|Time limit in minutes for each iteration.|\n",
-        "|**X**|(sparse) array-like, shape = [n_samples, n_features]|\n",
-        "|**y**|(sparse) array-like, shape = [n_samples, ], targets values.|\n",
-        "|**X_valid**|Data used to evaluate a model in a iteration. (sparse) array-like, shape = [n_samples, n_features]|\n",
-        "|**y_valid**|Data used to evaluate a model in a iteration. (sparse) array-like, shape = [n_samples, ], targets values.|\n",
-        "|**path**|Relative path to the project folder.  AutoML stores configuration files for the experiment under this folder. You can specify a new empty folder. "
+        "|**primary_metric**|This is the metric that you want to optimize.<br> Forecasting supports the following primary metrics <br><i>spearman_correlation</i><br><i>normalized_root_mean_squared_error</i><br><i>r2_score</i><br><i>normalized_mean_absolute_error</i>|\n",
+        "|**blacklist_models**|Models in blacklist won't be used by AutoML. All supported models can be found at [here](https://docs.microsoft.com/en-us/python/api/azureml-train-automl-client/azureml.train.automl.constants.supportedmodels.forecasting?view=azure-ml-py).|\n",
+        "|**experiment_timeout_minutes**|Maximum amount of time in minutes that the experiment take before it terminates.|\n",
+        "|**training_data**|The training data to be used within the experiment.|\n",
+        "|**label_column_name**|The name of the label column.|\n",
+        "|**compute_target**|The remote compute for training.|\n",
+        "|**n_cross_validations**|Number of cross validation splits. Rolling Origin Validation is used to split time-series in a temporally consistent way.|\n",
+        "|**enable_early_stopping**|Flag to enble early termination if the score is not improving in the short term.|\n",
+        "|**time_column_name**|The name of your time column.|\n",
+        "|**max_horizon**|The number of periods out you would like to predict past your training data. Periods are inferred from your data.|\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "This notebook uses the blacklist_models parameter to exclude some models that take a longer time to train on this dataset. You can choose to remove models from the blacklist_models list but you may need to increase the experiment_timeout_minutes parameter value to get results."
       ]
     },
     {
@@ -209,23 +325,21 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "time_column_name = 'timeStamp'\n",
         "automl_settings = {\n",
-        "    \"time_column_name\": time_column_name,\n",
+        "    'time_column_name': time_column_name,\n",
+        "    'max_horizon': max_horizon,\n",
         "}\n",
         "\n",
-        "\n",
-        "automl_config = AutoMLConfig(task = 'forecasting',\n",
-        "                             debug_log = 'automl_nyc_energy_errors.log',\n",
+        "automl_config = AutoMLConfig(task='forecasting',                             \n",
         "                             primary_metric='normalized_root_mean_squared_error',\n",
-        "                             iterations = 10,\n",
-        "                             iteration_timeout_minutes = 5,\n",
-        "                             X = X_train,\n",
-        "                             y = y_train,\n",
-        "                             X_valid = X_valid,\n",
-        "                             y_valid = y_valid,\n",
-        "                             path=project_folder,\n",
-        "                             verbosity = logging.INFO,\n",
+        "                             blacklist_models = ['ExtremeRandomTrees', 'AutoArima', 'Prophet'],                             \n",
+        "                             experiment_timeout_minutes=20,\n",
+        "                             training_data=train,\n",
+        "                             label_column_name=target_column_name,\n",
+        "                             compute_target=compute_target,\n",
+        "                             enable_early_stopping=True,\n",
+        "                             n_cross_validations=3,                             \n",
+        "                             verbosity=logging.INFO,\n",
         "                            **automl_settings)"
       ]
     },
@@ -233,8 +347,8 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "You can call the submit method on the experiment object and pass the run configuration. For Local runs the execution is synchronous. Depending on the data and number of iterations this can run for while.\n",
-        "You will see the currently running iterations printing to the console."
+        "Call the `submit` method on the experiment object and pass the run configuration. Depending on the data and the number of iterations this can run for a while.\n",
+        "One may specify `show_output = True` to print currently running iterations to the console."
       ]
     },
     {
@@ -243,7 +357,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "local_run = experiment.submit(automl_config, show_output=True)"
+        "remote_run = experiment.submit(automl_config, show_output=False)"
       ]
     },
     {
@@ -252,15 +366,24 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "local_run"
+        "remote_run"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "remote_run.wait_for_completion()"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "### Retrieve the Best Model\n",
-        "Below we select the best pipeline from our iterations. The get_output method on automl_classifier returns the best run and the fitted model for the last fit invocation. There are overloads on get_output that allow you to retrieve the best run and fitted model for any logged metric or a particular iteration."
+        "## Retrieve the Best Model\n",
+        "Below we select the best model from all the training iterations using get_output method."
       ]
     },
     {
@@ -269,7 +392,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "best_run, fitted_model = local_run.get_output()\n",
+        "best_run, fitted_model = remote_run.get_output()\n",
         "fitted_model.steps"
       ]
     },
@@ -277,9 +400,8 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "### Test the Best Fitted Model\n",
-        "\n",
-        "Predict on training and test set, and calculate residual values."
+        "## Featurization\n",
+        "You can access the engineered feature names generated in time-series featurization."
       ]
     },
     {
@@ -288,15 +410,21 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "y_pred = fitted_model.predict(X_test)\n",
-        "y_pred"
+        "fitted_model.named_steps['timeseriestransformer'].get_engineered_feature_names()"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "### Use the Check Data Function to remove the nan values from y_test to avoid error when calculate metrics "
+        "### View featurization summary\n",
+        "You can also see what featurization steps were performed on different raw features in the user data. For each raw feature in the user data, the following information is displayed:\n",
+        "\n",
+        "+ Raw feature name\n",
+        "+ Number of engineered features formed out of this raw feature\n",
+        "+ Type detected\n",
+        "+ If feature was dropped\n",
+        "+ List of feature transformations for the raw feature"
       ]
     },
     {
@@ -305,29 +433,19 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "if len(y_test) != len(y_pred):\n",
-        "    raise ValueError(\n",
-        "        'the true values and prediction values do not have equal length.')\n",
-        "elif len(y_test) == 0:\n",
-        "    raise ValueError(\n",
-        "        'y_true and y_pred are empty.')\n",
-        "\n",
-        "# if there is any non-numeric element in the y_true or y_pred,\n",
-        "# the ValueError exception will be thrown.\n",
-        "y_test_f = np.array(y_test).astype(float)\n",
-        "y_pred_f = np.array(y_pred).astype(float)\n",
-        "\n",
-        "# remove entries both in y_true and y_pred where at least\n",
-        "# one element in y_true or y_pred is missing\n",
-        "y_test = y_test_f[~(np.isnan(y_test_f) | np.isnan(y_pred_f))]\n",
-        "y_pred = y_pred_f[~(np.isnan(y_test_f) | np.isnan(y_pred_f))]"
+        "# Get the featurization summary as a list of JSON\n",
+        "featurization_summary = fitted_model.named_steps['timeseriestransformer'].get_featurization_summary()\n",
+        "# View the featurization summary as a pandas dataframe\n",
+        "pd.DataFrame.from_records(featurization_summary)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "### Calculate metrics for the prediction\n"
+        "## Forecasting\n",
+        "\n",
+        "Now that we have retrieved the best pipeline/model, it can be used to make predictions on test data. First, we remove the target values from the test set:"
       ]
     },
     {
@@ -336,17 +454,237 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "print(\"[Test Data] \\nRoot Mean squared error: %.2f\" % np.sqrt(mean_squared_error(y_test, y_pred)))\n",
-        "# Explained variance score: 1 is perfect prediction\n",
-        "print('mean_absolute_error score: %.2f' % mean_absolute_error(y_test, y_pred))\n",
-        "print('R2 score: %.2f' % r2_score(y_test, y_pred))\n",
+        "X_test = test.to_pandas_dataframe().reset_index(drop=True)\n",
+        "y_test = X_test.pop(target_column_name).values"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Forecast Function\n",
+        "For forecasting, we will use the forecast function instead of the predict function. Using the predict method would result in getting predictions for EVERY horizon the forecaster can predict at. This is useful when training and evaluating the performance of the forecaster at various horizons, but the level of detail is excessive for normal use. Forecast function also can handle more complicated scenarios, see notebook on [high frequency forecasting](https://github.com/Azure/MachineLearningNotebooks/blob/master/how-to-use-azureml/automated-machine-learning/forecasting-high-frequency/automl-forecasting-function.ipynb)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# The featurized data, aligned to y, will also be returned.\n",
+        "# This contains the assumptions that were made in the forecast\n",
+        "# and helps align the forecast to the original data\n",
+        "y_predictions, X_trans = fitted_model.forecast(X_test)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Evaluate\n",
+        "To evaluate the accuracy of the forecast, we'll compare against the actual sales quantities for some select metrics, included the mean absolute percentage error (MAPE).\n",
         "\n",
+        "It is a good practice to always align the output explicitly to the input, as the count and order of the rows may have changed during transformations that span multiple rows."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from forecasting_helper import align_outputs\n",
         "\n",
+        "df_all = align_outputs(y_predictions, X_trans, X_test, y_test, target_column_name)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.automl.core._vendor.automl.client.core.common import metrics\n",
+        "from matplotlib import pyplot as plt\n",
+        "from automl.client.core.common import constants\n",
         "\n",
+        "# use automl metrics module\n",
+        "scores = metrics.compute_metrics_regression(\n",
+        "    df_all['predicted'],\n",
+        "    df_all[target_column_name],\n",
+        "    list(constants.Metric.SCALAR_REGRESSION_SET),\n",
+        "    None, None, None)\n",
+        "\n",
+        "print(\"[Test data scores]\\n\")\n",
+        "for key, value in scores.items():    \n",
+        "    print('{}:   {:.3f}'.format(key, value))\n",
+        "    \n",
         "# Plot outputs\n",
-        "%matplotlib notebook\n",
-        "test_pred = plt.scatter(y_test, y_pred, color='b')\n",
-        "test_test = plt.scatter(y_test, y_test, color='g')\n",
+        "%matplotlib inline\n",
+        "test_pred = plt.scatter(df_all[target_column_name], df_all['predicted'], color='b')\n",
+        "test_test = plt.scatter(df_all[target_column_name], df_all[target_column_name], color='g')\n",
+        "plt.legend((test_pred, test_test), ('prediction', 'truth'), loc='upper left', fontsize=8)\n",
+        "plt.show()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Looking at `X_trans` is also useful to see what featurization happened to the data."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "X_trans"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Advanced Training <a id=\"advanced_training\"></a>\n",
+        "We did not use lags in the previous model specification. In effect, the prediction was the result of a simple regression on date, grain and any additional features. This is often a very good prediction as common time series patterns like seasonality and trends can be captured in this manner. Such simple regression is horizon-less: it doesn't matter how far into the future we are predicting, because we are not using past data. In the previous example, the horizon was only used to split the data for cross-validation."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Using lags and rolling window features\n",
+        "Now we will configure the target lags, that is the previous values of the target variables, meaning the prediction is no longer horizon-less. We therefore must still specify the `max_horizon` that the model will learn to forecast. The `target_lags` keyword specifies how far back we will construct the lags of the target variable, and the `target_rolling_window_size` specifies the size of the rolling window over which we will generate the `max`, `min` and `sum` features.\n",
+        "\n",
+        "This notebook uses the blacklist_models parameter to exclude some models that take a longer time to train on this dataset.  You can choose to remove models from the blacklist_models list but you may need to increase the iteration_timeout_minutes parameter value to get results."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "automl_advanced_settings = {\n",
+        "    'time_column_name': time_column_name,\n",
+        "    'max_horizon': max_horizon,\n",
+        "    'target_lags': 12,\n",
+        "    'target_rolling_window_size': 4,\n",
+        "}\n",
+        "\n",
+        "automl_config = AutoMLConfig(task='forecasting',                             \n",
+        "                             primary_metric='normalized_root_mean_squared_error',\n",
+        "                             blacklist_models = ['ElasticNet','ExtremeRandomTrees','GradientBoosting','XGBoostRegressor','ExtremeRandomTrees', 'AutoArima', 'Prophet'], #These models are blacklisted for tutorial purposes, remove this for real use cases.                            \n",
+        "                             experiment_timeout_minutes=20,\n",
+        "                             training_data=train,\n",
+        "                             label_column_name=target_column_name,\n",
+        "                             compute_target=compute_target,\n",
+        "                             enable_early_stopping = True,\n",
+        "                             n_cross_validations=3,                             \n",
+        "                             verbosity=logging.INFO,\n",
+        "                            **automl_advanced_settings)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "We now start a new remote run, this time with lag and rolling window featurization. AutoML applies featurizations in the setup stage, prior to iterating over ML models. The full training set is featurized first, followed by featurization of each of the CV splits. Lag and rolling window features introduce additional complexity, so the run will take longer than in the previous example that lacked these featurizations."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "advanced_remote_run = experiment.submit(automl_config, show_output=False)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "advanced_remote_run.wait_for_completion()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Retrieve the Best Model"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "best_run_lags, fitted_model_lags = advanced_remote_run.get_output()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Advanced Results<a id=\"advanced_results\"></a>\n",
+        "We did not use lags in the previous model specification. In effect, the prediction was the result of a simple regression on date, grain and any additional features. This is often a very good prediction as common time series patterns like seasonality and trends can be captured in this manner. Such simple regression is horizon-less: it doesn't matter how far into the future we are predicting, because we are not using past data. In the previous example, the horizon was only used to split the data for cross-validation."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# The featurized data, aligned to y, will also be returned.\n",
+        "# This contains the assumptions that were made in the forecast\n",
+        "# and helps align the forecast to the original data\n",
+        "y_predictions, X_trans = fitted_model_lags.forecast(X_test)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from forecasting_helper import align_outputs\n",
+        "\n",
+        "df_all = align_outputs(y_predictions, X_trans, X_test, y_test, target_column_name)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.automl.core._vendor.automl.client.core.common import metrics\n",
+        "from matplotlib import pyplot as plt\n",
+        "from automl.client.core.common import constants\n",
+        "\n",
+        "# use automl metrics module\n",
+        "scores = metrics.compute_metrics_regression(\n",
+        "    df_all['predicted'],\n",
+        "    df_all[target_column_name],\n",
+        "    list(constants.Metric.SCALAR_REGRESSION_SET),\n",
+        "    None, None, None)\n",
+        "\n",
+        "print(\"[Test data scores]\\n\")\n",
+        "for key, value in scores.items():    \n",
+        "    print('{}:   {:.3f}'.format(key, value))\n",
+        "    \n",
+        "# Plot outputs\n",
+        "%matplotlib inline\n",
+        "test_pred = plt.scatter(df_all[target_column_name], df_all['predicted'], color='b')\n",
+        "test_test = plt.scatter(df_all[target_column_name], df_all[target_column_name], color='g')\n",
         "plt.legend((test_pred, test_test), ('prediction', 'truth'), loc='upper left', fontsize=8)\n",
         "plt.show()"
       ]
@@ -355,9 +693,13 @@
   "metadata": {
     "authors": [
       {
-        "name": "xiaga"
+        "name": "erwright"
       }
     ],
+    "categories": [
+      "how-to-use-azureml",
+      "automated-machine-learning"
+    ],
     "kernelspec": {
       "display_name": "Python 3.6",
       "language": "python",

how-to-use-azureml/automated-machine-learning/forecasting-energy-demand/auto-ml-forecasting-energy-demand.yml

@@ -0,0 +1,12 @@
+name: auto-ml-forecasting-energy-demand
+dependencies:
+- pip:
+  - azureml-sdk
+  - interpret
+  - azureml-train-automl
+  - azureml-widgets
+  - matplotlib
+  - pandas_ml
+  - statsmodels
+  - azureml-explain-model
+  - azureml-contrib-interpret

how-to-use-azureml/automated-machine-learning/forecasting-energy-demand/forecasting_helper.py

@@ -0,0 +1,44 @@
+import pandas as pd
+import numpy as np
+from pandas.tseries.frequencies import to_offset
+
+
+def align_outputs(y_predicted, X_trans, X_test, y_test, target_column_name,
+                  predicted_column_name='predicted',
+                  horizon_colname='horizon_origin'):
+    """
+    Demonstrates how to get the output aligned to the inputs
+    using pandas indexes. Helps understand what happened if
+    the output's shape differs from the input shape, or if
+    the data got re-sorted by time and grain during forecasting.
+
+    Typical causes of misalignment are:
+    * we predicted some periods that were missing in actuals -> drop from eval
+    * model was asked to predict past max_horizon -> increase max horizon
+    * data at start of X_test was needed for lags -> provide previous periods
+    """
+
+    if (horizon_colname in X_trans):
+        df_fcst = pd.DataFrame({predicted_column_name: y_predicted,
+                                horizon_colname: X_trans[horizon_colname]})
+    else:
+        df_fcst = pd.DataFrame({predicted_column_name: y_predicted})
+
+    # y and X outputs are aligned by forecast() function contract
+    df_fcst.index = X_trans.index
+
+    # align original X_test to y_test
+    X_test_full = X_test.copy()
+    X_test_full[target_column_name] = y_test
+
+    # X_test_full's index does not include origin, so reset for merge
+    df_fcst.reset_index(inplace=True)
+    X_test_full = X_test_full.reset_index().drop(columns='index')
+    together = df_fcst.merge(X_test_full, how='right')
+
+    # drop rows where prediction or actuals are nan
+    # happens because of missing actuals
+    # or at edges of time due to lags/rolling windows
+    clean = together[together[[target_column_name,
+                               predicted_column_name]].notnull().all(axis=1)]
+    return(clean)

how-to-use-azureml/automated-machine-learning/forecasting-energy-demand/metrics_helper.py

@@ -0,0 +1,22 @@
+import pandas as pd
+import numpy as np
+
+
+def APE(actual, pred):
+    """
+    Calculate absolute percentage error.
+    Returns a vector of APE values with same length as actual/pred.
+    """
+    return 100 * np.abs((actual - pred) / actual)
+
+
+def MAPE(actual, pred):
+    """
+    Calculate mean absolute percentage error.
+    Remove NA and values where actual is close to zero
+    """
+    not_na = ~(np.isnan(actual) | np.isnan(pred))
+    not_zero = ~np.isclose(actual, 0.0)
+    actual_safe = actual[not_na & not_zero]
+    pred_safe = pred[not_na & not_zero]
+    return np.mean(APE(actual_safe, pred_safe))

how-to-use-azureml/automated-machine-learning/forecasting-grouping/auto-ml-forecasting-grouping.ipynb

@@ -0,0 +1,551 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
+        "\n",
+        "Licensed under the MIT License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/automated-machine-learning/forecasting-grouping/auto-ml-forecasting-grouping.png)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# Automated Machine Learning\n",
+        "\n",
+        "_**Forecasting with grouping using Pipelines**_\n",
+        "\n",
+        "## Contents\n",
+        "\n",
+        "1. [Introduction](#Introduction)\n",
+        "2. [Setup](#Setup)\n",
+        "3. [Data](#Data)\n",
+        "4. [Compute](#Compute)\n",
+        "4. [AutoMLConfig](#AutoMLConfig)\n",
+        "5. [Pipeline](#Pipeline)\n",
+        "5. [Train](#Train)\n",
+        "6. [Test](#Test)\n",
+        "\n",
+        "\n",
+        "## Introduction\n",
+        "In this example we use Automated ML and Pipelines to train, select, and operationalize forecasting models for multiple time-series.\n",
+        "\n",
+        "If you are using an Azure Machine Learning Notebook VM, you are all set. Otherwise, go through the [configuration notebook](../../../configuration.ipynb) first if you haven't already to establish your connection to the AzureML Workspace.\n",
+        "\n",
+        "In this notebook you will learn how to:\n",
+        "\n",
+        "* Create an Experiment in an existing Workspace.\n",
+        "* Configure AutoML using AutoMLConfig.\n",
+        "* Use our helper script to generate pipeline steps to split, train, and deploy the models.\n",
+        "* Explore the results.\n",
+        "* Test the models.\n",
+        "\n",
+        "It is advised you ensure your cluster has at least one node per group.\n",
+        "\n",
+        "An Enterprise workspace is required for this notebook. To learn more about creating an Enterprise workspace or upgrading to an Enterprise workspace from the Azure portal, please visit our [Workspace page.](https://docs.microsoft.com/azure/machine-learning/service/concept-workspace#upgrade)\n",
+        "\n",
+        "## Setup\n",
+        "As part of the setup you have already created an Azure ML `Workspace` object. For Automated ML you will need to create an `Experiment` object, which is a named object in a `Workspace` used to run experiments. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import json\n",
+        "import logging\n",
+        "import warnings\n",
+        "\n",
+        "import numpy as np\n",
+        "import pandas as pd\n",
+        "\n",
+        "import azureml.core\n",
+        "\n",
+        "from azureml.core.workspace import Workspace\n",
+        "from azureml.core.experiment import Experiment\n",
+        "from azureml.train.automl import AutoMLConfig"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Accessing the Azure ML workspace requires authentication with Azure.\n",
+        "\n",
+        "The default authentication is interactive authentication using the default tenant. Executing the ws = Workspace.from_config() line in the cell below will prompt for authentication the first time that it is run.\n",
+        "\n",
+        "If you have multiple Azure tenants, you can specify the tenant by replacing the ws = Workspace.from_config() line in the cell below with the following:\n",
+        "```\n",
+        "from azureml.core.authentication import InteractiveLoginAuthentication\n",
+        "auth = InteractiveLoginAuthentication(tenant_id = 'mytenantid')\n",
+        "ws = Workspace.from_config(auth = auth)\n",
+        "```\n",
+        "If you need to run in an environment where interactive login is not possible, you can use Service Principal authentication by replacing the ws = Workspace.from_config() line in the cell below with the following:\n",
+        "```\n",
+        "from azureml.core.authentication import ServicePrincipalAuthentication\n",
+        "auth = auth = ServicePrincipalAuthentication('mytenantid', 'myappid', 'mypassword')\n",
+        "ws = Workspace.from_config(auth = auth)\n",
+        "```\n",
+        "For more details, see aka.ms/aml-notebook-auth"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "ws = Workspace.from_config()\n",
+        "ds = ws.get_default_datastore()\n",
+        "\n",
+        "# choose a name for the run history container in the workspace\n",
+        "experiment_name = 'automl-grouping-oj'\n",
+        "# project folder\n",
+        "project_folder = './sample_projects/{}'.format(experiment_name)\n",
+        "\n",
+        "experiment = Experiment(ws, experiment_name)\n",
+        "\n",
+        "output = {}\n",
+        "output['SDK version'] = azureml.core.VERSION\n",
+        "output['Subscription ID'] = ws.subscription_id\n",
+        "output['Workspace'] = ws.name\n",
+        "output['Resource Group'] = ws.resource_group\n",
+        "output['Location'] = ws.location\n",
+        "output['Project Directory'] = project_folder\n",
+        "output['Run History Name'] = experiment_name\n",
+        "pd.set_option('display.max_colwidth', -1)\n",
+        "outputDf = pd.DataFrame(data = output, index = [''])\n",
+        "outputDf.T"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Data\n",
+        "Upload data to your default datastore and then load it as a `TabularDataset`"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.dataset import Dataset"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# upload training and test data to your default datastore\n",
+        "ds = ws.get_default_datastore()\n",
+        "ds.upload(src_dir='./data', target_path='groupdata', overwrite=True, show_progress=True)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# load data from your datastore\n",
+        "data = Dataset.Tabular.from_delimited_files(path=ds.path('groupdata/dominicks_OJ_2_5_8_train.csv'))\n",
+        "data_test = Dataset.Tabular.from_delimited_files(path=ds.path('groupdata/dominicks_OJ_2_5_8_test.csv'))\n",
+        "\n",
+        "data.take(5).to_pandas_dataframe()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Compute \n",
+        "\n",
+        "#### Create or Attach existing AmlCompute\n",
+        "\n",
+        "You will need to create a compute target for your automated ML run. In this tutorial, you create AmlCompute as your training compute resource.\n",
+        "#### Creation of AmlCompute takes approximately 5 minutes. \n",
+        "If the AmlCompute with that name is already in your workspace this code will skip the creation process.\n",
+        "As with other Azure services, there are limits on certain resources (e.g. AmlCompute) associated with the Azure Machine Learning service. Please read this article on the default limits and how to request more quota."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.compute import AmlCompute\n",
+        "from azureml.core.compute import ComputeTarget\n",
+        "\n",
+        "# Choose a name for your cluster.\n",
+        "amlcompute_cluster_name = \"cpu-cluster-11\"\n",
+        "\n",
+        "found = False\n",
+        "# Check if this compute target already exists in the workspace.\n",
+        "cts = ws.compute_targets\n",
+        "if amlcompute_cluster_name in cts and cts[amlcompute_cluster_name].type == 'AmlCompute':\n",
+        "    found = True\n",
+        "    print('Found existing compute target.')\n",
+        "    compute_target = cts[amlcompute_cluster_name]\n",
+        "    \n",
+        "if not found:\n",
+        "    print('Creating a new compute target...')\n",
+        "    provisioning_config = AmlCompute.provisioning_configuration(vm_size = \"STANDARD_D2_V2\", # for GPU, use \"STANDARD_NC6\"\n",
+        "                                                                #vm_priority = 'lowpriority', # optional\n",
+        "                                                                max_nodes = 6)\n",
+        "\n",
+        "    # Create the cluster.\n",
+        "    compute_target = ComputeTarget.create(ws, amlcompute_cluster_name, provisioning_config)\n",
+        "    \n",
+        "print('Checking cluster status...')\n",
+        "# Can poll for a minimum number of nodes and for a specific timeout.\n",
+        "# If no min_node_count is provided, it will use the scale settings for the cluster.\n",
+        "compute_target.wait_for_completion(show_output = True, min_node_count = None, timeout_in_minutes = 20)\n",
+        "    \n",
+        "# For a more detailed view of current AmlCompute status, use get_status()."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## AutoMLConfig\n",
+        "#### Create a base AutoMLConfig\n",
+        "This configuration will be used for all the groups in the pipeline."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "target_column = 'Quantity'\n",
+        "time_column_name = 'WeekStarting'\n",
+        "grain_column_names = ['Brand']\n",
+        "group_column_names = ['Store']\n",
+        "max_horizon = 20"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "automl_settings = {\n",
+        "    \"iteration_timeout_minutes\" : 5,\n",
+        "    \"experiment_timeout_minutes\" : 15,\n",
+        "    \"primary_metric\" : 'normalized_mean_absolute_error',\n",
+        "    \"time_column_name\": time_column_name,\n",
+        "    \"grain_column_names\": grain_column_names,\n",
+        "    \"max_horizon\": max_horizon,\n",
+        "    \"drop_column_names\": ['logQuantity'],\n",
+        "    \"max_concurrent_iterations\": 2,\n",
+        "    \"max_cores_per_iteration\": -1\n",
+        "}\n",
+        "base_configuration = AutoMLConfig(task = 'forecasting',\n",
+        "                             path = project_folder,\n",
+        "                             n_cross_validations=3,\n",
+        "                             **automl_settings\n",
+        "                            )"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Pipeline\n",
+        "We've written a script to generate the individual pipeline steps used to create each automl step. Calling this script will return a list of PipelineSteps that will train multiple groups concurrently and then deploy these models.\n",
+        "\n",
+        "This step requires an Enterprise workspace to gain access to this feature. To learn more about creating an Enterprise workspace or upgrading to an Enterprise workspace from the Azure portal, please visit our [Workspace page.](https://docs.microsoft.com/azure/machine-learning/service/concept-workspace#upgrade).\n",
+        "\n",
+        "### Call the method to build pipeline steps\n",
+        "\n",
+        "`build_pipeline_steps()` takes as input:\n",
+        "* **automlconfig**: This is the configuration used for every automl step\n",
+        "* **df**: This is the dataset to be used for training\n",
+        "* **target_column**: This is the target column of the dataset\n",
+        "* **compute_target**: The compute to be used for training\n",
+        "* **deploy**: The option on to deploy the models after training, if set to true an extra step will be added to deploy a webservice with all the models (default is `True`)\n",
+        "* **service_name**: The service name for the model query endpoint\n",
+        "* **time_column_name**: The time column of the data"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.webservice import Webservice\n",
+        "from azureml.exceptions import WebserviceException\n",
+        "\n",
+        "service_name = 'grouped-model'\n",
+        "try:\n",
+        "    # if you want to get existing service below is the command\n",
+        "    # since aci name needs to be unique in subscription deleting existing aci if any\n",
+        "    # we use aci_service_name to create azure aci\n",
+        "    service = Webservice(ws, name=service_name)\n",
+        "    if service:\n",
+        "        service.delete()\n",
+        "except WebserviceException as e:\n",
+        "    pass"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from build import build_pipeline_steps\n",
+        "\n",
+        "steps = build_pipeline_steps(\n",
+        "    base_configuration, \n",
+        "    data, \n",
+        "    target_column,\n",
+        "    compute_target, \n",
+        "    group_column_names=group_column_names, \n",
+        "    deploy=True, \n",
+        "    service_name=service_name, \n",
+        "    time_column_name=time_column_name\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Train\n",
+        "Use the list of steps generated from above to build the pipeline and submit it to your compute for remote training."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.pipeline.core import Pipeline\n",
+        "pipeline = Pipeline(\n",
+        "    description=\"A pipeline with one model per data group using Automated ML.\",\n",
+        "    workspace=ws,    \n",
+        "    steps=steps)\n",
+        "\n",
+        "pipeline_run = experiment.submit(pipeline)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.widgets import RunDetails\n",
+        "RunDetails(pipeline_run).show()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "pipeline_run.wait_for_completion(show_output=False)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Test\n",
+        "\n",
+        "Now we can use the holdout set to test our models and ensure our web-service is running as expected."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.webservice import AciWebservice\n",
+        "service = AciWebservice(ws, service_name)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "X_test = data_test.to_pandas_dataframe()\n",
+        "# Drop the column we are trying to predict (target column)\n",
+        "x_pred = X_test.drop(target_column, inplace=False, axis=1)\n",
+        "x_pred.head()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Get Predictions\n",
+        "test_sample = X_test.drop(target_column, inplace=False, axis=1).to_json()\n",
+        "predictions = service.run(input_data=test_sample)\n",
+        "print(predictions)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Convert predictions from JSON to DataFrame\n",
+        "pred_dict =json.loads(predictions)\n",
+        "X_pred = pd.read_json(pred_dict['predictions'])\n",
+        "X_pred.head()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Fix the index\n",
+        "PRED = 'pred_target'\n",
+        "X_pred[time_column_name] = pd.to_datetime(X_pred[time_column_name], unit='ms')\n",
+        "\n",
+        "X_pred.set_index([time_column_name] + grain_column_names, inplace=True, drop=True)\n",
+        "X_pred.rename({'_automl_target_col': PRED}, inplace=True, axis=1)\n",
+        "# Drop all but the target column and index\n",
+        "X_pred.drop(list(set(X_pred.columns.values).difference({PRED})), axis=1, inplace=True)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "X_test[time_column_name] = pd.to_datetime(X_test[time_column_name])\n",
+        "X_test.set_index([time_column_name] + grain_column_names, inplace=True, drop=True)\n",
+        "# Merge predictions with raw features\n",
+        "pred_test = X_test.merge(X_pred, left_index=True, right_index=True)\n",
+        "pred_test.head()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from sklearn.metrics import mean_absolute_error, mean_squared_error\n",
+        "def MAPE(actual, pred):\n",
+        "    \"\"\"\n",
+        "    Calculate mean absolute percentage error.\n",
+        "    Remove NA and values where actual is close to zero\n",
+        "    \"\"\"\n",
+        "    not_na = ~(np.isnan(actual) | np.isnan(pred))\n",
+        "    not_zero = ~np.isclose(actual, 0.0)\n",
+        "    actual_safe = actual[not_na & not_zero]\n",
+        "    pred_safe = pred[not_na & not_zero]\n",
+        "    APE = 100*np.abs((actual_safe - pred_safe)/actual_safe)\n",
+        "    return np.mean(APE)\n",
+        "\n",
+        "def get_metrics(actuals, preds):\n",
+        "    return pd.Series(\n",
+        "    {\n",
+        "        \"RMSE\": np.sqrt(mean_squared_error(actuals, preds)),\n",
+        "        \"NormRMSE\": np.sqrt(mean_squared_error(actuals, preds))/np.abs(actuals.max()-actuals.min()),\n",
+        "        \"MAE\": mean_absolute_error(actuals, preds),\n",
+        "        \"MAPE\": MAPE(actuals, preds)},\n",
+        "    )"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "get_metrics(pred_test[PRED].values, pred_test[target_column].values)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": []
+    }
+  ],
+  "metadata": {
+    "authors": [
+      {
+        "name": "alyerman"
+      }
+    ],
+    "category": "other",
+    "compute": [
+      "AML Compute"
+    ],
+    "datasets": [
+      "Orange Juice Sales"
+    ],
+    "deployment": [
+      "Azure Container Instance"
+    ],
+    "exclude_from_index": false,
+    "framework": [
+      "Scikit-learn",
+      "Pytorch"
+    ],
+    "friendly_name": "Automated ML Grouping with Pipeline.",
+    "index_order": 10,
+    "kernelspec": {
+      "display_name": "Python 3.6",
+      "language": "python",
+      "name": "python36"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.6.6"
+    },
+    "tags": [
+      "AutomatedML"
+    ],
+    "task": "Use AzureML Pipeline to trigger multiple Automated ML runs."
+  },
+  "nbformat": 4,
+  "nbformat_minor": 2
+}

how-to-use-azureml/automated-machine-learning/forecasting-grouping/auto-ml-forecasting-grouping.yml

@@ -0,0 +1,10 @@
+name: auto-ml-forecasting-grouping
+dependencies:
+- pip:
+  - azureml-sdk
+  - azureml-train-automl
+  - azureml-pipeline
+  - azureml-widgets
+  - pandas_ml
+  - statsmodels
+  - matplotlib

how-to-use-azureml/automated-machine-learning/forecasting-grouping/build.py

@@ -0,0 +1,144 @@
+from typing import List, Dict
+import copy
+import json
+import pandas as pd
+import re
+
+from azureml.core import RunConfiguration
+from azureml.core.compute import ComputeTarget
+from azureml.core.conda_dependencies import CondaDependencies
+from azureml.core.dataset import Dataset
+from azureml.data import TabularDataset
+from azureml.pipeline.core import PipelineData, PipelineParameter, TrainingOutput, StepSequence
+from azureml.pipeline.steps import PythonScriptStep
+from azureml.train.automl import AutoMLConfig
+from azureml.train.automl.runtime import AutoMLStep
+
+
+def _get_groups(data: Dataset, group_column_names: List[str]) -> pd.DataFrame:
+    return data._dataflow.distinct(columns=group_column_names)\
+        .keep_columns(columns=group_column_names).to_pandas_dataframe()
+
+
+def _get_configs(automlconfig: AutoMLConfig,
+                 data: Dataset,
+                 target_column: str,
+                 compute_target: ComputeTarget,
+                 group_column_names: List[str]) -> Dict[str, AutoMLConfig]:
+    # remove invalid characters regex
+    valid_chars = re.compile('[^a-zA-Z0-9-]')
+    groups = _get_groups(data, group_column_names)
+    configs = {}
+    for i, group in groups.iterrows():
+        single = data
+        group_name = "#####".join(str(x) for x in group.values)
+        group_name = valid_chars.sub('', group_name)
+        for key in group.index:
+            single = single._dataflow.filter(data._dataflow[key] == group[key])
+        t_dataset = TabularDataset._create(single)
+        group_conf = copy.deepcopy(automlconfig)
+        group_conf.user_settings['training_data'] = t_dataset
+        group_conf.user_settings['label_column_name'] = target_column
+        group_conf.user_settings['compute_target'] = compute_target
+        configs[group_name] = group_conf
+    return configs
+
+
+def build_pipeline_steps(automlconfig: AutoMLConfig,
+                         data: Dataset,
+                         target_column: str,
+                         compute_target: ComputeTarget,
+                         group_column_names: list,
+                         time_column_name: str,
+                         deploy: bool,
+                         service_name: str = 'grouping-demo') -> StepSequence:
+    steps = []
+
+    metrics_output_name = 'metrics_{}'
+    best_model_output_name = 'best_model_{}'
+    count = 0
+    model_names = []
+
+    # get all automl configs by group
+    configs = _get_configs(automlconfig, data, target_column, compute_target, group_column_names)
+
+    # build a runconfig for register model
+    register_config = RunConfiguration()
+    cd = CondaDependencies()
+    cd.add_pip_package('azureml-pipeline')
+    register_config.environment.python.conda_dependencies = cd
+
+    # create each automl step end-to-end (train, register)
+    for group_name, conf in configs.items():
+        # create automl metrics output
+        metirics_data = PipelineData(
+            name='metrics_data_{}'.format(group_name),
+            pipeline_output_name=metrics_output_name.format(group_name),
+            training_output=TrainingOutput(type='Metrics'))
+        # create automl model output
+        model_data = PipelineData(
+            name='model_data_{}'.format(group_name),
+            pipeline_output_name=best_model_output_name.format(group_name),
+            training_output=TrainingOutput(type='Model', metric=conf.user_settings['primary_metric']))
+
+        automl_step = AutoMLStep(
+            name='automl_{}'.format(group_name),
+            automl_config=conf,
+            outputs=[metirics_data, model_data],
+            allow_reuse=True)
+        steps.append(automl_step)
+
+        # pass the group name as a parameter to the register step ->
+        # this will become the name of the model for this group.
+        group_name_param = PipelineParameter("group_name_{}".format(count), default_value=group_name)
+        count += 1
+
+        reg_model_step = PythonScriptStep(
+            'register.py',
+            name='register_{}'.format(group_name),
+            arguments=["--model_name", group_name_param, "--model_path", model_data],
+            inputs=[model_data],
+            compute_target=compute_target,
+            runconfig=register_config,
+            source_directory="register",
+            allow_reuse=True
+        )
+        steps.append(reg_model_step)
+        model_names.append(group_name)
+
+    final_steps = steps
+    if deploy:
+        # modify the conda dependencies to ensure we pick up correct
+        # versions of azureml-defaults and azureml-train-automl
+        cd = CondaDependencies.create(pip_packages=['azureml-defaults', 'azureml-train-automl'])
+        automl_deps = CondaDependencies(conda_dependencies_file_path='deploy/myenv.yml')
+        cd._merge_dependencies(automl_deps)
+        cd.save('deploy/myenv.yml')
+
+        # add deployment step
+        pp_group_column_names = PipelineParameter(
+            "group_column_names",
+            default_value="#####".join(list(reversed(group_column_names))))
+
+        pp_model_names = PipelineParameter(
+            "model_names",
+            default_value=json.dumps(model_names))
+
+        pp_service_name = PipelineParameter(
+            "service_name",
+            default_value=service_name)
+
+        deployment_step = PythonScriptStep(
+            'deploy.py',
+            name='service_deploy',
+            arguments=["--group_column_names", pp_group_column_names,
+                       "--model_names", pp_model_names,
+                       "--service_name", pp_service_name,
+                       "--time_column_name", time_column_name],
+            compute_target=compute_target,
+            runconfig=RunConfiguration(),
+            source_directory="deploy"
+        )
+        final_steps = StepSequence(steps=[steps, deployment_step])
+
+    return final_steps

how-to-use-azureml/automated-machine-learning/forecasting-grouping/data/dominicks_OJ_2_5_8_test.csv

@@ -0,0 +1,61 @@
+WeekStarting,Store,Brand,Quantity,logQuantity,Advert,Price,Age60,COLLEGE,INCOME,Hincome150,Large HH,Minorities,WorkingWoman,SSTRDIST,SSTRVOL,CPDIST5,CPWVOL5
+1992-08-20,2,minute.maid,23488,10.06424493,1,1.94,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-08-20,2,tropicana,13376,9.501217335,1,2.79,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-08-27,2,tropicana,8128,9.00307017,0,2.75,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-08-27,2,minute.maid,19008,9.852615222,0,1.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-08-27,2,dominicks,9024,9.107642974,0,1.19,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-09-03,2,tropicana,19456,9.875910785,1,2.49,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-09-03,2,minute.maid,11584,9.357380115,0,1.81,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-09-03,2,dominicks,2048,7.624618986000001,0,2.09,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-09-10,2,tropicana,10048,9.215128888999999,0,2.64,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-09-10,2,minute.maid,26752,10.19436452,1,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-09-10,2,dominicks,1984,7.592870287999999,0,2.09,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-09-17,2,tropicana,6336,8.754002933999999,0,3.19,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-09-17,2,minute.maid,3904,8.269756948,0,2.83,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-09-17,2,dominicks,4160,8.333270353,0,1.77,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-09-24,2,tropicana,16192,9.692272572,1,2.79,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-09-24,2,minute.maid,3712,8.219326094,0,2.67,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-09-24,2,dominicks,35264,10.47061789,0,1.49,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-10-01,2,dominicks,8640,9.064157862,0,1.82,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-10-01,2,minute.maid,41216,10.62658181,1,2.19,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-10-01,2,tropicana,5824,8.66974259,0,2.97,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-08-20,5,tropicana,17728,9.78290059,1,2.79,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-08-20,5,minute.maid,27072,10.20625526,1,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-08-27,5,tropicana,9600,9.169518378,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-08-27,5,minute.maid,3840,8.253227646000001,0,1.69,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-08-27,5,dominicks,1856,7.526178913,0,1.29,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-09-03,5,tropicana,25664,10.15284451,1,2.49,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-09-03,5,minute.maid,6144,8.723231275,0,1.69,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-09-03,5,dominicks,3712,8.219326094,0,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-09-10,5,tropicana,9984,9.208739091,0,2.49,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-09-10,5,dominicks,2688,7.896552702,0,1.85,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-09-10,5,minute.maid,36416,10.50276352,1,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-09-17,5,tropicana,8576,9.056722882999999,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-09-17,5,minute.maid,5440,8.60153434,0,2.69,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-09-17,5,dominicks,6464,8.774003599999999,0,1.85,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-09-24,5,tropicana,13184,9.486759252,1,2.78,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-09-24,5,dominicks,40896,10.61878754,0,1.49,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-09-24,5,minute.maid,7680,8.946374826,0,2.49,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-10-01,5,dominicks,6144,8.723231275,0,1.85,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-10-01,5,minute.maid,50304,10.82583988,1,2.19,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-10-01,5,tropicana,7488,8.921057017999999,0,2.78,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-08-20,8,minute.maid,55552,10.9250748,1,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-08-20,8,tropicana,8576,9.056722882999999,1,2.79,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-08-27,8,tropicana,8000,8.987196821,0,2.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-08-27,8,minute.maid,18688,9.835636886,0,1.69,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-08-27,8,dominicks,19200,9.862665558,0,1.29,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-09-03,8,tropicana,21760,9.987828701,1,2.49,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-09-03,8,minute.maid,14656,9.592605087,0,1.69,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-09-03,8,dominicks,12800,9.45720045,0,1.79,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-09-10,8,tropicana,12800,9.45720045,0,2.49,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-09-10,8,minute.maid,30144,10.31374118,1,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-09-10,8,dominicks,15296,9.635346635,0,1.79,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-09-17,8,tropicana,10112,9.221478116,0,2.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-09-17,8,minute.maid,6208,8.733594062,0,2.49,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-09-17,8,dominicks,20992,9.951896692,0,1.79,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-09-24,8,tropicana,10304,9.240287448,1,2.79,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-09-24,8,minute.maid,7104,8.868413285,0,2.49,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-09-24,8,dominicks,73856,11.20987253,0,1.49,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-10-01,8,minute.maid,65856,11.09522582,1,2.19,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-10-01,8,dominicks,16192,9.692272572,0,1.79,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-10-01,8,tropicana,6400,8.764053269,0,2.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947

how-to-use-azureml/automated-machine-learning/forecasting-grouping/data/dominicks_OJ_2_5_8_train.csv

@@ -0,0 +1,973 @@
+WeekStarting,Store,Brand,Quantity,logQuantity,Advert,Price,Age60,COLLEGE,INCOME,Hincome150,Large HH,Minorities,WorkingWoman,SSTRDIST,SSTRVOL,CPDIST5,CPWVOL5
+1990-06-14,2,dominicks,10560,9.264828557000001,1,1.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-06-14,2,minute.maid,4480,8.407378325,0,3.17,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-06-14,2,tropicana,8256,9.018695487999999,0,3.87,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-07-26,2,dominicks,8000,8.987196821,0,2.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-07-26,2,minute.maid,4672,8.449342525,0,3.17,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-07-26,2,tropicana,6144,8.723231275,0,3.87,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-08-02,2,tropicana,3840,8.253227646000001,0,3.87,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-08-02,2,minute.maid,20160,9.911455722000001,1,2.39,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-08-02,2,dominicks,6848,8.831711918,1,2.09,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-08-09,2,dominicks,2880,7.965545572999999,0,2.09,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-08-09,2,minute.maid,2688,7.896552702,0,3.17,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-08-09,2,tropicana,8000,8.987196821,0,3.87,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-08-23,2,dominicks,1600,7.377758908,0,2.09,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-08-23,2,minute.maid,3008,8.009030685,0,3.17,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-08-23,2,tropicana,8896,9.093357017,0,3.87,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-08-30,2,tropicana,7168,8.877381955,0,3.87,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-08-30,2,minute.maid,4672,8.449342525,0,3.17,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-08-30,2,dominicks,25344,10.140297300000002,1,1.89,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-09-06,2,dominicks,10752,9.282847063,0,1.89,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-09-06,2,minute.maid,2752,7.920083199,0,3.17,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-09-06,2,tropicana,10880,9.29468152,0,3.29,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-09-13,2,minute.maid,26176,10.17259824,1,2.19,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-09-13,2,dominicks,6656,8.803273982999999,0,1.89,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-09-13,2,tropicana,7744,8.954673629,0,3.29,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-09-20,2,dominicks,6592,8.793612072,0,1.79,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-09-20,2,minute.maid,3712,8.219326094,0,3.17,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-09-20,2,tropicana,8512,9.049232212,0,3.29,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-10-11,2,tropicana,5504,8.61323038,0,3.29,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-10-11,2,minute.maid,30656,10.33058368,1,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-10-11,2,dominicks,1728,7.454719948999999,0,2.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-10-18,2,tropicana,5888,8.68067166,0,3.56,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-10-18,2,minute.maid,3840,8.253227646000001,0,2.98,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-10-18,2,dominicks,33792,10.42797937,1,1.24,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-10-25,2,tropicana,8384,9.034080407000001,0,3.56,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-10-25,2,minute.maid,2816,7.943072717000001,0,3.17,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-10-25,2,dominicks,1920,7.560080465,0,1.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-11-01,2,tropicana,5952,8.691482577,0,3.56,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-11-01,2,minute.maid,23104,10.04776104,1,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-11-01,2,dominicks,8960,9.100525506,1,1.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-11-08,2,dominicks,11392,9.340666634,0,1.29,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-11-08,2,tropicana,6848,8.831711918,0,3.56,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-11-08,2,minute.maid,3392,8.129174997,0,3.17,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-11-15,2,tropicana,9216,9.128696383,0,3.87,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-11-15,2,minute.maid,26304,10.1774763,1,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-11-15,2,dominicks,28416,10.25470765,0,0.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-11-22,2,dominicks,17152,9.749870064,1,1.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-11-22,2,tropicana,12160,9.405907156,0,2.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-11-22,2,minute.maid,6336,8.754002933999999,0,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-11-29,2,tropicana,12672,9.447150114,0,2.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-11-29,2,minute.maid,9920,9.2023082,0,3.17,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-11-29,2,dominicks,26560,10.1871616,1,2.49,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-12-06,2,dominicks,6336,8.754002933999999,0,2.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-12-06,2,minute.maid,25280,10.13776885,1,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-12-06,2,tropicana,6528,8.783855897,0,3.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-12-13,2,dominicks,26368,10.17990643,1,1.39,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-12-13,2,tropicana,6144,8.723231275,0,3.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-12-13,2,minute.maid,14848,9.605620455,0,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-12-20,2,tropicana,21120,9.957975738,0,2.39,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-12-20,2,minute.maid,12288,9.416378455,0,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-12-20,2,dominicks,896,6.797940412999999,0,2.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-12-27,2,tropicana,12416,9.426741242,0,2.39,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-12-27,2,minute.maid,6272,8.743850562,0,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-12-27,2,dominicks,1472,7.294377299,0,2.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-01-03,2,tropicana,9472,9.156095357,0,3.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-01-03,2,minute.maid,9152,9.121727714,0,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-01-03,2,dominicks,1344,7.2034055210000005,0,2.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-01-10,2,tropicana,17920,9.793672686,0,2.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-01-10,2,minute.maid,4160,8.333270353,0,2.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-01-10,2,dominicks,111680,11.62339292,1,0.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-01-17,2,tropicana,9408,9.14931567,0,2.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-01-17,2,minute.maid,10176,9.227787286,0,2.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-01-17,2,dominicks,1856,7.526178913,0,2.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-01-24,2,tropicana,6272,8.743850562,0,3.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-01-24,2,minute.maid,29056,10.27698028,1,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-01-24,2,dominicks,5568,8.624791202,0,2.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-01-31,2,tropicana,6912,8.841014311,0,3.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-01-31,2,minute.maid,7104,8.868413285,0,2.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-01-31,2,dominicks,32064,10.37548918,1,1.49,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-02-07,2,tropicana,16768,9.727227587,0,2.49,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-02-07,2,dominicks,4352,8.378390789,0,1.49,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-02-07,2,minute.maid,7488,8.921057017999999,0,2.49,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-02-14,2,dominicks,704,6.556778356000001,0,2.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-02-14,2,minute.maid,4224,8.348537825,0,2.49,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-02-14,2,tropicana,6272,8.743850562,0,3.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-02-21,2,tropicana,7936,8.979164649,0,3.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-02-21,2,minute.maid,8960,9.100525506,0,2.49,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-02-21,2,dominicks,13760,9.529521112000001,0,2.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-02-28,2,tropicana,6144,8.723231275,0,3.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-02-28,2,minute.maid,22464,10.01966931,1,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-02-28,2,dominicks,43328,10.67655436,1,1.09,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-03-07,2,tropicana,7936,8.979164649,0,3.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-03-07,2,minute.maid,3840,8.253227646000001,0,2.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-03-07,2,dominicks,57600,10.96127785,1,1.09,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-03-14,2,tropicana,7808,8.962904128,0,3.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-03-14,2,minute.maid,12992,9.472089062,0,2.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-03-14,2,dominicks,704,6.556778356000001,0,2.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-03-21,2,tropicana,6080,8.712759975,0,3.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-03-21,2,minute.maid,70144,11.15830555,1,1.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-03-21,2,dominicks,6016,8.702177866,0,2.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-03-28,2,tropicana,42176,10.64960662,1,1.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-03-28,2,dominicks,10368,9.246479419,1,1.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-03-28,2,minute.maid,21248,9.964018052,0,1.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-04-04,2,dominicks,12608,9.442086812000001,0,1.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-04-04,2,minute.maid,5696,8.647519453,1,2.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-04-04,2,tropicana,4928,8.502688505,0,3.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-04-11,2,tropicana,29504,10.29228113,1,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-04-11,2,minute.maid,7680,8.946374826,0,2.09,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-04-11,2,dominicks,6336,8.754002933999999,0,1.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-04-18,2,tropicana,9984,9.208739091,0,3.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-04-18,2,minute.maid,6336,8.754002933999999,0,2.09,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-04-18,2,dominicks,140736,11.85464107,1,0.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-04-25,2,tropicana,35200,10.46880136,1,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-04-25,2,dominicks,960,6.866933285,1,2.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-04-25,2,minute.maid,8576,9.056722882999999,0,2.09,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-05-02,2,dominicks,1216,7.103322062999999,0,2.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-05-02,2,minute.maid,15104,9.622714887999999,0,2.09,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-05-02,2,tropicana,23936,10.08313888,0,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-05-09,2,tropicana,7104,8.868413285,0,3.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-05-09,2,minute.maid,76480,11.24478455,1,1.39,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-05-09,2,dominicks,1664,7.416979621,0,2.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-05-16,2,dominicks,4992,8.51559191,0,2.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-05-16,2,minute.maid,5056,8.528330936,0,2.39,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-05-16,2,tropicana,24512,10.10691807,1,2.29,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-05-23,2,tropicana,6336,8.754002933999999,0,3.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-05-23,2,minute.maid,4736,8.462948177000001,0,2.39,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-05-23,2,dominicks,27968,10.23881628,1,1.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-05-30,2,dominicks,12160,9.405907156,0,1.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-05-30,2,minute.maid,4480,8.407378325,0,2.39,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-05-30,2,tropicana,6080,8.712759975,0,3.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-06-06,2,tropicana,33536,10.42037477,0,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-06-06,2,minute.maid,4032,8.30201781,0,2.39,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-06-06,2,dominicks,2240,7.714231145,0,2.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-06-13,2,dominicks,5504,8.61323038,1,1.49,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-06-13,2,minute.maid,14784,9.601300794,1,1.79,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-06-13,2,tropicana,13248,9.491601877,0,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-06-20,2,tropicana,6208,8.733594062,0,3.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-06-20,2,dominicks,8832,9.086136769,0,1.29,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-06-20,2,minute.maid,12096,9.400630097999999,0,2.39,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-06-27,2,dominicks,2624,7.87245515,0,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-06-27,2,minute.maid,41792,10.64046021,1,1.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-06-27,2,tropicana,10624,9.270870872,0,3.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-07-04,2,tropicana,44672,10.70710219,0,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-07-04,2,minute.maid,10560,9.264828557000001,0,1.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-07-04,2,dominicks,10432,9.252633284,0,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-07-18,2,tropicana,20096,9.908276069,0,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-07-18,2,dominicks,8320,9.026417534,0,1.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-07-18,2,minute.maid,4224,8.348537825,0,2.39,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-07-25,2,dominicks,6784,8.822322178,0,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-07-25,2,minute.maid,2880,7.965545572999999,0,2.39,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-07-25,2,tropicana,9152,9.121727714,1,3.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-08-01,2,tropicana,21952,9.996613531,0,2.19,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-08-01,2,minute.maid,3968,8.286017467999999,0,2.39,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-08-01,2,dominicks,60544,11.01112565,1,0.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-08-08,2,dominicks,20608,9.933434629,0,0.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-08-08,2,minute.maid,3712,8.219326094,0,2.39,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-08-08,2,tropicana,13568,9.515469357999999,0,2.19,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-08-29,2,tropicana,4160,8.333270353,0,3.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-08-29,2,minute.maid,2816,7.943072717000001,0,2.39,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-08-29,2,dominicks,16064,9.684336023,0,1.39,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-09-05,2,tropicana,39424,10.58213005,1,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-09-05,2,minute.maid,4288,8.363575702999999,0,2.39,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-09-05,2,dominicks,12480,9.431882642,0,1.39,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-09-12,2,tropicana,5632,8.636219898,0,3.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-09-12,2,minute.maid,18240,9.811372264,1,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-09-12,2,dominicks,17024,9.742379392,0,1.39,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-09-19,2,dominicks,13440,9.505990614,1,1.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-09-19,2,minute.maid,7360,8.903815212,0,1.95,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-09-19,2,tropicana,9024,9.107642974,1,2.68,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-09-26,2,tropicana,6016,8.702177866,0,3.44,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-09-26,2,minute.maid,7808,8.962904128,0,1.83,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-09-26,2,dominicks,10112,9.221478116,0,1.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-10-03,2,dominicks,9088,9.114710141,0,1.56,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-10-03,2,minute.maid,13504,9.510741217,0,1.79,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-10-03,2,tropicana,7744,8.954673629,0,3.14,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-10-10,2,tropicana,6784,8.822322178,0,3.07,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-10-10,2,dominicks,22848,10.03661887,1,1.49,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-10-10,2,minute.maid,10048,9.215128888999999,0,1.91,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-10-17,2,dominicks,6976,8.850230966,0,1.65,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-10-17,2,minute.maid,135936,11.81993947,1,1.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-10-17,2,tropicana,6784,8.822322178,0,3.07,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-10-24,2,tropicana,6272,8.743850562,0,3.07,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-10-24,2,minute.maid,5056,8.528330936,0,2.39,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-10-24,2,dominicks,4160,8.333270353,0,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-10-31,2,tropicana,5312,8.577723691000001,0,3.07,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-10-31,2,minute.maid,27968,10.23881628,0,1.49,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-10-31,2,dominicks,3328,8.110126802,0,1.83,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-11-07,2,tropicana,9216,9.128696383,0,3.11,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-11-07,2,minute.maid,4736,8.462948177000001,0,2.39,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-11-07,2,dominicks,12096,9.400630097999999,1,1.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-11-14,2,tropicana,7296,8.895081532,0,3.19,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-11-14,2,minute.maid,7808,8.962904128,0,2.14,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-11-14,2,dominicks,6208,8.733594062,0,1.76,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-11-21,2,tropicana,34240,10.44114983,1,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-11-21,2,minute.maid,12480,9.431882642,0,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-11-21,2,dominicks,3008,8.009030685,0,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-11-28,2,dominicks,19456,9.875910785,1,1.5,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-11-28,2,minute.maid,9664,9.17616292,0,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-11-28,2,tropicana,7168,8.877381955,0,2.64,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-12-05,2,minute.maid,7168,8.877381955,0,2.06,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-12-05,2,dominicks,16768,9.727227587,0,1.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-12-05,2,tropicana,6080,8.712759975,0,3.19,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-12-12,2,dominicks,13568,9.515469357999999,1,1.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-12-12,2,minute.maid,4480,8.407378325,0,2.39,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-12-12,2,tropicana,5120,8.540909718,0,3.19,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-12-19,2,tropicana,8320,9.026417534,0,2.74,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-12-19,2,minute.maid,5952,8.691482577,0,2.22,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-12-19,2,dominicks,6080,8.712759975,0,1.61,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-12-26,2,dominicks,10432,9.252633284,1,1.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-12-26,2,minute.maid,21696,9.984883191,1,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1991-12-26,2,tropicana,17728,9.78290059,0,2.39,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-01-02,2,minute.maid,12032,9.395325046,0,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-01-02,2,dominicks,11712,9.368369236,0,1.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-01-02,2,tropicana,13120,9.481893063,0,2.35,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-01-09,2,dominicks,4032,8.30201781,0,1.76,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-01-09,2,minute.maid,7040,8.859363449,0,2.12,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-01-09,2,tropicana,13120,9.481893063,0,2.29,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-01-16,2,dominicks,6336,8.754002933999999,0,1.82,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-01-16,2,tropicana,9792,9.189321005,0,2.43,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-01-16,2,minute.maid,10240,9.234056899,1,2.49,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-01-23,2,tropicana,3520,8.166216269,0,3.19,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-01-23,2,minute.maid,6848,8.831711918,1,2.49,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-01-23,2,dominicks,13632,9.520175249,0,1.47,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-01-30,2,tropicana,5504,8.61323038,0,3.19,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-01-30,2,minute.maid,3968,8.286017467999999,0,2.61,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-01-30,2,dominicks,45120,10.71708089,0,1.29,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-02-06,2,tropicana,6720,8.812843434,0,3.19,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-02-06,2,minute.maid,5888,8.68067166,0,2.26,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-02-06,2,dominicks,9984,9.208739091,0,1.39,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-02-13,2,tropicana,20224,9.914625297,1,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-02-13,2,dominicks,4800,8.476371197,0,1.82,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-02-13,2,minute.maid,6208,8.733594062,0,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-02-20,2,dominicks,11776,9.373818841,0,1.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-02-20,2,minute.maid,72256,11.18797065,1,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-02-20,2,tropicana,5056,8.528330936,0,3.19,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-02-27,2,tropicana,43584,10.68244539,1,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-02-27,2,minute.maid,11520,9.351839934,0,2.11,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-02-27,2,dominicks,11584,9.357380115,0,1.54,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-03-05,2,tropicana,25728,10.15533517,0,1.79,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-03-05,2,minute.maid,5824,8.66974259,0,2.35,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-03-05,2,dominicks,51264,10.84474403,1,1.39,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-03-12,2,tropicana,31808,10.36747311,0,1.79,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-03-12,2,minute.maid,19392,9.872615889,1,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-03-12,2,dominicks,14976,9.614204199,0,1.44,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-03-19,2,tropicana,20736,9.939626599,0,1.91,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-03-19,2,minute.maid,9536,9.162829389,0,2.1,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-03-19,2,dominicks,30784,10.33475035,0,1.59,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-03-26,2,tropicana,15168,9.626943225,0,2.81,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-03-26,2,minute.maid,5312,8.577723691000001,0,2.28,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-03-26,2,dominicks,12480,9.431882642,0,1.6,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-04-02,2,tropicana,28096,10.2433825,1,2.5,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-04-02,2,dominicks,3264,8.090708716,0,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-04-02,2,minute.maid,14528,9.583833101,1,1.9,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-04-09,2,dominicks,8768,9.078864009,0,1.48,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-04-09,2,minute.maid,12416,9.426741242,0,2.12,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-04-09,2,tropicana,12416,9.426741242,0,2.58,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-04-16,2,tropicana,5376,8.589699882,0,3.19,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-04-16,2,minute.maid,5376,8.589699882,0,2.79,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-04-16,2,dominicks,70848,11.16829202,1,1.29,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-04-23,2,tropicana,9792,9.189321005,0,2.67,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-04-23,2,minute.maid,19008,9.852615222,1,2.09,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-04-23,2,dominicks,18560,9.828764006,0,1.42,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-04-30,2,tropicana,16960,9.738612909,1,2.39,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-04-30,2,minute.maid,3904,8.269756948,0,2.79,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-04-30,2,dominicks,9152,9.121727714,0,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-05-07,2,tropicana,8320,9.026417534,0,3.19,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-05-07,2,minute.maid,6336,8.754002933999999,0,2.79,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-05-07,2,dominicks,9600,9.169518378,0,2.0,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-05-14,2,tropicana,6912,8.841014311,0,3.19,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-05-14,2,minute.maid,5440,8.60153434,0,2.79,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-05-14,2,dominicks,4800,8.476371197,0,2.09,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-05-21,2,tropicana,6976,8.850230966,0,3.19,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-05-21,2,minute.maid,22400,10.01681624,1,2.09,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-05-21,2,dominicks,9664,9.17616292,0,1.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-05-28,2,minute.maid,3968,8.286017467999999,0,2.84,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-05-28,2,tropicana,7232,8.886270902,0,3.19,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-05-28,2,dominicks,45568,10.726961,0,1.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-06-04,2,tropicana,51520,10.84972536,1,2.49,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-06-04,2,minute.maid,3264,8.090708716,0,2.89,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-06-04,2,dominicks,20992,9.951896692,0,1.74,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-06-11,2,minute.maid,4352,8.378390789,0,2.89,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-06-11,2,tropicana,22272,10.01108556,0,2.21,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-06-11,2,dominicks,6592,8.793612072,0,2.09,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-06-18,2,dominicks,4992,8.51559191,0,2.05,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-06-18,2,minute.maid,4480,8.407378325,0,2.89,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-06-18,2,tropicana,46144,10.73952222,1,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-06-25,2,tropicana,4352,8.378390789,1,3.19,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-06-25,2,minute.maid,3840,8.253227646000001,0,2.52,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-06-25,2,dominicks,8064,8.99516499,0,1.24,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-07-02,2,tropicana,17280,9.757305042,0,2.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-07-02,2,minute.maid,13312,9.496421162999999,1,2.0,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-07-02,2,dominicks,7360,8.903815212,0,1.61,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-07-09,2,tropicana,5696,8.647519453,0,3.19,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-07-09,2,minute.maid,3776,8.236420527,1,2.33,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-07-09,2,dominicks,10048,9.215128888999999,0,1.4,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-07-16,2,tropicana,6848,8.831711918,0,3.19,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-07-16,2,dominicks,10112,9.221478116,0,1.91,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-07-16,2,minute.maid,4800,8.476371197,0,2.89,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-07-23,2,dominicks,9152,9.121727714,0,1.69,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-07-23,2,minute.maid,24960,10.12502982,1,2.29,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-07-23,2,tropicana,4416,8.392989587999999,0,3.19,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-07-30,2,tropicana,4672,8.449342525,0,3.16,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-07-30,2,minute.maid,4544,8.42156296,0,2.86,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-07-30,2,dominicks,36288,10.49924239,1,1.49,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-08-06,2,tropicana,7168,8.877381955,1,3.09,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-08-06,2,minute.maid,3968,8.286017467999999,1,2.81,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-08-06,2,dominicks,3776,8.236420527,1,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-08-13,2,tropicana,5056,8.528330936,0,3.19,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-08-13,2,dominicks,3328,8.110126802,0,1.97,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-08-13,2,minute.maid,49600,10.81174611,1,1.99,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1992-08-20,2,dominicks,13824,9.534161491,0,1.36,0.232864734,0.248934934,10.55320518,0.463887065,0.103953406,0.114279949,0.303585347,2.110122129,1.142857143,1.927279669,0.37692661299999997
+1990-06-14,5,dominicks,1792,7.491087594,1,1.59,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-06-14,5,minute.maid,4224,8.348537825,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-06-14,5,tropicana,5888,8.68067166,0,3.66,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-06-28,5,minute.maid,4352,8.378390789,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-06-28,5,dominicks,2496,7.82244473,0,2.49,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-06-28,5,tropicana,6976,8.850230966,0,3.66,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-07-05,5,dominicks,2944,7.98752448,0,2.49,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-07-05,5,minute.maid,4928,8.502688505,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-07-05,5,tropicana,6528,8.783855897,0,3.66,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-07-12,5,dominicks,1024,6.931471806,0,2.49,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-07-12,5,minute.maid,31168,10.34714721,1,2.59,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-07-12,5,tropicana,4928,8.502688505,0,3.66,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-07-26,5,dominicks,4224,8.348537825,0,2.49,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-07-26,5,minute.maid,10048,9.215128888999999,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-07-26,5,tropicana,5312,8.577723691000001,0,3.66,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-08-02,5,minute.maid,21760,9.987828701,1,2.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-08-02,5,tropicana,5120,8.540909718,0,3.66,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-08-02,5,dominicks,4544,8.42156296,1,2.09,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-08-09,5,dominicks,1728,7.454719948999999,0,2.09,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-08-09,5,minute.maid,4544,8.42156296,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-08-09,5,tropicana,7936,8.979164649,0,3.66,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-08-16,5,tropicana,6080,8.712759975,0,3.66,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-08-16,5,minute.maid,52224,10.86329744,1,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-08-16,5,dominicks,1216,7.103322062999999,0,2.09,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-08-23,5,dominicks,1152,7.049254841000001,0,2.09,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-08-23,5,minute.maid,3584,8.184234774,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-08-23,5,tropicana,4160,8.333270353,0,3.66,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-08-30,5,minute.maid,5120,8.540909718,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-08-30,5,tropicana,5888,8.68067166,0,3.66,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-08-30,5,dominicks,30144,10.31374118,1,1.89,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-09-06,5,dominicks,8960,9.100525506,0,1.89,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-09-06,5,minute.maid,4416,8.392989587999999,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-09-06,5,tropicana,9536,9.162829389,0,3.29,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-09-13,5,tropicana,8320,9.026417534,0,3.29,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-09-13,5,dominicks,8192,9.010913347,0,1.89,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-09-13,5,minute.maid,30208,10.31586207,1,2.19,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-09-20,5,dominicks,6528,8.783855897,0,1.79,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-09-20,5,minute.maid,4160,8.333270353,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-09-20,5,tropicana,8000,8.987196821,0,3.29,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-09-27,5,dominicks,34688,10.45414909,1,1.79,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-09-27,5,minute.maid,4992,8.51559191,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-09-27,5,tropicana,5824,8.66974259,0,3.66,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-10-04,5,dominicks,4672,8.449342525,0,1.79,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-10-04,5,minute.maid,13952,9.543378146,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-10-04,5,tropicana,10624,9.270870872,1,3.29,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-10-11,5,tropicana,6656,8.803273982999999,0,3.29,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-10-11,5,dominicks,1088,6.992096427000001,0,2.49,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-10-11,5,minute.maid,47680,10.772267300000001,1,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-10-18,5,tropicana,5184,8.553332238,0,3.51,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-10-18,5,minute.maid,7616,8.938006577000001,0,2.69,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-10-18,5,dominicks,69440,11.14821835,1,1.24,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-10-25,5,tropicana,4928,8.502688505,0,3.51,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-10-25,5,minute.maid,8896,9.093357017,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-10-25,5,dominicks,1280,7.154615357000001,0,1.59,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-11-01,5,tropicana,5888,8.68067166,0,3.51,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-11-01,5,minute.maid,28544,10.25920204,1,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-11-01,5,dominicks,35456,10.47604777,1,1.59,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-11-08,5,tropicana,5312,8.577723691000001,0,3.51,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-11-08,5,dominicks,13824,9.534161491,0,1.29,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-11-08,5,minute.maid,5440,8.60153434,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-11-15,5,tropicana,9984,9.208739091,0,3.66,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-11-15,5,minute.maid,52416,10.86696717,1,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-11-15,5,dominicks,14208,9.561560465,0,0.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-11-22,5,tropicana,8448,9.041685006,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-11-22,5,dominicks,29312,10.28575227,1,1.59,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-11-22,5,minute.maid,11712,9.368369236,0,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-11-29,5,tropicana,10880,9.29468152,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-11-29,5,minute.maid,13952,9.543378146,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-11-29,5,dominicks,52992,10.87789624,1,2.49,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-12-06,5,dominicks,15680,9.660141293999999,0,2.19,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-12-06,5,minute.maid,36160,10.49570882,1,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-12-06,5,tropicana,5696,8.647519453,0,3.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-12-13,5,tropicana,5696,8.647519453,0,3.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-12-13,5,minute.maid,12864,9.462187991,0,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-12-13,5,dominicks,43520,10.68097588,1,1.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-12-20,5,tropicana,32384,10.38541975,0,2.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-12-20,5,minute.maid,22208,10.00820786,0,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-12-20,5,dominicks,3904,8.269756948,0,2.19,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-12-27,5,tropicana,10752,9.282847063,0,2.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-12-27,5,minute.maid,9984,9.208739091,0,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-12-27,5,dominicks,896,6.797940412999999,0,2.19,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-01-03,5,tropicana,6912,8.841014311,0,3.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-01-03,5,minute.maid,14016,9.547954812999999,0,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-01-03,5,dominicks,2240,7.714231145,0,2.19,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-01-10,5,tropicana,13440,9.505990614,0,2.59,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-01-10,5,minute.maid,6080,8.712759975,0,2.46,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-01-10,5,dominicks,125760,11.74213061,1,0.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-01-17,5,tropicana,7808,8.962904128,0,2.59,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-01-17,5,minute.maid,7808,8.962904128,0,2.46,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-01-17,5,dominicks,1408,7.249925537,0,2.19,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-01-24,5,tropicana,5248,8.565602331000001,0,3.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-01-24,5,minute.maid,40896,10.61878754,1,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-01-24,5,dominicks,7232,8.886270902,0,2.19,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-01-31,5,tropicana,6208,8.733594062,0,3.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-01-31,5,minute.maid,6272,8.743850562,0,2.46,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-01-31,5,dominicks,41216,10.62658181,1,1.49,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-02-07,5,tropicana,21440,9.973013615,0,2.49,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-02-07,5,minute.maid,7872,8.971067439,0,2.41,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-02-07,5,dominicks,9024,9.107642974,0,1.49,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-02-14,5,dominicks,1600,7.377758908,0,2.19,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-02-14,5,tropicana,7360,8.903815212,0,3.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-02-14,5,minute.maid,6144,8.723231275,0,2.41,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-02-21,5,tropicana,6720,8.812843434,0,3.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-02-21,5,minute.maid,8448,9.041685006,0,2.41,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-02-21,5,dominicks,2496,7.82244473,0,2.19,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-02-28,5,tropicana,6656,8.803273982999999,0,3.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-02-28,5,minute.maid,18688,9.835636886,1,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-02-28,5,dominicks,6336,8.754002933999999,1,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-03-07,5,tropicana,6016,8.702177866,0,3.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-03-07,5,minute.maid,6272,8.743850562,0,2.46,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-03-07,5,dominicks,56384,10.93994071,1,1.09,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-03-14,5,tropicana,6144,8.723231275,0,3.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-03-14,5,minute.maid,12096,9.400630097999999,0,2.46,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-03-14,5,dominicks,1600,7.377758908,0,2.19,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-03-21,5,tropicana,4928,8.502688505,0,3.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-03-21,5,minute.maid,73216,11.20116926,1,1.69,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-03-21,5,dominicks,2944,7.98752448,0,2.19,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-03-28,5,tropicana,67712,11.1230187,1,1.69,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-03-28,5,minute.maid,18944,9.849242538,0,1.69,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-03-28,5,dominicks,13504,9.510741217,1,1.59,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-04-04,5,dominicks,5376,8.589699882,0,1.59,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-04-04,5,tropicana,8640,9.064157862,0,3.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-04-04,5,minute.maid,6400,8.764053269,1,2.46,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-04-11,5,tropicana,35520,10.477851199999998,1,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-04-11,5,minute.maid,8640,9.064157862,0,2.09,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-04-11,5,dominicks,6656,8.803273982999999,0,1.59,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-04-18,5,tropicana,9664,9.17616292,0,3.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-04-18,5,minute.maid,7296,8.895081532,0,2.09,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-04-18,5,dominicks,95680,11.46876457,1,0.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-04-25,5,tropicana,49088,10.80136989,1,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-04-25,5,minute.maid,12480,9.431882642,0,2.09,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-04-25,5,dominicks,896,6.797940412999999,1,2.19,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-05-02,5,dominicks,1728,7.454719948999999,0,2.19,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-05-02,5,minute.maid,14144,9.557045785,0,2.09,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-05-02,5,tropicana,14912,9.609921537,0,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-05-09,5,minute.maid,88256,11.38799696,1,1.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-05-09,5,tropicana,6464,8.774003599999999,0,3.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-05-09,5,dominicks,1280,7.154615357000001,0,2.19,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-05-16,5,dominicks,5696,8.647519453,0,2.19,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-05-16,5,minute.maid,6848,8.831711918,0,2.26,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-05-16,5,tropicana,25024,10.12759064,1,2.29,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-05-23,5,minute.maid,7808,8.962904128,0,2.26,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-05-23,5,tropicana,6272,8.743850562,0,3.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-05-23,5,dominicks,28288,10.25019297,1,1.59,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-05-30,5,dominicks,4864,8.489616424,0,1.59,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-05-30,5,minute.maid,6272,8.743850562,0,2.26,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-05-30,5,tropicana,5056,8.528330936,0,3.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-06-06,5,minute.maid,6144,8.723231275,0,2.26,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-06-06,5,tropicana,47616,10.77092412,0,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-06-06,5,dominicks,2880,7.965545572999999,0,2.09,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-06-13,5,dominicks,5760,8.658692754,1,1.41,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-06-13,5,minute.maid,27776,10.23192762,1,1.69,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-06-13,5,tropicana,13888,9.538780437,0,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-06-20,5,tropicana,6144,8.723231275,0,3.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-06-20,5,minute.maid,20800,9.942708266,0,2.26,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-06-20,5,dominicks,15040,9.618468598,0,1.29,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-06-27,5,dominicks,5120,8.540909718,0,1.89,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-06-27,5,minute.maid,45696,10.72976605,1,1.69,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-06-27,5,tropicana,9344,9.142489705,0,3.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-07-04,5,minute.maid,14336,9.570529135,0,1.69,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-07-04,5,tropicana,32896,10.40110635,0,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-07-04,5,dominicks,3264,8.090708716,0,1.89,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-07-11,5,dominicks,9536,9.162829389,1,1.59,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-07-11,5,minute.maid,4928,8.502688505,0,2.26,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-07-11,5,tropicana,21056,9.954940834,0,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-07-18,5,tropicana,15360,9.639522007,0,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-07-18,5,minute.maid,4608,8.435549202,0,2.26,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-07-18,5,dominicks,6208,8.733594062,0,1.59,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-07-25,5,dominicks,6592,8.793612072,0,1.89,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-07-25,5,tropicana,8000,8.987196821,1,3.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-07-25,5,minute.maid,5248,8.565602331000001,0,2.26,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-08-01,5,tropicana,21120,9.957975738,0,2.19,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-08-01,5,dominicks,63552,11.05961375,1,0.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-08-01,5,minute.maid,4224,8.348537825,0,2.26,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-08-08,5,dominicks,27968,10.23881628,0,0.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-08-08,5,minute.maid,4288,8.363575702999999,0,2.26,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-08-08,5,tropicana,11904,9.384629757,0,2.19,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-08-15,5,minute.maid,16896,9.734832187,0,2.26,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-08-15,5,tropicana,5056,8.528330936,0,3.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-08-15,5,dominicks,21760,9.987828701,1,1.49,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-08-22,5,dominicks,2688,7.896552702,0,1.49,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-08-22,5,minute.maid,77184,11.25394746,1,1.29,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-08-22,5,tropicana,4608,8.435549202,0,3.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-08-29,5,tropicana,6016,8.702177866,0,3.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-08-29,5,minute.maid,5184,8.553332238,0,2.26,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-08-29,5,dominicks,10432,9.252633284,0,1.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-09-05,5,tropicana,50752,10.83470631,1,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-09-05,5,minute.maid,5248,8.565602331000001,0,2.26,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-09-05,5,dominicks,9792,9.189321005,0,1.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-09-12,5,minute.maid,20672,9.936535407000001,1,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-09-12,5,tropicana,5632,8.636219898,0,3.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-09-12,5,dominicks,8448,9.041685006,0,1.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-09-26,5,tropicana,6400,8.764053269,0,3.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-09-26,5,dominicks,6912,8.841014311,0,1.58,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-09-26,5,minute.maid,12352,9.421573272,0,1.89,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-10-03,5,dominicks,8256,9.018695487999999,0,1.58,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-10-03,5,minute.maid,12032,9.395325046,0,1.79,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-10-03,5,tropicana,5440,8.60153434,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-10-10,5,minute.maid,13440,9.505990614,0,1.79,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-10-10,5,dominicks,28672,10.26367632,1,1.58,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-10-10,5,tropicana,8128,9.00307017,0,2.94,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-10-24,5,tropicana,7232,8.886270902,0,2.94,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-10-24,5,minute.maid,5824,8.66974259,0,2.26,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-10-24,5,dominicks,4416,8.392989587999999,0,1.58,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-10-31,5,tropicana,7168,8.877381955,0,2.94,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-10-31,5,minute.maid,50112,10.82201578,0,1.49,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-10-31,5,dominicks,1856,7.526178913,0,1.58,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-11-07,5,minute.maid,5184,8.553332238,0,2.26,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-11-07,5,tropicana,7872,8.971067439,0,2.94,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-11-07,5,dominicks,6528,8.783855897,1,1.58,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-11-14,5,tropicana,7552,8.929567707999999,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-11-14,5,minute.maid,8384,9.034080407000001,0,2.26,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-11-14,5,dominicks,6080,8.712759975,0,1.58,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-11-21,5,tropicana,69504,11.14913958,1,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-11-21,5,dominicks,3456,8.14786713,0,1.58,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-11-21,5,minute.maid,10112,9.221478116,0,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-11-28,5,dominicks,25856,10.16029796,1,1.58,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-11-28,5,minute.maid,8384,9.034080407000001,0,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-11-28,5,tropicana,8960,9.100525506,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-12-05,5,tropicana,6912,8.841014311,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-12-05,5,dominicks,25728,10.15533517,0,1.58,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-12-05,5,minute.maid,11456,9.346268889,0,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-12-12,5,dominicks,23552,10.06696602,1,1.58,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-12-12,5,minute.maid,5952,8.691482577,0,2.26,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-12-12,5,tropicana,6656,8.803273982999999,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-12-19,5,tropicana,8192,9.010913347,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-12-19,5,dominicks,2944,7.98752448,0,1.58,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-12-19,5,minute.maid,8512,9.049232212,0,2.26,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-12-26,5,dominicks,5888,8.68067166,1,1.58,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-12-26,5,minute.maid,27968,10.23881628,1,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1991-12-26,5,tropicana,13440,9.505990614,0,2.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-01-02,5,tropicana,12160,9.405907156,0,2.39,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-01-02,5,dominicks,6848,8.831711918,0,1.58,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-01-02,5,minute.maid,24000,10.08580911,0,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-01-09,5,dominicks,1792,7.491087594,0,1.58,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-01-09,5,minute.maid,6848,8.831711918,0,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-01-09,5,tropicana,11840,9.379238908,0,2.29,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-01-16,5,tropicana,8640,9.064157862,0,2.29,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-01-16,5,dominicks,5248,8.565602331000001,0,1.58,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-01-16,5,minute.maid,15104,9.622714887999999,1,2.49,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-01-23,5,tropicana,5888,8.68067166,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-01-23,5,minute.maid,11392,9.340666634,1,2.49,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-01-23,5,dominicks,16768,9.727227587,0,1.58,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-01-30,5,tropicana,7424,8.912473275,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-01-30,5,minute.maid,5824,8.66974259,0,2.49,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-01-30,5,dominicks,52160,10.8620712,0,1.58,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-02-06,5,tropicana,5632,8.636219898,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-02-06,5,minute.maid,7488,8.921057017999999,0,2.66,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-02-06,5,dominicks,16640,9.719564714,0,1.58,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-02-13,5,tropicana,33600,10.42228135,1,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-02-13,5,minute.maid,8320,9.026417534,0,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-02-13,5,dominicks,1344,7.2034055210000005,0,1.58,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-02-20,5,dominicks,4608,8.435549202,0,1.58,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-02-20,5,tropicana,5376,8.589699882,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-02-20,5,minute.maid,99904,11.511965,1,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-02-27,5,tropicana,54272,10.90176372,1,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-02-27,5,minute.maid,6976,8.850230966,0,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-02-27,5,dominicks,12672,9.447150114,0,1.58,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-03-05,5,tropicana,33600,10.42228135,0,1.79,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-03-05,5,minute.maid,9984,9.208739091,0,2.66,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-03-05,5,dominicks,48640,10.79220152,1,1.58,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-03-12,5,tropicana,24448,10.10430369,0,1.79,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-03-12,5,minute.maid,32832,10.39915893,1,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-03-12,5,dominicks,13248,9.491601877,0,1.58,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-03-19,5,tropicana,22784,10.03381381,0,1.79,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-03-19,5,minute.maid,8128,9.00307017,0,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-03-19,5,dominicks,29248,10.28356647,0,1.58,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-03-26,5,tropicana,19008,9.852615222,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-03-26,5,minute.maid,6464,8.774003599999999,0,2.66,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-03-26,5,dominicks,4608,8.435549202,0,1.58,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-04-02,5,tropicana,15808,9.66827142,1,2.5,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-04-02,5,minute.maid,36800,10.51325312,1,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-04-02,5,dominicks,3136,8.050703382,0,1.58,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-04-09,5,dominicks,13184,9.486759252,0,1.58,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-04-09,5,tropicana,14144,9.557045785,0,2.5,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-04-09,5,minute.maid,12928,9.467150781,0,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-04-16,5,tropicana,9600,9.169518378,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-04-16,5,minute.maid,7424,8.912473275,0,2.66,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-04-16,5,dominicks,67712,11.1230187,1,1.29,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-04-23,5,tropicana,10112,9.221478116,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-04-23,5,minute.maid,34176,10.43927892,1,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-04-23,5,dominicks,18880,9.84585844,0,1.29,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-04-30,5,minute.maid,4160,8.333270353,0,2.66,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-04-30,5,tropicana,31872,10.36948316,1,2.24,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-04-30,5,dominicks,6208,8.733594062,0,1.89,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-05-07,5,tropicana,9280,9.135616826,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-05-07,5,minute.maid,5952,8.691482577,0,2.66,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-05-07,5,dominicks,5952,8.691482577,0,1.89,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-05-14,5,tropicana,7680,8.946374826,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-05-14,5,minute.maid,6528,8.783855897,0,2.66,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-05-14,5,dominicks,4160,8.333270353,0,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-05-21,5,tropicana,8704,9.071537969,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-05-21,5,minute.maid,30656,10.33058368,1,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-05-21,5,dominicks,23488,10.06424493,0,1.69,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-05-28,5,tropicana,9920,9.2023082,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-05-28,5,dominicks,60480,11.01006801,0,1.69,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-05-28,5,minute.maid,6656,8.803273982999999,0,2.66,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-06-04,5,tropicana,91968,11.42919597,1,2.49,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-06-04,5,minute.maid,4416,8.392989587999999,0,2.69,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-06-04,5,dominicks,20416,9.924074186,0,1.69,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-06-11,5,tropicana,44096,10.69412435,0,2.49,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-06-11,5,dominicks,6336,8.754002933999999,0,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-06-11,5,minute.maid,5696,8.647519453,0,2.69,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-06-25,5,minute.maid,5696,8.647519453,0,2.69,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-06-25,5,tropicana,7296,8.895081532,1,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-06-25,5,dominicks,1408,7.249925537,0,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-07-02,5,tropicana,12928,9.467150781,0,2.69,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-07-02,5,minute.maid,39680,10.58860256,1,2.01,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-07-02,5,dominicks,4672,8.449342525,0,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-07-09,5,tropicana,6848,8.831711918,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-07-09,5,minute.maid,6208,8.733594062,1,2.19,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-07-09,5,dominicks,19520,9.87919486,0,1.29,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-07-16,5,tropicana,8064,8.99516499,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-07-16,5,minute.maid,7872,8.971067439,0,2.69,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-07-16,5,dominicks,7872,8.971067439,0,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-07-23,5,dominicks,5184,8.553332238,0,1.69,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-07-23,5,tropicana,4992,8.51559191,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-07-23,5,minute.maid,54528,10.90646961,1,2.29,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-07-30,5,tropicana,7360,8.903815212,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-07-30,5,minute.maid,6400,8.764053269,0,2.69,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-07-30,5,dominicks,42240,10.65112292,1,1.49,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-08-06,5,tropicana,8384,9.034080407000001,1,2.89,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-08-06,5,minute.maid,5888,8.68067166,1,2.65,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-08-06,5,dominicks,6592,8.793612072,1,1.89,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-08-13,5,tropicana,8832,9.086136769,0,2.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-08-13,5,minute.maid,56384,10.93994071,1,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-08-13,5,dominicks,2112,7.655390645,0,1.99,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1992-08-20,5,dominicks,21248,9.964018052,0,1.79,0.117368032,0.32122573,10.92237097,0.535883355,0.103091585,0.053875277,0.410568032,3.801997814,0.681818182,1.600573425,0.736306837
+1990-06-14,8,dominicks,14336,9.570529135,1,1.59,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-06-14,8,minute.maid,6080,8.712759975,0,2.62,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-06-14,8,tropicana,8896,9.093357017,0,3.19,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-06-21,8,dominicks,6400,8.764053269,0,2.29,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-06-21,8,minute.maid,51968,10.85838342,1,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-06-21,8,tropicana,7296,8.895081532,0,3.19,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-06-28,8,tropicana,10368,9.246479419,0,3.19,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-06-28,8,minute.maid,4928,8.502688505,0,2.62,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-06-28,8,dominicks,3968,8.286017467999999,0,2.29,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-07-05,8,dominicks,4352,8.378390789,0,2.29,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-07-05,8,minute.maid,5312,8.577723691000001,0,2.62,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-07-05,8,tropicana,6976,8.850230966,0,3.19,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-07-12,8,tropicana,6464,8.774003599999999,0,3.19,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-07-12,8,dominicks,3520,8.166216269,0,2.29,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-07-12,8,minute.maid,39424,10.58213005,1,2.59,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-07-19,8,tropicana,8192,9.010913347,0,3.19,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-07-19,8,dominicks,6464,8.774003599999999,0,2.29,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-07-19,8,minute.maid,5568,8.624791202,0,2.62,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-07-26,8,dominicks,5952,8.691482577,0,2.29,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-07-26,8,minute.maid,14592,9.588228712000001,0,2.62,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-07-26,8,tropicana,7936,8.979164649,0,3.19,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-08-02,8,tropicana,6656,8.803273982999999,0,3.19,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-08-02,8,minute.maid,22208,10.00820786,1,2.39,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-08-02,8,dominicks,8832,9.086136769,1,2.09,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-08-09,8,dominicks,7232,8.886270902,0,2.09,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-08-09,8,minute.maid,5760,8.658692754,0,2.62,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-08-09,8,tropicana,8256,9.018695487999999,0,3.19,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-08-16,8,tropicana,5568,8.624791202,0,3.19,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-08-16,8,minute.maid,54016,10.89703558,1,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-08-16,8,dominicks,5504,8.61323038,0,2.09,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-08-23,8,dominicks,4800,8.476371197,0,2.09,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-08-23,8,minute.maid,5824,8.66974259,0,2.62,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-08-23,8,tropicana,7488,8.921057017999999,0,3.19,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-08-30,8,tropicana,6144,8.723231275,0,3.19,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-08-30,8,minute.maid,6528,8.783855897,0,2.62,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-08-30,8,dominicks,52672,10.87183928,1,1.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-09-06,8,dominicks,16448,9.707959168,0,1.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-09-06,8,minute.maid,5440,8.60153434,0,2.62,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-09-06,8,tropicana,11008,9.30637756,0,3.19,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-09-13,8,minute.maid,36544,10.50627229,1,2.19,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-09-13,8,dominicks,19072,9.85597657,0,1.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-09-13,8,tropicana,5760,8.658692754,0,3.19,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-09-20,8,dominicks,13376,9.501217335,0,1.79,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-09-20,8,minute.maid,3776,8.236420527,0,2.62,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-09-20,8,tropicana,10112,9.221478116,0,3.19,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-09-27,8,tropicana,8448,9.041685006,0,3.19,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-09-27,8,minute.maid,5504,8.61323038,0,2.62,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-09-27,8,dominicks,61440,11.02581637,1,1.79,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-10-04,8,tropicana,8448,9.041685006,1,3.19,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-10-04,8,dominicks,13760,9.529521112000001,0,1.79,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-10-04,8,minute.maid,12416,9.426741242,0,2.62,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-10-11,8,minute.maid,53696,10.89109379,1,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-10-11,8,dominicks,3136,8.050703382,0,2.29,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-10-11,8,tropicana,7424,8.912473275,0,3.19,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-10-18,8,tropicana,5824,8.66974259,0,3.04,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-10-18,8,minute.maid,5696,8.647519453,0,2.51,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-10-18,8,dominicks,186176,12.13444774,1,1.14,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-10-25,8,tropicana,6656,8.803273982999999,0,3.04,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-10-25,8,minute.maid,4864,8.489616424,0,2.62,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-10-25,8,dominicks,3712,8.219326094,0,1.59,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-11-01,8,tropicana,6272,8.743850562,0,3.04,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-11-01,8,minute.maid,37184,10.52363384,1,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-11-01,8,dominicks,35776,10.48503256,1,1.59,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-11-08,8,tropicana,6912,8.841014311,0,3.04,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-11-08,8,minute.maid,5504,8.61323038,0,2.62,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-11-08,8,dominicks,26880,10.1991378,0,1.29,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-11-15,8,tropicana,10496,9.258749511,0,3.19,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-11-15,8,minute.maid,51008,10.83973776,1,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-11-15,8,dominicks,71680,11.17996705,0,0.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-11-22,8,tropicana,11840,9.379238908,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-11-22,8,minute.maid,11072,9.312174678,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-11-22,8,dominicks,25088,10.13014492,1,1.59,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-11-29,8,tropicana,9664,9.17616292,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-11-29,8,minute.maid,12160,9.405907156,0,2.62,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-11-29,8,dominicks,91456,11.42361326,1,2.29,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-12-06,8,minute.maid,30528,10.32639957,1,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-12-06,8,dominicks,23808,10.07777694,0,1.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-12-06,8,tropicana,6272,8.743850562,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-12-13,8,dominicks,89856,11.40596367,1,1.39,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-12-13,8,minute.maid,12096,9.400630097999999,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-12-13,8,tropicana,7168,8.877381955,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-12-20,8,minute.maid,16448,9.707959168,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-12-20,8,dominicks,12224,9.411156511,0,1.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-12-20,8,tropicana,29504,10.29228113,0,2.39,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-12-27,8,minute.maid,9344,9.142489705,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-12-27,8,dominicks,3776,8.236420527,0,1.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1990-12-27,8,tropicana,8704,9.071537969,0,2.39,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-01-03,8,tropicana,9280,9.135616826,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-01-03,8,minute.maid,16128,9.688312171,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-01-03,8,dominicks,13824,9.534161491,0,1.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-01-10,8,minute.maid,5376,8.589699882,0,2.17,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-01-10,8,dominicks,251072,12.43349503,1,0.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-01-10,8,tropicana,12224,9.411156511,0,2.59,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-01-17,8,minute.maid,6656,8.803273982999999,0,2.17,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-01-17,8,tropicana,10368,9.246479419,0,2.59,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-01-17,8,dominicks,4864,8.489616424,0,1.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-01-24,8,minute.maid,59712,10.99728828,1,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-01-24,8,dominicks,10176,9.227787286,0,1.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-01-24,8,tropicana,8128,9.00307017,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-01-31,8,tropicana,5952,8.691482577,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-01-31,8,minute.maid,9856,9.195835686,0,2.17,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-01-31,8,dominicks,105344,11.56498647,1,1.49,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-02-07,8,minute.maid,6720,8.812843434,0,2.12,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-02-07,8,dominicks,33600,10.42228135,0,1.49,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-02-07,8,tropicana,21696,9.984883191,0,2.49,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-02-14,8,dominicks,4736,8.462948177000001,0,1.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-02-14,8,minute.maid,4224,8.348537825,0,2.12,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-02-14,8,tropicana,7808,8.962904128,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-02-21,8,tropicana,8128,9.00307017,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-02-21,8,minute.maid,9728,9.182763604,0,2.12,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-02-21,8,dominicks,10304,9.240287448,0,1.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-02-28,8,tropicana,7424,8.912473275,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-02-28,8,minute.maid,40320,10.604602900000001,1,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-02-28,8,dominicks,5056,8.528330936,1,1.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-03-07,8,dominicks,179968,12.10053434,1,0.94,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-03-07,8,tropicana,5952,8.691482577,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-03-07,8,minute.maid,5120,8.540909718,0,2.17,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-03-14,8,minute.maid,19264,9.865993348,0,2.17,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-03-14,8,dominicks,4992,8.51559191,0,1.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-03-14,8,tropicana,7616,8.938006577000001,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-03-21,8,tropicana,5312,8.577723691000001,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-03-21,8,minute.maid,170432,12.04609167,1,1.69,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-03-21,8,dominicks,6400,8.764053269,0,1.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-03-28,8,minute.maid,39680,10.58860256,0,1.69,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-03-28,8,dominicks,14912,9.609921537,1,1.59,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-03-28,8,tropicana,161792,11.99406684,1,1.49,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-04-04,8,dominicks,34624,10.45230236,0,1.59,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-04-04,8,minute.maid,8128,9.00307017,1,2.17,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-04-04,8,tropicana,17280,9.757305042,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-04-11,8,tropicana,47040,10.75875358,1,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-04-11,8,minute.maid,9088,9.114710141,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-04-11,8,dominicks,10368,9.246479419,0,1.59,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-04-18,8,tropicana,14464,9.579418083,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-04-18,8,minute.maid,6720,8.812843434,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-04-18,8,dominicks,194880,12.18013926,1,0.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-04-25,8,tropicana,52928,10.87668778,1,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-04-25,8,dominicks,5696,8.647519453,1,1.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-04-25,8,minute.maid,7552,8.929567707999999,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-05-02,8,dominicks,7168,8.877381955,0,1.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-05-02,8,minute.maid,24768,10.11730778,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-05-02,8,tropicana,21184,9.961001459,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-05-09,8,tropicana,7360,8.903815212,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-05-09,8,minute.maid,183296,12.11885761,1,1.39,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-05-09,8,dominicks,2880,7.965545572999999,0,1.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-05-16,8,dominicks,12288,9.416378455,0,1.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-05-16,8,minute.maid,8896,9.093357017,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-05-16,8,tropicana,15744,9.664214619,1,2.29,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-06-06,8,dominicks,9280,9.135616826,0,1.69,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-06-06,8,tropicana,46912,10.75602879,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-06-06,8,minute.maid,6656,8.803273982999999,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-06-13,8,tropicana,18240,9.811372264,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-06-13,8,dominicks,25856,10.16029796,1,1.26,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-06-13,8,minute.maid,35456,10.47604777,1,1.49,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-06-20,8,dominicks,19264,9.865993348,0,1.29,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-06-20,8,minute.maid,17408,9.76468515,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-06-20,8,tropicana,6464,8.774003599999999,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-06-27,8,dominicks,6848,8.831711918,0,1.69,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-06-27,8,minute.maid,75520,11.2321528,1,1.69,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-06-27,8,tropicana,8512,9.049232212,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-07-04,8,tropicana,28416,10.25470765,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-07-04,8,minute.maid,21632,9.981928979,0,1.69,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-07-04,8,dominicks,12928,9.467150781,0,1.69,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-07-11,8,dominicks,44032,10.69267192,1,1.59,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-07-11,8,minute.maid,8384,9.034080407000001,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-07-11,8,tropicana,16960,9.738612909,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-07-18,8,minute.maid,9920,9.2023082,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-07-18,8,dominicks,25408,10.14281936,0,1.59,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-07-18,8,tropicana,8320,9.026417534,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-07-25,8,dominicks,38336,10.55414468,0,1.69,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-07-25,8,minute.maid,6592,8.793612072,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-07-25,8,tropicana,11136,9.317938383,1,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-08-01,8,tropicana,27712,10.22962081,0,2.19,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-08-01,8,minute.maid,7168,8.877381955,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-08-01,8,dominicks,152384,11.93415893,1,0.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-08-08,8,dominicks,54464,10.90529521,0,0.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-08-08,8,minute.maid,6208,8.733594062,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-08-08,8,tropicana,7744,8.954673629,0,2.19,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-08-15,8,minute.maid,30528,10.32639957,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-08-15,8,dominicks,47680,10.772267300000001,1,1.49,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-08-15,8,tropicana,5184,8.553332238,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-08-22,8,dominicks,14720,9.596962392,0,1.49,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-08-22,8,minute.maid,155840,11.95658512,1,1.29,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-08-22,8,tropicana,6272,8.743850562,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-08-29,8,tropicana,7744,8.954673629,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-08-29,8,dominicks,53248,10.88271552,0,1.39,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-08-29,8,minute.maid,10752,9.282847063,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-09-05,8,tropicana,53184,10.88151288,1,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-09-05,8,minute.maid,6976,8.850230966,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-09-05,8,dominicks,40576,10.61093204,0,1.39,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-09-12,8,dominicks,25856,10.16029796,0,1.39,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-09-12,8,tropicana,6784,8.822322178,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-09-12,8,minute.maid,31872,10.36948316,1,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-09-19,8,dominicks,24064,10.08847223,1,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-09-19,8,minute.maid,5312,8.577723691000001,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-09-19,8,tropicana,8000,8.987196821,1,2.49,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-09-26,8,tropicana,6592,8.793612072,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-09-26,8,minute.maid,33344,10.41463313,0,1.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-09-26,8,dominicks,15680,9.660141293999999,0,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-10-03,8,minute.maid,13504,9.510741217,0,1.79,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-10-03,8,dominicks,16576,9.715711145,0,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-10-03,8,tropicana,5248,8.565602331000001,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-10-10,8,dominicks,49664,10.8130356,1,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-10-10,8,tropicana,6592,8.793612072,0,2.94,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-10-10,8,minute.maid,13504,9.510741217,0,1.79,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-10-17,8,dominicks,10752,9.282847063,0,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-10-17,8,minute.maid,335808,12.72429485,1,1.69,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-10-17,8,tropicana,5888,8.68067166,0,2.94,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-10-24,8,tropicana,6336,8.754002933999999,0,2.94,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-10-24,8,dominicks,9792,9.189321005,0,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-10-24,8,minute.maid,13120,9.481893063,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-10-31,8,tropicana,5888,8.68067166,0,2.94,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-10-31,8,minute.maid,49664,10.8130356,0,1.49,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-10-31,8,dominicks,7104,8.868413285,0,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-11-07,8,dominicks,9216,9.128696383,1,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-11-07,8,tropicana,6080,8.712759975,0,2.94,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-11-07,8,minute.maid,10880,9.29468152,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-11-14,8,tropicana,6848,8.831711918,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-11-14,8,minute.maid,9984,9.208739091,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-11-14,8,dominicks,12608,9.442086812000001,0,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-11-21,8,tropicana,54016,10.89703558,1,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-11-21,8,minute.maid,9216,9.128696383,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-11-21,8,dominicks,16448,9.707959168,0,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-11-28,8,tropicana,10368,9.246479419,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-11-28,8,dominicks,27968,10.23881628,1,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-11-28,8,minute.maid,7680,8.946374826,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-12-05,8,minute.maid,7296,8.895081532,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-12-05,8,dominicks,37824,10.5406991,0,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-12-05,8,tropicana,5568,8.624791202,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-12-12,8,dominicks,33664,10.4241843,1,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-12-12,8,minute.maid,8192,9.010913347,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-12-12,8,tropicana,4864,8.489616424,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-12-19,8,tropicana,7232,8.886270902,0,2.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-12-19,8,minute.maid,6080,8.712759975,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-12-19,8,dominicks,17728,9.78290059,0,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-12-26,8,tropicana,15232,9.631153757,0,2.39,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-12-26,8,dominicks,25088,10.13014492,1,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1991-12-26,8,minute.maid,15040,9.618468598,1,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-01-02,8,minute.maid,9472,9.156095357,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-01-02,8,dominicks,13184,9.486759252,0,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-01-02,8,tropicana,47040,10.75875358,0,2.39,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-01-09,8,dominicks,3136,8.050703382,0,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-01-09,8,minute.maid,5888,8.68067166,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-01-09,8,tropicana,9280,9.135616826,0,2.29,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-01-16,8,tropicana,6720,8.812843434,0,2.29,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-01-16,8,minute.maid,14336,9.570529135,1,2.49,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-01-16,8,dominicks,5696,8.647519453,0,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-01-23,8,minute.maid,11712,9.368369236,1,2.49,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-01-23,8,dominicks,19008,9.852615222,0,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-01-23,8,tropicana,5056,8.528330936,0,2.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-01-30,8,minute.maid,7936,8.979164649,0,2.49,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-01-30,8,dominicks,121664,11.70901843,0,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-01-30,8,tropicana,6080,8.712759975,0,2.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-02-06,8,tropicana,10496,9.258749511,0,2.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-02-06,8,minute.maid,5184,8.553332238,0,2.39,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-02-06,8,dominicks,38848,10.56741187,0,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-02-13,8,minute.maid,7168,8.877381955,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-02-13,8,dominicks,6144,8.723231275,0,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-02-13,8,tropicana,39040,10.57234204,1,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-02-20,8,dominicks,13632,9.520175249,0,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-02-20,8,minute.maid,216064,12.28332994,1,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-02-20,8,tropicana,4480,8.407378325,0,2.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-02-27,8,tropicana,61760,11.03101119,1,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-02-27,8,minute.maid,15040,9.618468598,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-02-27,8,dominicks,9792,9.189321005,0,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-03-05,8,tropicana,15360,9.639522007,0,1.79,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-03-05,8,minute.maid,11840,9.379238908,0,2.39,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-03-05,8,dominicks,86912,11.37265139,1,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-03-12,8,minute.maid,25472,10.14533509,1,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-03-12,8,dominicks,24512,10.10691807,0,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-03-12,8,tropicana,54976,10.91465201,0,1.79,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-03-19,8,minute.maid,16384,9.704060528,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-03-19,8,dominicks,58048,10.96902553,0,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-03-19,8,tropicana,34368,10.44488118,0,1.79,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-03-26,8,tropicana,10752,9.282847063,0,2.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-03-26,8,minute.maid,20480,9.927204079,0,2.39,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-03-26,8,dominicks,13952,9.543378146,0,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-04-02,8,minute.maid,34688,10.45414909,1,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-04-02,8,dominicks,15168,9.626943225,0,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-04-02,8,tropicana,20096,9.908276069,1,2.5,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-04-09,8,dominicks,14592,9.588228712000001,0,1.58,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-04-09,8,minute.maid,22400,10.01681624,0,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-04-09,8,tropicana,16192,9.692272572,0,2.5,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-04-16,8,tropicana,6528,8.783855897,0,2.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-04-16,8,minute.maid,7808,8.962904128,0,2.39,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-04-16,8,dominicks,145088,11.88509573,1,1.29,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-04-23,8,tropicana,8320,9.026417534,0,2.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-04-23,8,minute.maid,48064,10.78028874,1,1.79,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-04-23,8,dominicks,43712,10.68537794,0,1.29,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-04-30,8,tropicana,30784,10.33475035,1,2.16,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-04-30,8,minute.maid,7360,8.903815212,0,2.39,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-04-30,8,dominicks,20608,9.933434629,0,1.69,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-05-07,8,tropicana,18048,9.800790154,0,2.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-05-07,8,minute.maid,6272,8.743850562,0,2.39,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-05-07,8,dominicks,18752,9.839055692,0,1.69,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-05-14,8,tropicana,12864,9.462187991,0,2.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-05-14,8,minute.maid,6400,8.764053269,0,2.39,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-05-14,8,dominicks,20160,9.911455722000001,0,1.79,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-05-21,8,tropicana,7168,8.877381955,0,2.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-05-21,8,minute.maid,54592,10.90764263,1,1.79,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-05-21,8,dominicks,18688,9.835636886,0,1.69,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-05-28,8,minute.maid,8128,9.00307017,0,2.39,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-05-28,8,tropicana,9024,9.107642974,0,2.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-05-28,8,dominicks,133824,11.80428078,0,1.69,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-06-04,8,tropicana,84992,11.35031241,1,2.49,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-06-04,8,minute.maid,4928,8.502688505,0,2.49,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-06-04,8,dominicks,63488,11.05860619,0,1.69,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-06-11,8,minute.maid,5440,8.60153434,0,2.49,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-06-11,8,tropicana,14144,9.557045785,0,2.49,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-06-11,8,dominicks,71040,11.17099838,0,1.79,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-06-25,8,tropicana,7488,8.921057017999999,1,2.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-06-25,8,minute.maid,5888,8.68067166,0,2.49,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-06-25,8,dominicks,15360,9.639522007,0,1.79,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-07-02,8,minute.maid,23872,10.0804615,1,2.02,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-07-02,8,dominicks,17728,9.78290059,0,1.79,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-07-02,8,tropicana,12352,9.421573272,0,2.69,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-07-09,8,tropicana,5696,8.647519453,0,2.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-07-09,8,minute.maid,6848,8.831711918,1,2.19,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-07-09,8,dominicks,24256,10.09641929,0,1.29,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-07-16,8,minute.maid,8192,9.010913347,0,2.49,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-07-16,8,dominicks,19968,9.901886271,0,1.79,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-07-16,8,tropicana,7680,8.946374826,0,2.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-07-23,8,dominicks,15936,9.67633598,0,1.69,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-07-23,8,minute.maid,55040,10.91581547,1,2.29,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-07-23,8,tropicana,5440,8.60153434,0,2.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-07-30,8,tropicana,5632,8.636219898,0,2.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-07-30,8,minute.maid,6528,8.783855897,0,2.49,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-07-30,8,dominicks,76352,11.24310951,1,1.49,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-08-06,8,tropicana,8960,9.100525506,1,2.79,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-08-06,8,minute.maid,6208,8.733594062,1,2.45,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-08-06,8,dominicks,17408,9.76468515,1,1.69,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-08-13,8,minute.maid,94720,11.45868045,1,1.99,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-08-13,8,tropicana,6080,8.712759975,0,2.89,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-08-13,8,dominicks,17536,9.77201119,0,1.79,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947
+1992-08-20,8,dominicks,31232,10.34919849,0,1.59,0.252394035,0.095173274,10.59700966,0.054227156,0.131749698,0.035243328,0.283074736,2.636332801,1.5,2.905384316,0.641015947

how-to-use-azureml/automated-machine-learning/forecasting-grouping/deploy/deploy.py

@@ -0,0 +1,66 @@
+import argparse
+import json
+
+from azureml.core import Run, Model, Workspace
+from azureml.core.conda_dependencies import CondaDependencies
+from azureml.core.model import InferenceConfig
+from azureml.core.webservice import AciWebservice
+
+
+script_file_name = 'score.py'
+conda_env_file_name = 'myenv.yml'
+
+print("In deploy.py")
+parser = argparse.ArgumentParser()
+parser.add_argument("--time_column_name", type=str, help="time column name")
+parser.add_argument("--group_column_names", type=str, help="group column names")
+parser.add_argument("--model_names", type=str, help="model names")
+parser.add_argument("--service_name", type=str, help="service name")
+
+args = parser.parse_args()
+
+# replace the group column names in scoring script to the ones set by user
+print("Update group_column_names")
+print(args.group_column_names)
+
+with open(script_file_name, 'r') as cefr:
+    content = cefr.read()
+with open(script_file_name, 'w') as cefw:
+    content = content.replace('<<groups>>', args.group_column_names.rstrip())
+    cefw.write(content.replace('<<time_colname>>', args.time_column_name.rstrip()))
+
+with open(script_file_name, 'r') as cefr1:
+    content1 = cefr1.read()
+print(content1)
+
+model_list = json.loads(args.model_names)
+print(model_list)
+
+run = Run.get_context()
+ws = run.experiment.workspace
+
+deployment_config = AciWebservice.deploy_configuration(
+    cpu_cores=1,
+    memory_gb=2,
+    tags={"method": "grouping"},
+    description='grouping demo aci deployment'
+)
+
+inference_config = InferenceConfig(
+    entry_script=script_file_name,
+    runtime='python',
+    conda_file=conda_env_file_name
+)
+
+models = []
+for model_name in model_list:
+    models.append(Model(ws, name=model_name))
+
+service = Model.deploy(
+    ws,
+    name=args.service_name,
+    models=models,
+    inference_config=inference_config,
+    deployment_config=deployment_config
+)
+service.wait_for_deployment(True)

how-to-use-azureml/automated-machine-learning/forecasting-grouping/deploy/myenv.yml

@@ -0,0 +1,11 @@
+name: automl_grouping_env
+dependencies:
+  # The python interpreter version.
+
+  # Currently Azure ML only supports 3.5.2 and later.
+
+- python=3.6.2
+- numpy>=1.16.0,<=1.16.2
+- scikit-learn>=0.19.0,<=0.20.3
+- conda-forge::fbprophet==0.5
+

how-to-use-azureml/automated-machine-learning/forecasting-grouping/deploy/score.py

@@ -0,0 +1,55 @@
+import json
+import pickle
+import re
+
+import numpy as np
+import pandas as pd
+from sklearn.externals import joblib
+from sklearn.linear_model import Ridge
+
+from azureml.core.model import Model
+import azureml.train.automl
+
+
+def init():
+    global models
+    models = {}
+    global group_columns_str
+    group_columns_str = "<<groups>>"
+    global time_column_name
+    time_column_name = "<<time_colname>>"
+
+    global group_columns
+    group_columns = group_columns_str.split("#####")
+    global valid_chars
+    valid_chars = re.compile('[^a-zA-Z0-9-]')
+
+
+def run(raw_data):
+    try:
+        data = pd.read_json(raw_data)
+        # Make sure we have correct time points.
+        data[time_column_name] = pd.to_datetime(data[time_column_name], unit='ms')
+        dfs = []
+        for grain, df_one in data.groupby(group_columns):
+            if isinstance(grain, int):
+                cur_group = str(grain)
+            elif isinstance(grain, str):
+                cur_group = grain
+            else:
+                cur_group = "#####".join(list(grain))
+            cur_group = valid_chars.sub('', cur_group)
+            print("Query model for group {}".format(cur_group))
+            if cur_group not in models:
+                model_path = Model.get_model_path(cur_group)
+                model = joblib.load(model_path)
+                models[cur_group] = model
+            _, xtrans = models[cur_group].forecast(df_one)
+            dfs.append(xtrans)
+        df_ret = pd.concat(dfs)
+        df_ret.reset_index(drop=False, inplace=True)
+        return json.dumps({'predictions': df_ret.to_json()})
+
+    except Exception as e:
+        error = str(e)
+        return error

how-to-use-azureml/automated-machine-learning/forecasting-grouping/register/register.py

@@ -0,0 +1,22 @@
+import argparse
+
+from azureml.core import Run, Model
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--model_name")
+parser.add_argument("--model_path")
+
+args = parser.parse_args()
+
+run = Run.get_context()
+ws = run.experiment.workspace
+print('retrieved ws: {}'.format(ws))
+
+print('begin register model')
+model = Model.register(
+    workspace=ws,
+    model_path=args.model_path,
+    model_name=args.model_name
+)
+print('model registered: {}'.format(model))
+print('complete')

how-to-use-azureml/automated-machine-learning/forecasting-high-frequency/automl-forecasting-function.ipynb

@@ -0,0 +1,748 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# Automated Machine Learning\n",
+        "\n",
+        "#### Forecasting away from training data\n",
+        "\n",
+        "\n",
+        "## Contents\n",
+        "1. [Introduction](#Introduction)\n",
+        "2. [Setup](#Setup)\n",
+        "3. [Data](#Data)\n",
+        "4. [Prepare remote compute and data.](#prepare_remote)\n",
+        "4. [Create the configuration and train a forecaster](#train)\n",
+        "5. [Forecasting from the trained model](#forecasting)\n",
+        "6. [Forecasting away from training data](#forecasting_away)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Introduction\n",
+        "This notebook demonstrates the full interface to the `forecast()` function. \n",
+        "\n",
+        "The best known and most frequent usage of `forecast` enables forecasting on test sets that immediately follows training data. \n",
+        "\n",
+        "However, in many use cases it is necessary to continue using the model for some time before retraining it. This happens especially in **high frequency forecasting** when forecasts need to be made more frequently than the model can be retrained. Examples are in Internet of Things and predictive cloud resource scaling.\n",
+        "\n",
+        "Here we show how to use the `forecast()` function when a time gap exists between training data and prediction period.\n",
+        "\n",
+        "Terminology:\n",
+        "* forecast origin: the last period when the target value is known\n",
+        "* forecast periods(s): the period(s) for which the value of the target is desired.\n",
+        "* forecast horizon: the number of forecast periods\n",
+        "* lookback: how many past periods (before forecast origin) the model function depends on. The larger of number of lags and length of rolling window.\n",
+        "* prediction context: `lookback` periods immediately preceding the forecast origin\n",
+        "\n",
+        "![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/automated-machine-learning/automl-forecasting-function.png)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Please make sure you have followed the `configuration.ipynb` notebook so that your ML workspace information is saved in the config file."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "import pandas as pd\n",
+        "import numpy as np\n",
+        "import logging\n",
+        "import warnings\n",
+        "\n",
+        "from azureml.core.dataset import Dataset\n",
+        "from pandas.tseries.frequencies import to_offset\n",
+        "from azureml.core.compute import AmlCompute\n",
+        "from azureml.core.compute import ComputeTarget\n",
+        "from azureml.core.runconfig import RunConfiguration\n",
+        "from azureml.core.conda_dependencies import CondaDependencies\n",
+        "\n",
+        "# Squash warning messages for cleaner output in the notebook\n",
+        "warnings.showwarning = lambda *args, **kwargs: None\n",
+        "\n",
+        "np.set_printoptions(precision=4, suppress=True, linewidth=120)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import azureml.core\n",
+        "from azureml.core.workspace import Workspace\n",
+        "from azureml.core.experiment import Experiment\n",
+        "from azureml.train.automl import AutoMLConfig\n",
+        "\n",
+        "ws = Workspace.from_config()\n",
+        "\n",
+        "# choose a name for the run history container in the workspace\n",
+        "experiment_name = 'automl-forecast-function-demo'\n",
+        "\n",
+        "experiment = Experiment(ws, experiment_name)\n",
+        "\n",
+        "output = {}\n",
+        "output['SDK version'] = azureml.core.VERSION\n",
+        "output['Subscription ID'] = ws.subscription_id\n",
+        "output['Workspace'] = ws.name\n",
+        "output['SKU'] = ws.sku\n",
+        "output['Resource Group'] = ws.resource_group\n",
+        "output['Location'] = ws.location\n",
+        "output['Run History Name'] = experiment_name\n",
+        "pd.set_option('display.max_colwidth', -1)\n",
+        "outputDf = pd.DataFrame(data = output, index = [''])\n",
+        "outputDf.T"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Data\n",
+        "For the demonstration purposes we will generate the data artificially and use them for the forecasting."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "TIME_COLUMN_NAME = 'date'\n",
+        "GRAIN_COLUMN_NAME = 'grain'\n",
+        "TARGET_COLUMN_NAME = 'y'\n",
+        "\n",
+        "def get_timeseries(train_len: int,\n",
+        "                   test_len: int,\n",
+        "                   time_column_name: str,\n",
+        "                   target_column_name: str,\n",
+        "                   grain_column_name: str,\n",
+        "                   grains: int = 1,\n",
+        "                   freq: str = 'H'):\n",
+        "    \"\"\"\n",
+        "    Return the time series of designed length.\n",
+        "\n",
+        "    :param train_len: The length of training data (one series).\n",
+        "    :type train_len: int\n",
+        "    :param test_len: The length of testing data (one series).\n",
+        "    :type test_len: int\n",
+        "    :param time_column_name: The desired name of a time column.\n",
+        "    :type time_column_name: str\n",
+        "    :param\n",
+        "    :param grains: The number of grains.\n",
+        "    :type grains: int\n",
+        "    :param freq: The frequency string representing pandas offset.\n",
+        "                 see https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html\n",
+        "    :type freq: str\n",
+        "    :returns: the tuple of train and test data sets.\n",
+        "    :rtype: tuple\n",
+        "\n",
+        "    \"\"\"\n",
+        "    data_train = []  # type: List[pd.DataFrame]\n",
+        "    data_test = []  # type: List[pd.DataFrame]\n",
+        "    data_length = train_len + test_len\n",
+        "    for i in range(grains):\n",
+        "        X = pd.DataFrame({\n",
+        "            time_column_name: pd.date_range(start='2000-01-01',\n",
+        "                                            periods=data_length,\n",
+        "                                            freq=freq),\n",
+        "            target_column_name: np.arange(data_length).astype(float) + np.random.rand(data_length) + i*5,\n",
+        "            'ext_predictor': np.asarray(range(42, 42 + data_length)),\n",
+        "            grain_column_name: np.repeat('g{}'.format(i), data_length)\n",
+        "        })\n",
+        "        data_train.append(X[:train_len])\n",
+        "        data_test.append(X[train_len:])\n",
+        "    X_train = pd.concat(data_train)\n",
+        "    y_train = X_train.pop(target_column_name).values\n",
+        "    X_test = pd.concat(data_test)\n",
+        "    y_test = X_test.pop(target_column_name).values\n",
+        "    return X_train, y_train, X_test, y_test\n",
+        "\n",
+        "n_test_periods = 6\n",
+        "n_train_periods = 30\n",
+        "X_train, y_train, X_test, y_test = get_timeseries(train_len=n_train_periods,\n",
+        "                                                  test_len=n_test_periods,\n",
+        "                                                  time_column_name=TIME_COLUMN_NAME,\n",
+        "                                                  target_column_name=TARGET_COLUMN_NAME,\n",
+        "                                                  grain_column_name=GRAIN_COLUMN_NAME,\n",
+        "                                                  grains=2)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Let's see what the training data looks like."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "X_train.tail()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# plot the example time series\n",
+        "import matplotlib.pyplot as plt\n",
+        "whole_data = X_train.copy()\n",
+        "target_label = 'y'\n",
+        "whole_data[target_label] = y_train\n",
+        "for g in whole_data.groupby('grain'):    \n",
+        "    plt.plot(g[1]['date'].values, g[1]['y'].values, label=g[0])\n",
+        "plt.legend()\n",
+        "plt.show()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Prepare remote compute and data. <a id=\"prepare_remote\"></a>\n",
+        "The [Machine Learning service workspace](https://docs.microsoft.com/en-us/azure/machine-learning/service/concept-workspace), is paired with the storage account, which contains the default data store. We will use it to upload the artificial data and create [tabular dataset](https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.data.tabulardataset?view=azure-ml-py) for training. A tabular dataset defines a series of lazily-evaluated, immutable operations to load data from the data source into tabular representation."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# We need to save thw artificial data and then upload them to default workspace datastore.\n",
+        "DATA_PATH = \"fc_fn_data\"\n",
+        "DATA_PATH_X = \"{}/data_train.csv\".format(DATA_PATH)\n",
+        "if not os.path.isdir('data'):\n",
+        "    os.mkdir('data')\n",
+        "pd.DataFrame(whole_data).to_csv(\"data/data_train.csv\", index=False)\n",
+        "# Upload saved data to the default data store.\n",
+        "ds = ws.get_default_datastore()\n",
+        "ds.upload(src_dir='./data', target_path=DATA_PATH, overwrite=True, show_progress=True)\n",
+        "train_data = Dataset.Tabular.from_delimited_files(path=ds.path(DATA_PATH_X))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "You will need to create a [compute target](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-set-up-training-targets#amlcompute) for your AutoML run. In this tutorial, you create AmlCompute as your training compute resource."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "amlcompute_cluster_name = \"cpu-cluster-fcfn\"\n",
+        "        \n",
+        "found = False\n",
+        "# Check if this compute target already exists in the workspace.\n",
+        "cts = ws.compute_targets\n",
+        "if amlcompute_cluster_name in cts and cts[amlcompute_cluster_name].type == 'AmlCompute':\n",
+        "    found = True\n",
+        "    print('Found existing compute target.')\n",
+        "    compute_target = cts[amlcompute_cluster_name]\n",
+        "\n",
+        "if not found:\n",
+        "    print('Creating a new compute target...')\n",
+        "    provisioning_config = AmlCompute.provisioning_configuration(vm_size = \"STANDARD_D2_V2\", # for GPU, use \"STANDARD_NC6\"\n",
+        "                                                                #vm_priority = 'lowpriority', # optional\n",
+        "                                                                max_nodes = 6)\n",
+        "\n",
+        "    # Create the cluster.\\n\",\n",
+        "    compute_target = ComputeTarget.create(ws, amlcompute_cluster_name, provisioning_config)\n",
+        "\n",
+        "print('Checking cluster status...')\n",
+        "# Can poll for a minimum number of nodes and for a specific timeout.\n",
+        "# If no min_node_count is provided, it will use the scale settings for the cluster.\n",
+        "compute_target.wait_for_completion(show_output = True, min_node_count = None, timeout_in_minutes = 20)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Create the configuration and train a forecaster <a id=\"train\"></a>\n",
+        "First generate the configuration, in which we:\n",
+        "* Set metadata columns: target, time column and grain column names.\n",
+        "* Validate our data using cross validation with rolling window method.\n",
+        "* Set normalized root mean squared error as a metric to select the best model.\n",
+        "* Set early termination to True, so the iterations through the models will stop when no improvements in accuracy score will be made.\n",
+        "* Set limitations on the length of experiment run to 15 minutes.\n",
+        "* Finally, we set the task to be forecasting.\n",
+        "* We apply the lag lead operator to the target value i.e. we use the previous values as a predictor for the future ones."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "lags = [1,2,3]\n",
+        "max_horizon = n_test_periods\n",
+        "time_series_settings = {    \n",
+        "    'time_column_name': TIME_COLUMN_NAME,\n",
+        "    'grain_column_names': [ GRAIN_COLUMN_NAME ],\n",
+        "    'max_horizon': max_horizon,\n",
+        "    'target_lags': lags\n",
+        "}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Run the model selection and training process."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.workspace import Workspace\n",
+        "from azureml.core.experiment import Experiment\n",
+        "from azureml.train.automl import AutoMLConfig\n",
+        "\n",
+        "\n",
+        "automl_config = AutoMLConfig(task='forecasting',\n",
+        "                             debug_log='automl_forecasting_function.log',\n",
+        "                             primary_metric='normalized_root_mean_squared_error',\n",
+        "                             experiment_timeout_minutes=15,\n",
+        "                             enable_early_stopping=True,\n",
+        "                             training_data=train_data,\n",
+        "                             compute_target=compute_target,\n",
+        "                             n_cross_validations=3,\n",
+        "                             verbosity = logging.INFO,\n",
+        "                             max_concurrent_iterations=4,\n",
+        "                             max_cores_per_iteration=-1,\n",
+        "                             label_column_name=target_label,\n",
+        "                             **time_series_settings)\n",
+        "\n",
+        "remote_run = experiment.submit(automl_config, show_output=False)\n",
+        "remote_run.wait_for_completion()\n",
+        "\n",
+        "# Retrieve the best model to use it further.\n",
+        "_, fitted_model = remote_run.get_output()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Forecasting from the trained model <a id=\"forecasting\"></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "In this section we will review the `forecast` interface for two main scenarios: forecasting right after the training data, and the more complex interface for forecasting when there is a gap (in the time sense) between training and testing data."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### X_train is directly followed by the X_test\n",
+        "\n",
+        "Let's first consider the case when the prediction period immediately follows the training data. This is typical in scenarios where we have the time to retrain the model every time we wish to forecast. Forecasts that are made on daily and slower cadence typically fall into this category. Retraining the model every time benefits the accuracy because the most recent data is often the most informative.\n",
+        "\n",
+        "![Forecasting after training](forecast_function_at_train.png)\n",
+        "\n",
+        "We use `X_test` as a **forecast request** to generate the predictions."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Typical path: X_test is known, forecast all upcoming periods"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# The data set contains hourly data, the training set ends at 01/02/2000 at 05:00\n",
+        "\n",
+        "# These are predictions we are asking the model to make (does not contain thet target column y),\n",
+        "# for 6 periods beginning with 2000-01-02 06:00, which immediately follows the training data\n",
+        "X_test"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "y_pred_no_gap, xy_nogap =  fitted_model.forecast(X_test)\n",
+        "\n",
+        "# xy_nogap contains the predictions in the _automl_target_col column.\n",
+        "# Those same numbers are output in y_pred_no_gap\n",
+        "xy_nogap"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Confidence intervals"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Forecasting model may be used for the prediction of forecasting intervals by running ```forecast_quantiles()```. \n",
+        "This method accepts the same parameters as forecast()."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "quantiles =  fitted_model.forecast_quantiles(X_test)\n",
+        "quantiles"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Distribution forecasts\n",
+        "\n",
+        "Often the figure of interest is not just the point prediction, but the prediction at some quantile of the distribution. \n",
+        "This arises when the forecast is used to control some kind of inventory, for example of grocery items or virtual machines for a cloud service. In such case, the control point is usually something like \"we want the item to be in stock and not run out 99% of the time\". This is called a \"service level\". Here is how you get quantile forecasts."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# specify which quantiles you would like \n",
+        "fitted_model.quantiles = [0.01, 0.5, 0.95]\n",
+        "# use forecast_quantiles function, not the forecast() one\n",
+        "y_pred_quantiles =  fitted_model.forecast_quantiles(X_test)\n",
+        "\n",
+        "# it all nicely aligns column-wise\n",
+        "pd.concat([X_test.reset_index(), y_pred_quantiles], axis=1)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Destination-date forecast: \"just do something\"\n",
+        "\n",
+        "In some scenarios, the X_test is not known. The forecast is likely to be weak, because it is missing contemporaneous predictors, which we will need to impute. If you still wish to predict forward under the assumption that the last known values will be carried forward, you can forecast out to \"destination date\". The destination date still needs to fit within the maximum horizon from training."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# We will take the destination date as a last date in the test set.\n",
+        "dest = max(X_test[TIME_COLUMN_NAME])\n",
+        "y_pred_dest, xy_dest = fitted_model.forecast(forecast_destination=dest)\n",
+        "\n",
+        "# This form also shows how we imputed the predictors which were not given. (Not so well! Use with caution!)\n",
+        "xy_dest"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Forecasting away from training data <a id=\"forecasting_away\"></a>\n",
+        "\n",
+        "Suppose we trained a model, some time passed, and now we want to apply the model without re-training. If the model \"looks back\" -- uses previous values of the target -- then we somehow need to provide those values to the model.\n",
+        "\n",
+        "![Forecasting after training](forecast_function_away_from_train.png)\n",
+        "\n",
+        "The notion of forecast origin comes into play: the forecast origin is **the last period for which we have seen the target value**. This applies per grain, so each grain can have a different forecast origin. \n",
+        "\n",
+        "The part of data before the forecast origin is the **prediction context**. To provide the context values the model needs when it looks back, we pass definite values in `y_test` (aligned with corresponding times in `X_test`)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# generate the same kind of test data we trained on, \n",
+        "# but now make the train set much longer, so that the test set will be in the future\n",
+        "X_context, y_context, X_away, y_away = get_timeseries(train_len=42, # train data was 30 steps long\n",
+        "                                      test_len=4,\n",
+        "                                      time_column_name=TIME_COLUMN_NAME,\n",
+        "                                      target_column_name=TARGET_COLUMN_NAME,\n",
+        "                                      grain_column_name=GRAIN_COLUMN_NAME,\n",
+        "                                      grains=2)\n",
+        "\n",
+        "# end of the data we trained on\n",
+        "print(X_train.groupby(GRAIN_COLUMN_NAME)[TIME_COLUMN_NAME].max())\n",
+        "# start of the data we want to predict on\n",
+        "print(X_away.groupby(GRAIN_COLUMN_NAME)[TIME_COLUMN_NAME].min())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "There is a gap of 12 hours between end of training and beginning of `X_away`. (It looks like 13 because all timestamps point to the start of the one hour periods.) Using only `X_away` will fail without adding context data for the model to consume."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "try: \n",
+        "    y_pred_away, xy_away = fitted_model.forecast(X_away)\n",
+        "    xy_away\n",
+        "except Exception as e:\n",
+        "    print(e)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "How should we read that eror message? The forecast origin is at the last time the model saw an actual value of `y` (the target). That was at the end of the training data! The model is attempting to forecast from the end of training data. But the requested forecast periods are past the maximum horizon. We need to provide a define `y` value to establish the forecast origin.\n",
+        "\n",
+        "We will use this helper function to take the required amount of context from the data preceding the testing data. It's definition is intentionally simplified to keep the idea in the clear."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "def make_forecasting_query(fulldata, time_column_name, target_column_name, forecast_origin, horizon, lookback):\n",
+        "\n",
+        "    \"\"\"\n",
+        "    This function will take the full dataset, and create the query\n",
+        "    to predict all values of the grain from the `forecast_origin`\n",
+        "    forward for the next `horizon` horizons. Context from previous\n",
+        "    `lookback` periods will be included.\n",
+        "\n",
+        "    \n",
+        "\n",
+        "    fulldata: pandas.DataFrame           a time series dataset. Needs to contain X and y.\n",
+        "    time_column_name: string             which column (must be in fulldata) is the time axis\n",
+        "    target_column_name: string           which column (must be in fulldata) is to be forecast\n",
+        "    forecast_origin: datetime type       the last time we (pretend to) have target values \n",
+        "    horizon: timedelta                   how far forward, in time units (not periods)\n",
+        "    lookback: timedelta                  how far back does the model look?\n",
+        "\n",
+        "    Example:\n",
+        "\n",
+        "\n",
+        "    ```\n",
+        "\n",
+        "    forecast_origin = pd.to_datetime('2012-09-01') + pd.DateOffset(days=5) # forecast 5 days after end of training\n",
+        "    print(forecast_origin)\n",
+        "\n",
+        "    X_query, y_query = make_forecasting_query(data, \n",
+        "                       forecast_origin = forecast_origin,\n",
+        "                       horizon = pd.DateOffset(days=7), # 7 days into the future\n",
+        "                       lookback = pd.DateOffset(days=1), # model has lag 1 period (day)\n",
+        "                      )\n",
+        "\n",
+        "    ```\n",
+        "    \"\"\"\n",
+        "\n",
+        "    X_past = fulldata[ (fulldata[ time_column_name ] > forecast_origin - lookback) &\n",
+        "                       (fulldata[ time_column_name ] <= forecast_origin)\n",
+        "                     ]\n",
+        "\n",
+        "    X_future = fulldata[ (fulldata[ time_column_name ] > forecast_origin) &\n",
+        "                         (fulldata[ time_column_name ] <= forecast_origin + horizon)\n",
+        "                       ]\n",
+        "\n",
+        "    y_past = X_past.pop(target_column_name).values.astype(np.float)\n",
+        "    y_future = X_future.pop(target_column_name).values.astype(np.float)\n",
+        "\n",
+        "    # Now take y_future and turn it into question marks\n",
+        "    y_query = y_future.copy().astype(np.float)  # because sometimes life hands you an int\n",
+        "    y_query.fill(np.NaN)\n",
+        "\n",
+        "\n",
+        "    print(\"X_past is \" + str(X_past.shape) + \" - shaped\")\n",
+        "    print(\"X_future is \" + str(X_future.shape) + \" - shaped\")\n",
+        "    print(\"y_past is \" + str(y_past.shape) + \" - shaped\")\n",
+        "    print(\"y_query is \" + str(y_query.shape) + \" - shaped\")\n",
+        "\n",
+        "\n",
+        "    X_pred = pd.concat([X_past, X_future])\n",
+        "    y_pred = np.concatenate([y_past, y_query])\n",
+        "    return X_pred, y_pred"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Let's see where the context data ends - it ends, by construction, just before the testing data starts."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "print(X_context.groupby(GRAIN_COLUMN_NAME)[TIME_COLUMN_NAME].agg(['min','max','count']))\n",
+        "print(X_away.groupby(GRAIN_COLUMN_NAME)[TIME_COLUMN_NAME].agg(['min','max','count']))\n",
+        "X_context.tail(5)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Since the length of the lookback is 3, \n",
+        "# we need to add 3 periods from the context to the request\n",
+        "# so that the model has the data it needs\n",
+        "\n",
+        "# Put the X and y back together for a while. \n",
+        "# They like each other and it makes them happy.\n",
+        "X_context[TARGET_COLUMN_NAME] = y_context\n",
+        "X_away[TARGET_COLUMN_NAME] = y_away\n",
+        "fulldata = pd.concat([X_context, X_away])\n",
+        "\n",
+        "# forecast origin is the last point of data, which is one 1-hr period before test\n",
+        "forecast_origin = X_away[TIME_COLUMN_NAME].min() - pd.DateOffset(hours=1)\n",
+        "# it is indeed the last point of the context\n",
+        "assert forecast_origin == X_context[TIME_COLUMN_NAME].max()\n",
+        "print(\"Forecast origin: \" + str(forecast_origin))\n",
+        "      \n",
+        "# the model uses lags and rolling windows to look back in time\n",
+        "n_lookback_periods = max(lags)\n",
+        "lookback = pd.DateOffset(hours=n_lookback_periods)\n",
+        "\n",
+        "horizon = pd.DateOffset(hours=max_horizon)\n",
+        "\n",
+        "# now make the forecast query from context (refer to figure)\n",
+        "X_pred, y_pred = make_forecasting_query(fulldata, TIME_COLUMN_NAME, TARGET_COLUMN_NAME,\n",
+        "                                        forecast_origin, horizon, lookback)\n",
+        "\n",
+        "# show the forecast request aligned\n",
+        "X_show = X_pred.copy()\n",
+        "X_show[TARGET_COLUMN_NAME] = y_pred\n",
+        "X_show"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Note that the forecast origin is at 17:00 for both grains, and periods from 18:00 are to be forecast."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Now everything works\n",
+        "y_pred_away, xy_away = fitted_model.forecast(X_pred, y_pred)\n",
+        "\n",
+        "# show the forecast aligned\n",
+        "X_show = xy_away.reset_index()\n",
+        "# without the generated features\n",
+        "X_show[['date', 'grain', 'ext_predictor', '_automl_target_col']]\n",
+        "# prediction is in _automl_target_col"
+      ]
+    }
+  ],
+  "metadata": {
+    "authors": [
+      {
+        "name": "erwright, nirovins"
+      }
+    ],
+    "category": "tutorial",
+    "compute": [
+      "Remote"
+    ],
+    "datasets": [
+      "None"
+    ],
+    "deployment": [
+      "None"
+    ],
+    "exclude_from_index": false,
+    "framework": [
+      "Azure ML AutoML"
+    ],
+    "friendly_name": "Forecasting away from training data",
+    "index_order": 3,
+    "kernelspec": {
+      "display_name": "Python 3.6",
+      "language": "python",
+      "name": "python36"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.6.8"
+    },
+    "tags": [
+      "Forecasting",
+      "Confidence Intervals"
+    ],
+    "task": "Forecasting"
+  },
+  "nbformat": 4,
+  "nbformat_minor": 2
+}

how-to-use-azureml/automated-machine-learning/forecasting-high-frequency/automl-forecasting-function.yml

@@ -0,0 +1,11 @@
+name: automl-forecasting-function
+dependencies:
+- fbprophet==0.5
+- py-xgboost<=0.80
+- pip:
+  - azureml-sdk
+  - azureml-train-automl
+  - azureml-widgets
+  - pandas_ml
+  - statsmodels
+  - matplotlib

how-to-use-azureml/automated-machine-learning/forecasting-orange-juice-sales/auto-ml-forecasting-orange-juice-sales.ipynb

@@ -9,6 +9,13 @@
         "Licensed under the MIT License."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/automated-machine-learning/forecasting-orange-juice-sales/auto-ml-forecasting-orange-juice-sales.png)"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {},
@@ -19,8 +26,11 @@
         "## Contents\n",
         "1. [Introduction](#Introduction)\n",
         "1. [Setup](#Setup)\n",
+        "1. [Compute](#Compute)\n",
         "1. [Data](#Data)\n",
-        "1. [Train](#Train)"
+        "1. [Train](#Train)\n",
+        "1. [Predict](#Predict)\n",
+        "1. [Operationalize](#Operationalize)"
       ]
     },
     {
@@ -28,16 +38,10 @@
       "metadata": {},
       "source": [
         "## Introduction\n",
-        "In this example, we use AutoML to find and tune a time-series forecasting model.\n",
+        "In this example, we use AutoML to train, select, and operationalize a time-series forecasting model for multiple time-series.\n",
         "\n",
         "Make sure you have executed the [configuration notebook](../../../configuration.ipynb) before running this notebook.\n",
         "\n",
-        "In this notebook, you will:\n",
-        "1. Create an Experiment in an existing Workspace\n",
-        "2. Instantiate an AutoMLConfig \n",
-        "3. Find and train a forecasting model using local compute\n",
-        "4. Evaluate the performance of the model\n",
-        "\n",
         "The examples in the follow code samples use the University of Chicago's Dominick's Finer Foods dataset to forecast orange juice sales. Dominick's was a grocery chain in the Chicago metropolitan area."
       ]
     },
@@ -58,22 +62,17 @@
         "import pandas as pd\n",
         "import numpy as np\n",
         "import logging\n",
-        "import warnings\n",
-        "# Squash warning messages for cleaner output in the notebook\n",
-        "warnings.showwarning = lambda *args, **kwargs: None\n",
-        "\n",
         "\n",
         "from azureml.core.workspace import Workspace\n",
         "from azureml.core.experiment import Experiment\n",
-        "from azureml.train.automl import AutoMLConfig\n",
-        "from sklearn.metrics import mean_absolute_error, mean_squared_error"
+        "from azureml.train.automl import AutoMLConfig"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "As part of the setup you have already created a <b>Workspace</b>. To run AutoML, you also need to create an <b>Experiment</b>. An Experiment is a named object in a Workspace which represents a predictive task, the output of which is a trained model and a set of evaluation metrics for the model. "
+        "As part of the setup you have already created a <b>Workspace</b>. To run AutoML, you also need to create an <b>Experiment</b>. An Experiment corresponds to a prediction problem you are trying to solve, while a Run corresponds to a specific approach to the problem. "
       ]
     },
     {
@@ -85,9 +84,7 @@
         "ws = Workspace.from_config()\n",
         "\n",
         "# choose a name for the run history container in the workspace\n",
-        "experiment_name = 'automl-ojsalesforecasting'\n",
-        "# project folder\n",
-        "project_folder = './sample_projects/automl-local-ojsalesforecasting'\n",
+        "experiment_name = 'automl-ojforecasting'\n",
         "\n",
         "experiment = Experiment(ws, experiment_name)\n",
         "\n",
@@ -95,15 +92,63 @@
         "output['SDK version'] = azureml.core.VERSION\n",
         "output['Subscription ID'] = ws.subscription_id\n",
         "output['Workspace'] = ws.name\n",
+        "output['SKU'] = ws.sku\n",
         "output['Resource Group'] = ws.resource_group\n",
         "output['Location'] = ws.location\n",
-        "output['Project Directory'] = project_folder\n",
         "output['Run History Name'] = experiment_name\n",
         "pd.set_option('display.max_colwidth', -1)\n",
         "outputDf = pd.DataFrame(data = output, index = [''])\n",
         "outputDf.T"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Compute\n",
+        "You will need to create a [compute target](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-set-up-training-targets#amlcompute) for your AutoML run. In this tutorial, you create AmlCompute as your training compute resource.\n",
+        "#### Creation of AmlCompute takes approximately 5 minutes. \n",
+        "If the AmlCompute with that name is already in your workspace this code will skip the creation process.\n",
+        "As with other Azure services, there are limits on certain resources (e.g. AmlCompute) associated with the Azure Machine Learning service. Please read this article on the default limits and how to request more quota."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.compute import AmlCompute\n",
+        "from azureml.core.compute import ComputeTarget\n",
+        "\n",
+        "# Choose a name for your cluster.\n",
+        "amlcompute_cluster_name = \"cpu-cluster-oj\"\n",
+        "\n",
+        "found = False\n",
+        "# Check if this compute target already exists in the workspace.\n",
+        "cts = ws.compute_targets\n",
+        "if amlcompute_cluster_name in cts and cts[amlcompute_cluster_name].type == 'AmlCompute':\n",
+        "    found = True\n",
+        "    print('Found existing compute target.')\n",
+        "    compute_target = cts[amlcompute_cluster_name]\n",
+        "    \n",
+        "if not found:\n",
+        "    print('Creating a new compute target...')\n",
+        "    provisioning_config = AmlCompute.provisioning_configuration(vm_size = \"STANDARD_D2_V2\", # for GPU, use \"STANDARD_NC6\"\n",
+        "                                                                #vm_priority = 'lowpriority', # optional\n",
+        "                                                                max_nodes = 6)\n",
+        "\n",
+        "    # Create the cluster.\n",
+        "    compute_target = ComputeTarget.create(ws, amlcompute_cluster_name, provisioning_config)\n",
+        "    \n",
+        "print('Checking cluster status...')\n",
+        "# Can poll for a minimum number of nodes and for a specific timeout.\n",
+        "# If no min_node_count is provided, it will use the scale settings for the cluster.\n",
+        "compute_target.wait_for_completion(show_output = True, min_node_count = None, timeout_in_minutes = 20)\n",
+        "    \n",
+        "# For a more detailed view of current AmlCompute status, use get_status()."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {},
@@ -186,7 +231,61 @@
         "    df_tail = df_grouped.apply(lambda dfg: dfg.iloc[-n:])\n",
         "    return df_head, df_tail\n",
         "\n",
-        "X_train, X_test = split_last_n_by_grain(data_subset, n_test_periods)"
+        "train, test = split_last_n_by_grain(data_subset, n_test_periods)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Upload data to datastore\n",
+        "The [Machine Learning service workspace](https://docs.microsoft.com/en-us/azure/machine-learning/service/concept-workspace), is paired with the storage account, which contains the default data store. We will use it to upload the train and test data and create [tabular datasets](https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.data.tabulardataset?view=azure-ml-py) for training and testing. A tabular dataset defines a series of lazily-evaluated, immutable operations to load data from the data source into tabular representation."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "train.to_csv (r'./dominicks_OJ_train.csv', index = None, header=True)\n",
+        "test.to_csv (r'./dominicks_OJ_test.csv', index = None, header=True)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "datastore = ws.get_default_datastore()\n",
+        "datastore.upload_files(files = ['./dominicks_OJ_train.csv', './dominicks_OJ_test.csv'], target_path = 'dataset/', overwrite = True,show_progress = True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Create dataset for training"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.dataset import Dataset\n",
+        "train_dataset = Dataset.Tabular.from_delimited_files(path=datastore.path('dataset/dominicks_OJ_train.csv'))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "train_dataset.to_pandas_dataframe().tail()"
       ]
     },
     {
@@ -202,7 +301,7 @@
         "* Create time-based features to assist in learning seasonal patterns\n",
         "* Encode categorical variables to numeric quantities\n",
         "\n",
-        "AutoML will currently train a single, regression-type model across **all** time-series in a given training set. This allows the model to generalize across related series.\n",
+        "In this notebook, AutoML will train a single, regression-type model across **all** time-series in a given training set. This allows the model to generalize across related series. If you're looking for training multiple models for different time-series, please check out the forecasting grouping notebook. \n",
         "\n",
         "You are almost ready to start an AutoML training job. First, we need to separate the target column from the rest of the DataFrame: "
       ]
@@ -213,8 +312,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "target_column_name = 'Quantity'\n",
-        "y_train = X_train.pop(target_column_name).values"
+        "target_column_name = 'Quantity'"
       ]
     },
     {
@@ -227,9 +325,9 @@
         "\n",
         "For forecasting tasks, there are some additional parameters that can be set: the name of the column holding the date/time, the grain column names, and the maximum forecast horizon. A time column is required for forecasting, while the grain is optional. If a grain is not given, AutoML assumes that the whole dataset is a single time-series. We also pass a list of columns to drop prior to modeling. The _logQuantity_ column is completely correlated with the target quantity, so it must be removed to prevent a target leak.\n",
         "\n",
-        "The forecast horizon is given in units of the time-series frequency; for instance, the OJ series frequency is weekly, so a horizon of 20 means that a trained model will estimate sales up-to 20 weeks beyond the latest date in the training data for each series. In this example, we set the maximum horizon to the number of samples per series in the test set (n_test_periods). Generally, the value of this parameter will be dictated by business needs. For example, a demand planning organizaion that needs to estimate the next month of sales would set the horizon accordingly.   \n",
+        "The forecast horizon is given in units of the time-series frequency; for instance, the OJ series frequency is weekly, so a horizon of 20 means that a trained model will estimate sales up to 20 weeks beyond the latest date in the training data for each series. In this example, we set the maximum horizon to the number of samples per series in the test set (n_test_periods). Generally, the value of this parameter will be dictated by business needs. For example, a demand planning organizaion that needs to estimate the next month of sales would set the horizon accordingly. Please see the [energy_demand notebook](https://github.com/Azure/MachineLearningNotebooks/tree/master/how-to-use-azureml/automated-machine-learning/forecasting-energy-demand) for more discussion of forecast horizon.\n",
         "\n",
-        "Finally, a note about the cross-validation (CV) procedure for time-series data. AutoML uses out-of-sample error estimates to select a best pipeline/model, so it is important that the CV fold splitting is done correctly. Time-series can violate the basic statistical assumptions of the canonical K-Fold CV strategy, so AutoML implements a [rolling origin validation](https://robjhyndman.com/hyndsight/tscv/) procedure to create CV folds for time-series data. To use this procedure, you just need to specify the desired number of CV folds in the AutoMLConfig object. It is also possible to bypass CV and use your own validation set by setting the *X_valid* and *y_valid* parameters of AutoMLConfig.\n",
+        "Finally, a note about the cross-validation (CV) procedure for time-series data. AutoML uses out-of-sample error estimates to select a best pipeline/model, so it is important that the CV fold splitting is done correctly. Time-series can violate the basic statistical assumptions of the canonical K-Fold CV strategy, so AutoML implements a [rolling origin validation](https://robjhyndman.com/hyndsight/tscv/) procedure to create CV folds for time-series data. To use this procedure, you just need to specify the desired number of CV folds in the AutoMLConfig object. It is also possible to bypass CV and use your own validation set by setting the *validation_data* parameter of AutoMLConfig.\n",
         "\n",
         "Here is a summary of AutoMLConfig parameters used for training the OJ model:\n",
         "\n",
@@ -237,13 +335,15 @@
         "|-|-|\n",
         "|**task**|forecasting|\n",
         "|**primary_metric**|This is the metric that you want to optimize.<br> Forecasting supports the following primary metrics <br><i>spearman_correlation</i><br><i>normalized_root_mean_squared_error</i><br><i>r2_score</i><br><i>normalized_mean_absolute_error</i>\n",
-        "|**iterations**|Number of iterations. In each iteration, Auto ML trains a specific pipeline on the given data|\n",
-        "|**X**|Training matrix of features as a pandas DataFrame, shape = [n_training_samples, n_features]|\n",
-        "|**y**|Target values as a numpy.ndarray, shape = [n_training_samples, ]|\n",
+        "|**experiment_timeout_minutes**|Experimentation timeout in minutes.|\n",
+        "|**enable_early_stopping**|If early stopping is on, training will stop when the primary metric is no longer improving.|\n",
+        "|**training_data**|Input dataset, containing both features and label column.|\n",
+        "|**label_column_name**|The name of the label column.|\n",
+        "|**compute_target**|The remote compute for training.|\n",
         "|**n_cross_validations**|Number of cross-validation folds to use for model/pipeline selection|\n",
-        "|**enable_ensembling**|Allow AutoML to create ensembles of the best performing models\n",
-        "|**debug_log**|Log file path for writing debugging information\n",
-        "|**path**|Relative path to the project folder.  AutoML stores configuration files for the experiment under this folder. You can specify a new empty folder.|\n",
+        "|**enable_voting_ensemble**|Allow AutoML to create a Voting ensemble of the best performing models|\n",
+        "|**enable_stack_ensemble**|Allow AutoML to create a Stack ensemble of the best performing models|\n",
+        "|**debug_log**|Log file path for writing debugging information|\n",
         "|**time_column_name**|Name of the datetime column in the input data|\n",
         "|**grain_column_names**|Name(s) of the columns defining individual series in the input data|\n",
         "|**drop_column_names**|Name(s) of columns to drop prior to modeling|\n",
@@ -259,19 +359,19 @@
         "time_series_settings = {\n",
         "    'time_column_name': time_column_name,\n",
         "    'grain_column_names': grain_column_names,\n",
-        "    'drop_column_names': ['logQuantity'],\n",
+        "    'drop_column_names': ['logQuantity'],  # 'logQuantity' is a leaky feature, so we remove it.\n",
         "    'max_horizon': n_test_periods\n",
         "}\n",
         "\n",
         "automl_config = AutoMLConfig(task='forecasting',\n",
         "                             debug_log='automl_oj_sales_errors.log',\n",
-        "                             primary_metric='normalized_root_mean_squared_error',\n",
-        "                             iterations=10,\n",
-        "                             X=X_train,\n",
-        "                             y=y_train,\n",
-        "                             n_cross_validations=5,\n",
-        "                             enable_ensembling=False,\n",
-        "                             path=project_folder,\n",
+        "                             primary_metric='normalized_mean_absolute_error',\n",
+        "                             experiment_timeout_minutes=15,\n",
+        "                             training_data=train_dataset,\n",
+        "                             label_column_name=target_column_name,\n",
+        "                             compute_target=compute_target,\n",
+        "                             enable_early_stopping=True,\n",
+        "                             n_cross_validations=3,\n",
         "                             verbosity=logging.INFO,\n",
         "                             **time_series_settings)"
       ]
@@ -280,7 +380,7 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "You can now submit a new training run. For local runs, the execution is synchronous. Depending on the data and number of iterations this operation may take several minutes.\n",
+        "You can now submit a new training run. Depending on the data and number of iterations this operation may take several minutes.\n",
         "Information from each iteration will be printed to the console."
       ]
     },
@@ -290,7 +390,8 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "local_run = experiment.submit(automl_config, show_output=True)"
+        "remote_run = experiment.submit(automl_config, show_output=False)\n",
+        "remote_run"
       ]
     },
     {
@@ -299,7 +400,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "local_run"
+        "remote_run.wait_for_completion()"
       ]
     },
     {
@@ -316,15 +417,17 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "best_run, fitted_pipeline = local_run.get_output()\n",
-        "fitted_pipeline.steps"
+        "best_run, fitted_model = remote_run.get_output()\n",
+        "print(fitted_model.steps)\n",
+        "model_name = best_run.properties['model_name']"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "### Make Predictions from the Best Fitted Model\n",
+        "# Forecasting\n",
+        "\n",
         "Now that we have retrieved the best pipeline/model, it can be used to make predictions on test data. First, we remove the target values from the test set:"
       ]
     },
@@ -334,6 +437,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
+        "X_test = test\n",
         "y_test = X_test.pop(target_column_name).values"
       ]
     },
@@ -350,9 +454,7 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "To produce predictions on the test set, we need to know the feature values at all dates in the test set. This requirement is somewhat reasonable for the OJ sales data since the features mainly consist of price, which is usually set in advance, and customer demographics which are approximately constant for each store over the 20 week forecast horizon in the testing data. \n",
-        "\n",
-        "The target predictions can be retrieved by calling the `predict` method on the best model:"
+        "To produce predictions on the test set, we need to know the feature values at all dates in the test set. This requirement is somewhat reasonable for the OJ sales data since the features mainly consist of price, which is usually set in advance, and customer demographics which are approximately constant for each store over the 20 week forecast horizon in the testing data."
       ]
     },
     {
@@ -361,15 +463,30 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "y_pred = fitted_pipeline.predict(X_test)"
+        "# The featurized data, aligned to y, will also be returned.\n",
+        "# This contains the assumptions that were made in the forecast\n",
+        "# and helps align the forecast to the original data\n",
+        "y_predictions, X_trans = fitted_model.forecast(X_test)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "### Calculate evaluation metrics for the prediction\n",
-        "To evaluate the accuracy of the forecast, we'll compare against the actual sales quantities for some select metrics, included the mean absolute percentage error (MAPE)."
+        "If you are used to scikit pipelines, perhaps you expected `predict(X_test)`. However, forecasting requires a more general interface that also supplies the past target `y` values. Please use `forecast(X,y)` as `predict(X)` is reserved for internal purposes on forecasting models.\n",
+        "\n",
+        "The [energy demand forecasting notebook](https://github.com/Azure/MachineLearningNotebooks/tree/master/how-to-use-azureml/automated-machine-learning/forecasting-energy-demand) demonstrates the use of the forecast function in more detail in the context of using lags and rolling window features. "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# Evaluate\n",
+        "\n",
+        "To evaluate the accuracy of the forecast, we'll compare against the actual sales quantities for some select metrics, included the mean absolute percentage error (MAPE). \n",
+        "\n",
+        "It is a good practice to always align the output explicitly to the input, as the count and order of the rows may have changed during transformations that span multiple rows."
       ]
     },
     {
@@ -378,21 +495,185 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "def MAPE(actual, pred):\n",
-        "    \"\"\"\n",
-        "    Calculate mean absolute percentage error.\n",
-        "    Remove NA and values where actual is close to zero\n",
-        "    \"\"\"\n",
-        "    not_na = ~(np.isnan(actual) | np.isnan(pred))\n",
-        "    not_zero = ~np.isclose(actual, 0.0)\n",
-        "    actual_safe = actual[not_na & not_zero]\n",
-        "    pred_safe = pred[not_na & not_zero]\n",
-        "    APE = 100*np.abs((actual_safe - pred_safe)/actual_safe)\n",
-        "    return np.mean(APE)\n",
+        "from forecasting_helper import align_outputs\n",
         "\n",
-        "print(\"[Test Data] \\nRoot Mean squared error: %.2f\" % np.sqrt(mean_squared_error(y_test, y_pred)))\n",
-        "print('mean_absolute_error score: %.2f' % mean_absolute_error(y_test, y_pred))\n",
-        "print('MAPE: %.2f' % MAPE(y_test, y_pred))"
+        "df_all = align_outputs(y_predictions, X_trans, X_test, y_test, target_column_name)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.automl.core._vendor.automl.client.core.common import metrics\n",
+        "from matplotlib import pyplot as plt\n",
+        "from automl.client.core.common import constants\n",
+        "\n",
+        "# use automl metrics module\n",
+        "scores = metrics.compute_metrics_regression(\n",
+        "    df_all['predicted'],\n",
+        "    df_all[target_column_name],\n",
+        "    list(constants.Metric.SCALAR_REGRESSION_SET),\n",
+        "    None, None, None)\n",
+        "\n",
+        "print(\"[Test data scores]\\n\")\n",
+        "for key, value in scores.items():    \n",
+        "    print('{}:   {:.3f}'.format(key, value))\n",
+        "    \n",
+        "# Plot outputs\n",
+        "%matplotlib inline\n",
+        "test_pred = plt.scatter(df_all[target_column_name], df_all['predicted'], color='b')\n",
+        "test_test = plt.scatter(df_all[target_column_name], df_all[target_column_name], color='g')\n",
+        "plt.legend((test_pred, test_test), ('prediction', 'truth'), loc='upper left', fontsize=8)\n",
+        "plt.show()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# Operationalize"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "_Operationalization_ means getting the model into the cloud so that other can run it after you close the notebook. We will create a docker running on Azure Container Instances with the model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "description = 'AutoML OJ forecaster'\n",
+        "tags = None\n",
+        "model = remote_run.register_model(model_name = model_name, description = description, tags = tags)\n",
+        "\n",
+        "print(remote_run.model_id)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Develop the scoring script\n",
+        "\n",
+        "For the deployment we need a function which will run the forecast on serialized data. It can be obtained from the best_run."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "script_file_name = 'score_fcast.py'\n",
+        "best_run.download_file('outputs/scoring_file_v_1_0_0.py', script_file_name)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Deploy the model as a Web Service on Azure Container Instance"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.model import InferenceConfig\n",
+        "from azureml.core.webservice import AciWebservice\n",
+        "from azureml.core.webservice import Webservice\n",
+        "from azureml.core.model import Model\n",
+        "\n",
+        "inference_config = InferenceConfig(environment = best_run.get_environment(), \n",
+        "                                   entry_script = script_file_name)\n",
+        "\n",
+        "aciconfig = AciWebservice.deploy_configuration(cpu_cores = 1, \n",
+        "                                               memory_gb = 2, \n",
+        "                                               tags = {'type': \"automl-forecasting\"},\n",
+        "                                               description = \"Automl forecasting sample service\")\n",
+        "\n",
+        "aci_service_name = 'automl-oj-forecast-01'\n",
+        "print(aci_service_name)\n",
+        "aci_service = Model.deploy(ws, aci_service_name, [model], inference_config, aciconfig)\n",
+        "aci_service.wait_for_deployment(True)\n",
+        "print(aci_service.state)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "aci_service.get_logs()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Call the service"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import json\n",
+        "# The request data frame needs to have y_query column which corresponds to query.\n",
+        "X_query = X_test.copy()\n",
+        "X_query['y_query'] = np.NaN\n",
+        "# We have to convert datetime to string, because Timestamps cannot be serialized to JSON.\n",
+        "X_query[time_column_name] = X_query[time_column_name].astype(str)\n",
+        "# The Service object accept the complex dictionary, which is internally converted to JSON string.\n",
+        "# The section 'data' contains the data frame in the form of dictionary.\n",
+        "test_sample = json.dumps({'data': X_query.to_dict(orient='records')})\n",
+        "response = aci_service.run(input_data = test_sample)\n",
+        "# translate from networkese to datascientese\n",
+        "try: \n",
+        "    res_dict = json.loads(response)\n",
+        "    y_fcst_all = pd.DataFrame(res_dict['index'])\n",
+        "    y_fcst_all[time_column_name] = pd.to_datetime(y_fcst_all[time_column_name], unit = 'ms')\n",
+        "    y_fcst_all['forecast'] = res_dict['forecast']    \n",
+        "except:\n",
+        "    print(res_dict)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "y_fcst_all.head()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Delete the web service if desired"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "serv = Webservice(ws, 'automl-oj-forecast-01')\n",
+        "serv.delete()     # don't do it accidentally"
       ]
     }
   ],
@@ -402,6 +683,23 @@
         "name": "erwright"
       }
     ],
+    "category": "tutorial",
+    "celltoolbar": "Raw Cell Format",
+    "compute": [
+      "Remote"
+    ],
+    "datasets": [
+      "Orange Juice Sales"
+    ],
+    "deployment": [
+      "Azure Container Instance"
+    ],
+    "exclude_from_index": false,
+    "framework": [
+      "Azure ML AutoML"
+    ],
+    "friendly_name": "Forecasting orange juice sales with deployment",
+    "index_order": 1,
     "kernelspec": {
       "display_name": "Python 3.6",
       "language": "python",
@@ -418,7 +716,11 @@
       "nbconvert_exporter": "python",
       "pygments_lexer": "ipython3",
       "version": "3.6.8"
-    }
+    },
+    "tags": [
+      "None"
+    ],
+    "task": "Forecasting"
   },
   "nbformat": 4,
   "nbformat_minor": 2

how-to-use-azureml/automated-machine-learning/forecasting-orange-juice-sales/auto-ml-forecasting-orange-juice-sales.yml

@@ -0,0 +1,11 @@
+name: auto-ml-forecasting-orange-juice-sales
+dependencies:
+- fbprophet==0.5
+- py-xgboost<=0.80
+- pip:
+  - azureml-sdk
+  - azureml-train-automl
+  - azureml-widgets
+  - matplotlib
+  - pandas_ml
+  - statsmodels

how-to-use-azureml/automated-machine-learning/forecasting-orange-juice-sales/forecasting_helper.py

@@ -0,0 +1,98 @@
+import pandas as pd
+import numpy as np
+from pandas.tseries.frequencies import to_offset
+
+
+def align_outputs(y_predicted, X_trans, X_test, y_test, target_column_name,
+                  predicted_column_name='predicted',
+                  horizon_colname='horizon_origin'):
+    """
+    Demonstrates how to get the output aligned to the inputs
+    using pandas indexes. Helps understand what happened if
+    the output's shape differs from the input shape, or if
+    the data got re-sorted by time and grain during forecasting.
+
+    Typical causes of misalignment are:
+    * we predicted some periods that were missing in actuals -> drop from eval
+    * model was asked to predict past max_horizon -> increase max horizon
+    * data at start of X_test was needed for lags -> provide previous periods
+    """
+
+    if (horizon_colname in X_trans):
+        df_fcst = pd.DataFrame({predicted_column_name: y_predicted,
+                                horizon_colname: X_trans[horizon_colname]})
+    else:
+        df_fcst = pd.DataFrame({predicted_column_name: y_predicted})
+
+    # y and X outputs are aligned by forecast() function contract
+    df_fcst.index = X_trans.index
+
+    # align original X_test to y_test
+    X_test_full = X_test.copy()
+    X_test_full[target_column_name] = y_test
+
+    # X_test_full's index does not include origin, so reset for merge
+    df_fcst.reset_index(inplace=True)
+    X_test_full = X_test_full.reset_index().drop(columns='index')
+    together = df_fcst.merge(X_test_full, how='right')
+
+    # drop rows where prediction or actuals are nan
+    # happens because of missing actuals
+    # or at edges of time due to lags/rolling windows
+    clean = together[together[[target_column_name,
+                               predicted_column_name]].notnull().all(axis=1)]
+    return(clean)
+
+
+def do_rolling_forecast(fitted_model, X_test, y_test, target_column_name, time_column_name, max_horizon, freq='D'):
+    """
+    Produce forecasts on a rolling origin over the given test set.
+
+    Each iteration makes a forecast for the next 'max_horizon' periods
+    with respect to the current origin, then advances the origin by the
+    horizon time duration. The prediction context for each forecast is set so
+    that the forecaster uses the actual target values prior to the current
+    origin time for constructing lag features.
+
+    This function returns a concatenated DataFrame of rolling forecasts.
+     """
+    df_list = []
+    origin_time = X_test[time_column_name].min()
+    while origin_time <= X_test[time_column_name].max():
+        # Set the horizon time - end date of the forecast
+        horizon_time = origin_time + max_horizon * to_offset(freq)
+
+        # Extract test data from an expanding window up-to the horizon
+        expand_wind = (X_test[time_column_name] < horizon_time)
+        X_test_expand = X_test[expand_wind]
+        y_query_expand = np.zeros(len(X_test_expand)).astype(np.float)
+        y_query_expand.fill(np.NaN)
+
+        if origin_time != X_test[time_column_name].min():
+            # Set the context by including actuals up-to the origin time
+            test_context_expand_wind = (X_test[time_column_name] < origin_time)
+            context_expand_wind = (
+                X_test_expand[time_column_name] < origin_time)
+            y_query_expand[context_expand_wind] = y_test[
+                test_context_expand_wind]
+
+        # Make a forecast out to the maximum horizon
+        y_fcst, X_trans = fitted_model.forecast(X_test_expand, y_query_expand)
+
+        # Align forecast with test set for dates within the
+        # current rolling window
+        trans_tindex = X_trans.index.get_level_values(time_column_name)
+        trans_roll_wind = (trans_tindex >= origin_time) & (
+            trans_tindex < horizon_time)
+        test_roll_wind = expand_wind & (
+            X_test[time_column_name] >= origin_time)
+        df_list.append(align_outputs(y_fcst[trans_roll_wind],
+                                     X_trans[trans_roll_wind],
+                                     X_test[test_roll_wind],
+                                     y_test[test_roll_wind],
+                                     target_column_name))
+
+        # Advance the origin time
+        origin_time = horizon_time
+
+    return pd.concat(df_list, ignore_index=True)

how-to-use-azureml/automated-machine-learning/forecasting-orange-juice-sales/metrics_helper.py

@@ -0,0 +1,22 @@
+import pandas as pd
+import numpy as np
+
+
+def APE(actual, pred):
+    """
+    Calculate absolute percentage error.
+    Returns a vector of APE values with same length as actual/pred.
+    """
+    return 100 * np.abs((actual - pred) / actual)
+
+
+def MAPE(actual, pred):
+    """
+    Calculate mean absolute percentage error.
+    Remove NA and values where actual is close to zero
+    """
+    not_na = ~(np.isnan(actual) | np.isnan(pred))
+    not_zero = ~np.isclose(actual, 0.0)
+    actual_safe = actual[not_na & not_zero]
+    pred_safe = pred[not_na & not_zero]
+    return np.mean(APE(actual_safe, pred_safe))

how-to-use-azureml/automated-machine-learning/local-run-classification-credit-card-fraud/auto-ml-classification-credit-card-fraud-local.ipynb

@@ -0,0 +1,422 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
+        "\n",
+        "Licensed under the MIT License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/automated-machine-learning/classification-credit-card-fraud/auto-ml-classification-credit-card-fraud.png)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# Automated Machine Learning\n",
+        "_**Classification of credit card fraudulent transactions with local run **_\n",
+        "\n",
+        "## Contents\n",
+        "1. [Introduction](#Introduction)\n",
+        "1. [Setup](#Setup)\n",
+        "1. [Train](#Train)\n",
+        "1. [Results](#Results)\n",
+        "1. [Test](#Test)\n",
+        "1. [Acknowledgements](#Acknowledgements)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Introduction\n",
+        "\n",
+        "In this example we use the associated credit card dataset to showcase how you can use AutoML for a simple classification problem. The goal is to predict if a credit card transaction is considered a fraudulent charge.\n",
+        "\n",
+        "This notebook is using the local machine compute to train the model.\n",
+        "\n",
+        "If you are using an Azure Machine Learning [Notebook VM](https://docs.microsoft.com/en-us/azure/machine-learning/service/tutorial-1st-experiment-sdk-setup), you are all set. Otherwise, go through the [configuration](../../../configuration.ipynb) notebook first if you haven't already to establish your connection to the AzureML Workspace. \n",
+        "\n",
+        "In this notebook you will learn how to:\n",
+        "1. Create an experiment using an existing workspace.\n",
+        "2. Configure AutoML using `AutoMLConfig`.\n",
+        "3. Train the model.\n",
+        "4. Explore the results.\n",
+        "5. Test the fitted model."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Setup\n",
+        "\n",
+        "As part of the setup you have already created an Azure ML `Workspace` object. For Automated ML you will need to create an `Experiment` object, which is a named object in a `Workspace` used to run experiments."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import logging\n",
+        "\n",
+        "from matplotlib import pyplot as plt\n",
+        "import pandas as pd\n",
+        "import os\n",
+        "\n",
+        "import azureml.core\n",
+        "from azureml.core.experiment import Experiment\n",
+        "from azureml.core.workspace import Workspace\n",
+        "from azureml.core.dataset import Dataset\n",
+        "from azureml.train.automl import AutoMLConfig"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "ws = Workspace.from_config()\n",
+        "\n",
+        "# choose a name for experiment\n",
+        "experiment_name = 'automl-classification-ccard-local'\n",
+        "\n",
+        "experiment=Experiment(ws, experiment_name)\n",
+        "\n",
+        "output = {}\n",
+        "output['SDK version'] = azureml.core.VERSION\n",
+        "output['Subscription ID'] = ws.subscription_id\n",
+        "output['Workspace'] = ws.name\n",
+        "output['Resource Group'] = ws.resource_group\n",
+        "output['Location'] = ws.location\n",
+        "output['Experiment Name'] = experiment.name\n",
+        "pd.set_option('display.max_colwidth', -1)\n",
+        "outputDf = pd.DataFrame(data = output, index = [''])\n",
+        "outputDf.T"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Load Data\n",
+        "\n",
+        "Load the credit card dataset from a csv file containing both training features and labels. The features are inputs to the model, while the training labels represent the expected output of the model. Next, we'll split the data using random_split and extract the training data for the model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "data = \"https://automlsamplenotebookdata.blob.core.windows.net/automl-sample-notebook-data/creditcard.csv\"\n",
+        "dataset = Dataset.Tabular.from_delimited_files(data)\n",
+        "training_data, validation_data = dataset.random_split(percentage=0.8, seed=223)\n",
+        "label_column_name = 'Class'"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Train\n",
+        "\n",
+        "Instantiate a AutoMLConfig object. This defines the settings and data used to run the experiment.\n",
+        "\n",
+        "|Property|Description|\n",
+        "|-|-|\n",
+        "|**task**|classification or regression|\n",
+        "|**primary_metric**|This is the metric that you want to optimize. Classification supports the following primary metrics: <br><i>accuracy</i><br><i>AUC_weighted</i><br><i>average_precision_score_weighted</i><br><i>norm_macro_recall</i><br><i>precision_score_weighted</i>|\n",
+        "|**enable_early_stopping**|Stop the run if the metric score is not showing improvement.|\n",
+        "|**n_cross_validations**|Number of cross validation splits.|\n",
+        "|**training_data**|Input dataset, containing both features and label column.|\n",
+        "|**label_column_name**|The name of the label column.|\n",
+        "\n",
+        "**_You can find more information about primary metrics_** [here](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-configure-auto-train#primary-metric)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "automl_settings = {\n",
+        "    \"n_cross_validations\": 3,\n",
+        "    \"primary_metric\": 'average_precision_score_weighted',\n",
+        "    \"preprocess\": True,\n",
+        "    \"experiment_timeout_minutes\": 10, # This is a time limit for testing purposes, remove it for real use cases, this will drastically limit ablity to find the best model possible\n",
+        "    \"verbosity\": logging.INFO,\n",
+        "    \"enable_stack_ensemble\": False\n",
+        "}\n",
+        "\n",
+        "automl_config = AutoMLConfig(task = 'classification',\n",
+        "                             debug_log = 'automl_errors.log',\n",
+        "                             training_data = training_data,\n",
+        "                             label_column_name = label_column_name,\n",
+        "                             **automl_settings\n",
+        "                            )"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Call the `submit` method on the experiment object and pass the run configuration. Depending on the data and the number of iterations this can run for a while.\n",
+        "In this example, we specify `show_output = True` to print currently running iterations to the console."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "local_run = experiment.submit(automl_config, show_output = True)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# If you need to retrieve a run that already started, use the following code\n",
+        "#from azureml.train.automl.run import AutoMLRun\n",
+        "#local_run = AutoMLRun(experiment = experiment, run_id = '<replace with your run id>')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "local_run"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Results"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Widget for Monitoring Runs\n",
+        "\n",
+        "The widget will first report a \"loading\" status while running the first iteration. After completing the first iteration, an auto-updating graph and table will be shown. The widget will refresh once per minute, so you should see the graph update as child runs complete.\n",
+        "\n",
+        "**Note:** The widget displays a link at the bottom. Use this link to open a web interface to explore the individual run details"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.widgets import RunDetails\n",
+        "RunDetails(local_run).show()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Analyze results\n",
+        "\n",
+        "### Retrieve the Best Model\n",
+        "\n",
+        "Below we select the best pipeline from our iterations. The `get_output` method on `automl_classifier` returns the best run and the fitted model for the last invocation. Overloads on `get_output` allow you to retrieve the best run and fitted model for *any* logged metric or for a particular *iteration*."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "best_run, fitted_model = local_run.get_output()\n",
+        "fitted_model"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Print the properties of the model\n",
+        "The fitted_model is a python object and you can read the different properties of the object.\n",
+        "See *Print the properties of the model* section in [this sample notebook](https://github.com/Azure/MachineLearningNotebooks/blob/master/how-to-use-azureml/automated-machine-learning/classification/auto-ml-classification.ipynb)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Deploy\n",
+        "\n",
+        "To deploy the model into a web service endpoint, see _Deploy_ section in [this sample notebook](https://github.com/Azure/MachineLearningNotebooks/blob/master/how-to-use-azureml/automated-machine-learning/classification-with-deployment/auto-ml-classification-with-deployment.ipynb)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Test the fitted model\n",
+        "\n",
+        "Now that the model is trained, split the data in the same way the data was split for training (The difference here is the data is being split locally) and then run the test data through the trained model to get the predicted values."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# convert the test data to dataframe\n",
+        "X_test_df = validation_data.drop_columns(columns=[label_column_name]).to_pandas_dataframe()\n",
+        "y_test_df = validation_data.keep_columns(columns=[label_column_name], validate=True).to_pandas_dataframe()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# call the predict functions on the model\n",
+        "y_pred = fitted_model.predict(X_test_df)\n",
+        "y_pred"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Calculate metrics for the prediction\n",
+        "\n",
+        "Now visualize the data on a scatter plot to show what our truth (actual) values are compared to the predicted values \n",
+        "from the trained model that was returned."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from sklearn.metrics import confusion_matrix\n",
+        "import numpy as np\n",
+        "import itertools\n",
+        "\n",
+        "cf =confusion_matrix(y_test_df.values,y_pred)\n",
+        "plt.imshow(cf,cmap=plt.cm.Blues,interpolation='nearest')\n",
+        "plt.colorbar()\n",
+        "plt.title('Confusion Matrix')\n",
+        "plt.xlabel('Predicted')\n",
+        "plt.ylabel('Actual')\n",
+        "class_labels = ['False','True']\n",
+        "tick_marks = np.arange(len(class_labels))\n",
+        "plt.xticks(tick_marks,class_labels)\n",
+        "plt.yticks([-0.5,0,1,1.5],['','False','True',''])\n",
+        "# plotting text value inside cells\n",
+        "thresh = cf.max() / 2.\n",
+        "for i,j in itertools.product(range(cf.shape[0]),range(cf.shape[1])):\n",
+        "    plt.text(j,i,format(cf[i,j],'d'),horizontalalignment='center',color='white' if cf[i,j] >thresh else 'black')\n",
+        "plt.show()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Acknowledgements"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "This Credit Card fraud Detection dataset is made available under the Open Database License: http://opendatacommons.org/licenses/odbl/1.0/. Any rights in individual contents of the database are licensed under the Database Contents License: http://opendatacommons.org/licenses/dbcl/1.0/ and is available at: https://www.kaggle.com/mlg-ulb/creditcardfraud\n",
+        "\n",
+        "\n",
+        "The dataset has been collected and analysed during a research collaboration of Worldline and the Machine Learning Group (http://mlg.ulb.ac.be) of ULB (Universit\u00c3\u0192\u00c2\u00a9 Libre de Bruxelles) on big data mining and fraud detection. More details on current and past projects on related topics are available on https://www.researchgate.net/project/Fraud-detection-5 and the page of the DefeatFraud project\n",
+        "Please cite the following works: \n",
+        "\u00c3\u00a2\u00e2\u201a\u00ac\u00c2\u00a2\tAndrea Dal Pozzolo, Olivier Caelen, Reid A. Johnson and Gianluca Bontempi. Calibrating Probability with Undersampling for Unbalanced Classification. In Symposium on Computational Intelligence and Data Mining (CIDM), IEEE, 2015\n",
+        "\u00c3\u00a2\u00e2\u201a\u00ac\u00c2\u00a2\tDal Pozzolo, Andrea; Caelen, Olivier; Le Borgne, Yann-Ael; Waterschoot, Serge; Bontempi, Gianluca. Learned lessons in credit card fraud detection from a practitioner perspective, Expert systems with applications,41,10,4915-4928,2014, Pergamon\n",
+        "\u00c3\u00a2\u00e2\u201a\u00ac\u00c2\u00a2\tDal Pozzolo, Andrea; Boracchi, Giacomo; Caelen, Olivier; Alippi, Cesare; Bontempi, Gianluca. Credit card fraud detection: a realistic modeling and a novel learning strategy, IEEE transactions on neural networks and learning systems,29,8,3784-3797,2018,IEEE\n",
+        "o\tDal Pozzolo, Andrea Adaptive Machine learning for credit card fraud detection ULB MLG PhD thesis (supervised by G. Bontempi)\n",
+        "\u00c3\u00a2\u00e2\u201a\u00ac\u00c2\u00a2\tCarcillo, Fabrizio; Dal Pozzolo, Andrea; Le Borgne, Yann-A\u00c3\u0192\u00c2\u00abl; Caelen, Olivier; Mazzer, Yannis; Bontempi, Gianluca. Scarff: a scalable framework for streaming credit card fraud detection with Spark, Information fusion,41, 182-194,2018,Elsevier\n",
+        "\u00c3\u00a2\u00e2\u201a\u00ac\u00c2\u00a2\tCarcillo, Fabrizio; Le Borgne, Yann-A\u00c3\u0192\u00c2\u00abl; Caelen, Olivier; Bontempi, Gianluca. Streaming active learning strategies for real-life credit card fraud detection: assessment and visualization, International Journal of Data Science and Analytics, 5,4,285-300,2018,Springer International Publishing"
+      ]
+    }
+  ],
+  "metadata": {
+    "authors": [
+      {
+        "name": "tzvikei"
+      }
+    ],
+    "category": "tutorial",
+    "compute": [
+      "Local"
+    ],
+    "datasets": [
+      "creditcard"
+    ],
+    "deployment": [
+      "None"
+    ],
+    "exclude_from_index": true,
+    "file_extension": ".py",
+    "framework": [
+      "None"
+    ],
+    "friendly_name": "Classification of credit card fraudulent transactions using Automated ML",
+    "index_order": 5,
+    "kernelspec": {
+      "display_name": "Python 3.6",
+      "language": "python",
+      "name": "python36"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.6.7"
+    },
+    "mimetype": "text/x-python",
+    "name": "python",
+    "nbconvert_exporter": "python",
+    "pygments_lexer": "ipython3",
+    "tags": [
+      "local_run",
+      "AutomatedML"
+    ],
+    "task": "Classification",
+    "version": "3.6.7"
+  },
+  "nbformat": 4,
+  "nbformat_minor": 2
+}

how-to-use-azureml/automated-machine-learning/local-run-classification-credit-card-fraud/auto-ml-classification-credit-card-fraud-local.yml

@@ -0,0 +1,11 @@
+name: auto-ml-classification-credit-card-fraud-local
+dependencies:
+- pip:
+  - azureml-sdk
+  - interpret
+  - azureml-defaults
+  - azureml-explain-model
+  - azureml-train-automl
+  - azureml-widgets
+  - matplotlib
+  - pandas_ml

how-to-use-azureml/automated-machine-learning/missing-data-blacklist-early-termination/auto-ml-missing-data-blacklist-early-termination.ipynb

@@ -1,377 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
-        "\n",
-        "Licensed under the MIT License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "# Automated Machine Learning\n",
-        "_**Blacklisting Models, Early Termination, and Handling Missing Data**_\n",
-        "\n",
-        "## Contents\n",
-        "1. [Introduction](#Introduction)\n",
-        "1. [Setup](#Setup)\n",
-        "1. [Data](#Data)\n",
-        "1. [Train](#Train)\n",
-        "1. [Results](#Results)\n",
-        "1. [Test](#Test)\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Introduction\n",
-        "In this example we use the scikit-learn's [digit dataset](http://scikit-learn.org/stable/datasets/index.html#optical-recognition-of-handwritten-digits-dataset) to showcase how you can use AutoML for handling missing values in data. We also provide a stopping metric indicating a target for the primary metrics so that AutoML can terminate the run without necessarly going through all the iterations. Finally, if you want to avoid a certain pipeline, we allow you to specify a blacklist of algorithms that AutoML will ignore for this run.\n",
-        "\n",
-        "Make sure you have executed the [configuration](../../../configuration.ipynb) before running this notebook.\n",
-        "\n",
-        "In this notebook you will learn how to:\n",
-        "1. Create an `Experiment` in an existing `Workspace`.\n",
-        "2. Configure AutoML using `AutoMLConfig`.\n",
-        "4. Train the model.\n",
-        "5. Explore the results.\n",
-        "6. Test the best fitted model.\n",
-        "\n",
-        "In addition this notebook showcases the following features\n",
-        "- **Blacklisting** certain pipelines\n",
-        "- Specifying **target metrics** to indicate stopping criteria\n",
-        "- Handling **missing data** in the input"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Setup\n",
-        "\n",
-        "As part of the setup you have already created an Azure ML `Workspace` object. For AutoML you will need to create an `Experiment` object, which is a named object in a `Workspace` used to run experiments."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "import logging\n",
-        "\n",
-        "from matplotlib import pyplot as plt\n",
-        "import numpy as np\n",
-        "import pandas as pd\n",
-        "from sklearn import datasets\n",
-        "\n",
-        "import azureml.core\n",
-        "from azureml.core.experiment import Experiment\n",
-        "from azureml.core.workspace import Workspace\n",
-        "from azureml.train.automl import AutoMLConfig"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "ws = Workspace.from_config()\n",
-        "\n",
-        "# Choose a name for the experiment.\n",
-        "experiment_name = 'automl-local-missing-data'\n",
-        "project_folder = './sample_projects/automl-local-missing-data'\n",
-        "\n",
-        "experiment = Experiment(ws, experiment_name)\n",
-        "\n",
-        "output = {}\n",
-        "output['SDK version'] = azureml.core.VERSION\n",
-        "output['Subscription ID'] = ws.subscription_id\n",
-        "output['Workspace'] = ws.name\n",
-        "output['Resource Group'] = ws.resource_group\n",
-        "output['Location'] = ws.location\n",
-        "output['Project Directory'] = project_folder\n",
-        "output['Experiment Name'] = experiment.name\n",
-        "pd.set_option('display.max_colwidth', -1)\n",
-        "outputDf = pd.DataFrame(data = output, index = [''])\n",
-        "outputDf.T"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Data"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "digits = datasets.load_digits()\n",
-        "X_train = digits.data[10:,:]\n",
-        "y_train = digits.target[10:]\n",
-        "\n",
-        "# Add missing values in 75% of the lines.\n",
-        "missing_rate = 0.75\n",
-        "n_missing_samples = int(np.floor(X_train.shape[0] * missing_rate))\n",
-        "missing_samples = np.hstack((np.zeros(X_train.shape[0] - n_missing_samples, dtype=np.bool), np.ones(n_missing_samples, dtype=np.bool)))\n",
-        "rng = np.random.RandomState(0)\n",
-        "rng.shuffle(missing_samples)\n",
-        "missing_features = rng.randint(0, X_train.shape[1], n_missing_samples)\n",
-        "X_train[np.where(missing_samples)[0], missing_features] = np.nan"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "df = pd.DataFrame(data = X_train)\n",
-        "df['Label'] = pd.Series(y_train, index=df.index)\n",
-        "df.head()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Train\n",
-        "\n",
-        "Instantiate an `AutoMLConfig` object to specify the settings and data used to run the experiment.  This includes setting `experiment_exit_score`, which should cause the run to complete before the `iterations` count is reached.\n",
-        "\n",
-        "|Property|Description|\n",
-        "|-|-|\n",
-        "|**task**|classification or regression|\n",
-        "|**primary_metric**|This is the metric that you want to optimize. Classification supports the following primary metrics: <br><i>accuracy</i><br><i>AUC_weighted</i><br><i>average_precision_score_weighted</i><br><i>norm_macro_recall</i><br><i>precision_score_weighted</i>|\n",
-        "|**iteration_timeout_minutes**|Time limit in minutes for each iteration.|\n",
-        "|**iterations**|Number of iterations. In each iteration AutoML trains a specific pipeline with the data.|\n",
-        "|**preprocess**|Setting this to *True* enables AutoML to perform preprocessing on the input to handle *missing data*, and to perform some common *feature extraction*.|\n",
-        "|**experiment_exit_score**|*double* value indicating the target for *primary_metric*. <br>Once the target is surpassed the run terminates.|\n",
-        "|**blacklist_models**|*List* of *strings* indicating machine learning algorithms for AutoML to avoid in this run.<br><br> Allowed values for **Classification**<br><i>LogisticRegression</i><br><i>SGD</i><br><i>MultinomialNaiveBayes</i><br><i>BernoulliNaiveBayes</i><br><i>SVM</i><br><i>LinearSVM</i><br><i>KNN</i><br><i>DecisionTree</i><br><i>RandomForest</i><br><i>ExtremeRandomTrees</i><br><i>LightGBM</i><br><i>GradientBoosting</i><br><i>TensorFlowDNN</i><br><i>TensorFlowLinearClassifier</i><br><br>Allowed values for **Regression**<br><i>ElasticNet</i><br><i>GradientBoosting</i><br><i>DecisionTree</i><br><i>KNN</i><br><i>LassoLars</i><br><i>SGD</i><br><i>RandomForest</i><br><i>ExtremeRandomTrees</i><br><i>LightGBM</i><br><i>TensorFlowLinearRegressor</i><br><i>TensorFlowDNN</i>|\n",
-        "|**X**|(sparse) array-like, shape = [n_samples, n_features]|\n",
-        "|**y**|(sparse) array-like, shape = [n_samples, ], Multi-class targets.|\n",
-        "|**path**|Relative path to the project folder. AutoML stores configuration files for the experiment under this folder. You can specify a new empty folder.|"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "automl_config = AutoMLConfig(task = 'classification',\n",
-        "                             debug_log = 'automl_errors.log',\n",
-        "                             primary_metric = 'AUC_weighted',\n",
-        "                             iteration_timeout_minutes = 60,\n",
-        "                             iterations = 20,\n",
-        "                             preprocess = True,\n",
-        "                             experiment_exit_score = 0.9984,\n",
-        "                             blacklist_models = ['KNN','LinearSVM'],\n",
-        "                             verbosity = logging.INFO,\n",
-        "                             X = X_train, \n",
-        "                             y = y_train,\n",
-        "                             path = project_folder)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "Call the `submit` method on the experiment object and pass the run configuration. Execution of local runs is synchronous. Depending on the data and the number of iterations this can run for a while.\n",
-        "In this example, we specify `show_output = True` to print currently running iterations to the console."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "local_run = experiment.submit(automl_config, show_output = True)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "local_run"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Results"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Widget for Monitoring Runs\n",
-        "\n",
-        "The widget will first report a \"loading\" status while running the first iteration. After completing the first iteration, an auto-updating graph and table will be shown. The widget will refresh once per minute, so you should see the graph update as child runs complete.\n",
-        "\n",
-        "**Note:** The widget displays a link at the bottom. Use this link to open a web interface to explore the individual run details."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "from azureml.widgets import RunDetails\n",
-        "RunDetails(local_run).show() "
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "\n",
-        "#### Retrieve All Child Runs\n",
-        "You can also use SDK methods to fetch all the child runs and see individual metrics that we log."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "children = list(local_run.get_children())\n",
-        "metricslist = {}\n",
-        "for run in children:\n",
-        "    properties = run.get_properties()\n",
-        "    metrics = {k: v for k, v in run.get_metrics().items() if isinstance(v, float)}\n",
-        "    metricslist[int(properties['iteration'])] = metrics\n",
-        "\n",
-        "rundata = pd.DataFrame(metricslist).sort_index(1)\n",
-        "rundata"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Retrieve the Best Model\n",
-        "\n",
-        "Below we select the best pipeline from our iterations. The `get_output` method returns the best run and the fitted model. The Model includes the pipeline and any pre-processing.  Overloads on `get_output` allow you to retrieve the best run and fitted model for *any* logged metric or for a particular *iteration*."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "best_run, fitted_model = local_run.get_output()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Best Model Based on Any Other Metric\n",
-        "Show the run and the model which has the smallest `accuracy` value:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "# lookup_metric = \"accuracy\"\n",
-        "# best_run, fitted_model = local_run.get_output(metric = lookup_metric)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Model from a Specific Iteration\n",
-        "Show the run and the model from the third iteration:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "# iteration = 3\n",
-        "# best_run, fitted_model = local_run.get_output(iteration = iteration)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Test"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "digits = datasets.load_digits()\n",
-        "X_test = digits.data[:10, :]\n",
-        "y_test = digits.target[:10]\n",
-        "images = digits.images[:10]\n",
-        "\n",
-        "# Randomly select digits and test.\n",
-        "for index in np.random.choice(len(y_test), 2, replace = False):\n",
-        "    print(index)\n",
-        "    predicted = fitted_model.predict(X_test[index:index + 1])[0]\n",
-        "    label = y_test[index]\n",
-        "    title = \"Label value = %d  Predicted value = %d \" % (label, predicted)\n",
-        "    fig = plt.figure(1, figsize=(3,3))\n",
-        "    ax1 = fig.add_axes((0,0,.8,.8))\n",
-        "    ax1.set_title(title)\n",
-        "    plt.imshow(images[index], cmap = plt.cm.gray_r, interpolation = 'nearest')\n",
-        "    plt.show()\n"
-      ]
-    }
-  ],
-  "metadata": {
-    "authors": [
-      {
-        "name": "savitam"
-      }
-    ],
-    "kernelspec": {
-      "display_name": "Python 3.6",
-      "language": "python",
-      "name": "python36"
-    },
-    "language_info": {
-      "codemirror_mode": {
-        "name": "ipython",
-        "version": 3
-      },
-      "file_extension": ".py",
-      "mimetype": "text/x-python",
-      "name": "python",
-      "nbconvert_exporter": "python",
-      "pygments_lexer": "ipython3",
-      "version": "3.6.6"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 2
-}

how-to-use-azureml/automated-machine-learning/model-explanation/auto-ml-model-explanation.ipynb

@@ -1,348 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
-        "\n",
-        "Licensed under the MIT License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "# Automated Machine Learning\n",
-        "_**Explain classification model and visualize the explanation**_\n",
-        "\n",
-        "## Contents\n",
-        "1. [Introduction](#Introduction)\n",
-        "1. [Setup](#Setup)\n",
-        "1. [Data](#Data)\n",
-        "1. [Train](#Train)\n",
-        "1. [Results](#Results)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Introduction\n",
-        "In this example we use the sklearn's [iris dataset](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_iris.html) to showcase how you can use the AutoML Classifier for a simple classification problem.\n",
-        "\n",
-        "Make sure you have executed the [configuration](../../../configuration.ipynb) before running this notebook.\n",
-        "\n",
-        "In this notebook you would see\n",
-        "1. Creating an Experiment in an existing Workspace\n",
-        "2. Instantiating AutoMLConfig\n",
-        "3. Training the Model using local compute and explain the model\n",
-        "4. Visualization model's feature importance in widget\n",
-        "5. Explore best model's explanation"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Setup\n",
-        "\n",
-        "As part of the setup you have already created a <b>Workspace</b>. For AutoML you would need to create an <b>Experiment</b>. An <b>Experiment</b> is a named object in a <b>Workspace</b>, which is used to run experiments."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "import logging\n",
-        "\n",
-        "import pandas as pd\n",
-        "import azureml.core\n",
-        "from azureml.core.experiment import Experiment\n",
-        "from azureml.core.workspace import Workspace\n",
-        "from azureml.train.automl import AutoMLConfig"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "ws = Workspace.from_config()\n",
-        "\n",
-        "# choose a name for experiment\n",
-        "experiment_name = 'automl-model-explanation'\n",
-        "# project folder\n",
-        "project_folder = './sample_projects/automl-model-explanation'\n",
-        "\n",
-        "experiment=Experiment(ws, experiment_name)\n",
-        "\n",
-        "output = {}\n",
-        "output['SDK version'] = azureml.core.VERSION\n",
-        "output['Subscription ID'] = ws.subscription_id\n",
-        "output['Workspace Name'] = ws.name\n",
-        "output['Resource Group'] = ws.resource_group\n",
-        "output['Location'] = ws.location\n",
-        "output['Project Directory'] = project_folder\n",
-        "output['Experiment Name'] = experiment.name\n",
-        "pd.set_option('display.max_colwidth', -1)\n",
-        "outputDf = pd.DataFrame(data = output, index = [''])\n",
-        "outputDf.T"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Data"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "from sklearn import datasets\n",
-        "\n",
-        "iris = datasets.load_iris()\n",
-        "y = iris.target\n",
-        "X = iris.data\n",
-        "\n",
-        "features = iris.feature_names\n",
-        "\n",
-        "from sklearn.model_selection import train_test_split\n",
-        "X_train, X_test, y_train, y_test = train_test_split(X,\n",
-        "                                                    y,\n",
-        "                                                    test_size=0.1,\n",
-        "                                                    random_state=100,\n",
-        "                                                    stratify=y)\n",
-        "\n",
-        "X_train = pd.DataFrame(X_train, columns=features)\n",
-        "X_test = pd.DataFrame(X_test, columns=features)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Train\n",
-        "\n",
-        "Instantiate a AutoMLConfig object. This defines the settings and data used to run the experiment.\n",
-        "\n",
-        "|Property|Description|\n",
-        "|-|-|\n",
-        "|**task**|classification or regression|\n",
-        "|**primary_metric**|This is the metric that you want to optimize. Classification supports the following primary metrics: <br><i>accuracy</i><br><i>AUC_weighted</i><br><i>average_precision_score_weighted</i><br><i>norm_macro_recall</i><br><i>precision_score_weighted</i>|\n",
-        "|**max_time_sec**|Time limit in minutes for each iterations|\n",
-        "|**iterations**|Number of iterations. In each iteration Auto ML trains the data with a specific pipeline|\n",
-        "|**X**|(sparse) array-like, shape = [n_samples, n_features]|\n",
-        "|**y**|(sparse) array-like, shape = [n_samples, ], Multi-class targets.|\n",
-        "|**X_valid**|(sparse) array-like, shape = [n_samples, n_features]|\n",
-        "|**y_valid**|(sparse) array-like, shape = [n_samples, ], Multi-class targets.|\n",
-        "|**model_explainability**|Indicate to explain each trained pipeline or not |\n",
-        "|**path**|Relative path to the project folder.  AutoML stores configuration files for the experiment under this folder. You can specify a new empty folder. |"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "automl_config = AutoMLConfig(task = 'classification',\n",
-        "                             debug_log = 'automl_errors.log',\n",
-        "                             primary_metric = 'AUC_weighted',\n",
-        "                             iteration_timeout_minutes = 200,\n",
-        "                             iterations = 10,\n",
-        "                             verbosity = logging.INFO,\n",
-        "                             X = X_train, \n",
-        "                             y = y_train,\n",
-        "                             X_valid = X_test,\n",
-        "                             y_valid = y_test,\n",
-        "                             model_explainability=True,\n",
-        "                             path=project_folder)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "You can call the submit method on the experiment object and pass the run configuration. For Local runs the execution is synchronous. Depending on the data and number of iterations this can run for while.\n",
-        "You will see the currently running iterations printing to the console."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "local_run = experiment.submit(automl_config, show_output=True)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "local_run"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Results"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Widget for monitoring runs\n",
-        "\n",
-        "The widget will sit on \"loading\" until the first iteration completed, then you will see an auto-updating graph and table show up. It refreshed once per minute, so you should see the graph update as child runs complete.\n",
-        "\n",
-        "NOTE: The widget displays a link at the bottom. This links to a web-ui to explore the individual run details."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "from azureml.widgets import RunDetails\n",
-        "RunDetails(local_run).show() "
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Retrieve the Best Model\n",
-        "\n",
-        "Below we select the best pipeline from our iterations. The *get_output* method on automl_classifier returns the best run and the fitted model for the last *fit* invocation. There are overloads on *get_output* that allow you to retrieve the best run and fitted model for *any* logged metric or a particular *iteration*."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "best_run, fitted_model = local_run.get_output()\n",
-        "print(best_run)\n",
-        "print(fitted_model)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Best Model 's explanation\n",
-        "\n",
-        "Retrieve the explanation from the best_run. And explanation information includes:\n",
-        "\n",
-        "1.\tshap_values: The explanation information generated by shap lib\n",
-        "2.\texpected_values: The expected value of the model applied to set of X_train data.\n",
-        "3.\toverall_summary: The model level feature importance values sorted in descending order\n",
-        "4.\toverall_imp: The feature names sorted in the same order as in overall_summary\n",
-        "5.\tper_class_summary: The class level feature importance values sorted in descending order. Only available for the classification case\n",
-        "6.\tper_class_imp: The feature names sorted in the same order as in per_class_summary. Only available for the classification case"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "from azureml.train.automl.automlexplainer import retrieve_model_explanation\n",
-        "\n",
-        "shap_values, expected_values, overall_summary, overall_imp, per_class_summary, per_class_imp = \\\n",
-        "    retrieve_model_explanation(best_run)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "print(overall_summary)\n",
-        "print(overall_imp)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "print(per_class_summary)\n",
-        "print(per_class_imp)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "Beside retrieve the existed model explanation information, explain the model with different train/test data"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "from azureml.train.automl.automlexplainer import explain_model\n",
-        "\n",
-        "shap_values, expected_values, overall_summary, overall_imp, per_class_summary, per_class_imp = \\\n",
-        "    explain_model(fitted_model, X_train, X_test)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "print(overall_summary)\n",
-        "print(overall_imp)"
-      ]
-    }
-  ],
-  "metadata": {
-    "authors": [
-      {
-        "name": "xif"
-      }
-    ],
-    "kernelspec": {
-      "display_name": "Python 3.6",
-      "language": "python",
-      "name": "python36"
-    },
-    "language_info": {
-      "codemirror_mode": {
-        "name": "ipython",
-        "version": 3
-      },
-      "file_extension": ".py",
-      "mimetype": "text/x-python",
-      "name": "python",
-      "nbconvert_exporter": "python",
-      "pygments_lexer": "ipython3",
-      "version": "3.6.6"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 2
-}

how-to-use-azureml/automated-machine-learning/regression-hardware-performance-explanation-and-featurization/auto-ml-regression-hardware-performance-explanation-and-featurization.ipynb

@@ -0,0 +1,995 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
+        "\n",
+        "Licensed under the MIT License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/automated-machine-learning/regression-car-price-model-explaination-and-featurization/auto-ml-regression.png)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# Automated Machine Learning\n",
+        "_**Regression with Aml Compute**_\n",
+        "\n",
+        "## Contents\n",
+        "1. [Introduction](#Introduction)\n",
+        "1. [Setup](#Setup)\n",
+        "1. [Data](#Data)\n",
+        "1. [Train](#Train)\n",
+        "1. [Results](#Results)\n",
+        "1. [Test](#Test)\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Introduction\n",
+        "In this example we use the Hardware Performance Dataset to showcase how you can use AutoML for a simple regression problem. The Regression goal is to predict the performance of certain combinations of hardware parts.\n",
+        "After training AutoML models for this regression data set, we show how you can compute model explanations on your remote compute using a sample explainer script.\n",
+        "\n",
+        "If you are using an Azure Machine Learning Notebook VM, you are all set.  Otherwise, go through the [configuration](../../../configuration.ipynb)  notebook first if you haven't already to establish your connection to the AzureML Workspace. \n",
+        "\n",
+        "An Enterprise workspace is required for this notebook. To learn more about creating an Enterprise workspace or upgrading to an Enterprise workspace from the Azure portal, please visit our [Workspace page.](https://docs.microsoft.com/azure/machine-learning/service/concept-workspace#upgrade) \n",
+        "\n",
+        "In this notebook you will learn how to:\n",
+        "1. Create an `Experiment` in an existing `Workspace`.\n",
+        "2. Instantiating AutoMLConfig with FeaturizationConfig for customization\n",
+        "3. Train the model using remote compute.\n",
+        "4. Explore the results and featurization transparency options\n",
+        "5. Setup remote compute for computing the model explanations for a given AutoML model.\n",
+        "6. Start an AzureML experiment on your remote compute to compute explanations for an AutoML model.\n",
+        "7. Download the feature importance for engineered features and visualize the explanations for engineered features. \n",
+        "8. Download the feature importance for raw features and visualize the explanations for raw features. \n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Setup\n",
+        "\n",
+        "As part of the setup you have already created an Azure ML `Workspace` object. For Automated ML you will need to create an `Experiment` object, which is a named object in a `Workspace` used to run experiments."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import logging\n",
+        "\n",
+        "from matplotlib import pyplot as plt\n",
+        "import numpy as np\n",
+        "import pandas as pd\n",
+        "\n",
+        "import azureml.core\n",
+        "from azureml.core.experiment import Experiment\n",
+        "from azureml.core.workspace import Workspace\n",
+        "import azureml.dataprep as dprep\n",
+        "from azureml.automl.core.featurization import FeaturizationConfig\n",
+        "from azureml.train.automl import AutoMLConfig\n",
+        "from azureml.core.dataset import Dataset"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "ws = Workspace.from_config()\n",
+        "\n",
+        "# Choose a name for the experiment.\n",
+        "experiment_name = 'automl-regression-hardware-explain'\n",
+        "experiment = Experiment(ws, experiment_name)\n",
+        "\n",
+        "output = {}\n",
+        "output['SDK version'] = azureml.core.VERSION\n",
+        "output['Subscription ID'] = ws.subscription_id\n",
+        "output['Workspace Name'] = ws.name\n",
+        "output['Resource Group'] = ws.resource_group\n",
+        "output['Location'] = ws.location\n",
+        "output['Experiment Name'] = experiment.name\n",
+        "pd.set_option('display.max_colwidth', -1)\n",
+        "outputDf = pd.DataFrame(data = output, index = [''])\n",
+        "outputDf.T"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Create or Attach existing AmlCompute\n",
+        "You will need to create a [compute target](https://docs.microsoft.com/azure/machine-learning/service/concept-azure-machine-learning-architecture#compute-target) for your AutoML run. In this tutorial, you create `AmlCompute` as your training compute resource.\n",
+        "\n",
+        "**Creation of AmlCompute takes approximately 5 minutes.** If the AmlCompute with that name is already in your workspace this code will skip the creation process.\n",
+        "\n",
+        "As with other Azure services, there are limits on certain resources (e.g. AmlCompute) associated with the Azure Machine Learning service. Please read [this article](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-manage-quotas) on the default limits and how to request more quota."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.compute import AmlCompute\n",
+        "from azureml.core.compute import ComputeTarget\n",
+        "\n",
+        "# Choose a name for your cluster.\n",
+        "amlcompute_cluster_name = \"cpu-cluster-5\"\n",
+        "\n",
+        "found = False\n",
+        "# Check if this compute target already exists in the workspace.\n",
+        "cts = ws.compute_targets\n",
+        "if amlcompute_cluster_name in cts and cts[amlcompute_cluster_name].type == 'AmlCompute':\n",
+        "    found = True\n",
+        "    print('Found existing compute target.')\n",
+        "    compute_target = cts[amlcompute_cluster_name]\n",
+        "\n",
+        "if not found:\n",
+        "    print('Creating a new compute target...')\n",
+        "    provisioning_config = AmlCompute.provisioning_configuration(vm_size = \"STANDARD_D2_V2\", # for GPU, use \"STANDARD_NC6\"\n",
+        "                                                                #vm_priority = 'lowpriority', # optional\n",
+        "                                                                max_nodes = 4)\n",
+        "\n",
+        "    # Create the cluster.\\n\",\n",
+        "    compute_target = ComputeTarget.create(ws, amlcompute_cluster_name, provisioning_config)\n",
+        "\n",
+        "print('Checking cluster status...')\n",
+        "# Can poll for a minimum number of nodes and for a specific timeout.\n",
+        "# If no min_node_count is provided, it will use the scale settings for the cluster.\n",
+        "compute_target.wait_for_completion(show_output = True, min_node_count = None, timeout_in_minutes = 20)\n",
+        "\n",
+        "# For a more detailed view of current AmlCompute status, use get_status()."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Setup Training and Test Data for AutoML experiment\n",
+        "\n",
+        "Load the hardware dataset from a csv file containing both training features and labels. The features are inputs to the model, while the training labels represent the expected output of the model. Next, we'll split the data using random_split and extract the training data for the model.  We also register the datasets in your workspace using a name so that these datasets may be accessed from the remote compute."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "data = 'https://automlsamplenotebookdata.blob.core.windows.net/automl-sample-notebook-data/machineData.csv'\n",
+        "\n",
+        "dataset = Dataset.Tabular.from_delimited_files(data)\n",
+        "\n",
+        "# Split the dataset into train and test datasets\n",
+        "train_data, test_data = dataset.random_split(percentage=0.8, seed=223)\n",
+        "\n",
+        "\n",
+        "# Register the train dataset with your workspace\n",
+        "train_data.register(workspace = ws, name = 'machineData_train_dataset',\n",
+        "                       description = 'hardware performance training data',\n",
+        "                      create_new_version=True)\n",
+        "\n",
+        "# Register the test dataset with your workspace\n",
+        "test_data.register(workspace = ws, name = 'machineData_test_dataset', description = 'hardware performance test data', create_new_version=True)\n",
+        "\n",
+        "label =\"ERP\"\n",
+        "\n",
+        "train_data.to_pandas_dataframe().head()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Train\n",
+        "\n",
+        "Instantiate an `AutoMLConfig` object to specify the settings and data used to run the experiment.\n",
+        "\n",
+        "|Property|Description|\n",
+        "|-|-|\n",
+        "|**task**|classification, regression or forecasting|\n",
+        "|**primary_metric**|This is the metric that you want to optimize. Regression supports the following primary metrics: <br><i>spearman_correlation</i><br><i>normalized_root_mean_squared_error</i><br><i>r2_score</i><br><i>normalized_mean_absolute_error</i>|\n",
+        "|**experiment_timeout_minutes**| Maximum amount of time in minutes that all iterations combined can take before the experiment terminates.|\n",
+        "|**enable_early_stopping**| Flag to enble early termination if the score is not improving in the short term.|\n",
+        "|**featurization**| 'auto' / 'off' / FeaturizationConfig Indicator for whether featurization step should be done automatically or not, or whether customized featurization should be used. Note: If the input data is sparse, featurization cannot be turned on.|\n",
+        "|**n_cross_validations**|Number of cross validation splits.|\n",
+        "|**training_data**|(sparse) array-like, shape = [n_samples, n_features]|\n",
+        "|**label_column_name**|(sparse) array-like, shape = [n_samples, ], targets values.|"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Customization\n",
+        "\n",
+        "This step requires an Enterprise workspace to gain access to this feature. To learn more about creating an Enterprise workspace or upgrading to an Enterprise workspace from the Azure portal, please visit our [Workspace page.](https://docs.microsoft.com/azure/machine-learning/service/concept-workspace#upgrade). \n",
+        "\n",
+        "Supported customization includes:\n",
+        "1. Column purpose update: Override feature type for the specified column.\n",
+        "2. Transformer parameter update: Update parameters for the specified transformer. Currently supports Imputer and HashOneHotEncoder.\n",
+        "3. Drop columns: Columns to drop from being featurized.\n",
+        "4. Block transformers: Allow/Block transformers to be used on featurization process."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Create FeaturizationConfig object using API calls"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "featurization_config = FeaturizationConfig()\n",
+        "featurization_config.blocked_transformers = ['LabelEncoder']\n",
+        "#featurization_config.drop_columns = ['ERP', 'MMIN']\n",
+        "featurization_config.add_column_purpose('MYCT', 'Numeric')\n",
+        "featurization_config.add_column_purpose('VendorName', 'CategoricalHash')\n",
+        "#default strategy mean, add transformer param for for 3 columns\n",
+        "featurization_config.add_transformer_params('Imputer', ['CACH'], {\"strategy\": \"median\"})\n",
+        "featurization_config.add_transformer_params('Imputer', ['CHMIN'], {\"strategy\": \"median\"})\n",
+        "featurization_config.add_transformer_params('Imputer', ['PRP'], {\"strategy\": \"most_frequent\"})\n",
+        "#featurization_config.add_transformer_params('HashOneHotEncoder', [], {\"number_of_bits\": 3})"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "automl_settings = {\n",
+        "    \"enable_early_stopping\": True, \n",
+        "    \"experiment_timeout_minutes\" : 10,\n",
+        "    \"max_concurrent_iterations\": 4,\n",
+        "    \"max_cores_per_iteration\": -1,\n",
+        "    \"n_cross_validations\": 5,\n",
+        "    \"primary_metric\": 'normalized_root_mean_squared_error',\n",
+        "    \"verbosity\": logging.INFO\n",
+        "}\n",
+        "\n",
+        "automl_config = AutoMLConfig(task = 'regression',\n",
+        "                             debug_log = 'automl_errors.log',\n",
+        "                             compute_target=compute_target,\n",
+        "                             featurization=featurization_config,\n",
+        "                             training_data = train_data,\n",
+        "                             label_column_name = label,\n",
+        "                             **automl_settings\n",
+        "                            )"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Call the `submit` method on the experiment object and pass the run configuration. Execution of local runs is synchronous. Depending on the data and the number of iterations this can run for a while.\n",
+        "In this example, we specify `show_output = True` to print currently running iterations to the console."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "remote_run = experiment.submit(automl_config, show_output = False)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "remote_run"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Run the following cell to access previous runs. Uncomment the cell below and update the run_id."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "#from azureml.train.automl.run import AutoMLRun\n",
+        "#experiment_name = 'automl-regression-hardware'\n",
+        "#experiment = Experiment(ws, experiment_name)\n",
+        "#remote_run = AutoMLRun(experiment=experiment, run_id='<run_ID_goes_here')\n",
+        "#remote_run"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "remote_run.wait_for_completion()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "best_run, fitted_model = remote_run.get_output()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "best_run_customized, fitted_model_customized = remote_run.get_output()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Transparency\n",
+        "\n",
+        "View updated featurization summary"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "custom_featurizer = fitted_model_customized.named_steps['datatransformer']"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "custom_featurizer.get_featurization_summary()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "is_user_friendly=False allows for more detailed summary for transforms being applied"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "custom_featurizer.get_featurization_summary(is_user_friendly=False)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "custom_featurizer.get_stats_feature_type_summary()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Results"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Widget for Monitoring Runs\n",
+        "\n",
+        "The widget will first report a \"loading\" status while running the first iteration. After completing the first iteration, an auto-updating graph and table will be shown. The widget will refresh once per minute, so you should see the graph update as child runs complete.\n",
+        "\n",
+        "**Note:** The widget displays a link at the bottom. Use this link to open a web interface to explore the individual run details"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.widgets import RunDetails\n",
+        "RunDetails(remote_run).show() "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Explanations\n",
+        "This step requires an Enterprise workspace to gain access to this feature. To learn more about creating an Enterprise workspace or upgrading to an Enterprise workspace from the Azure portal, please visit our [Workspace page.](https://docs.microsoft.com/azure/machine-learning/service/concept-workspace#upgrade). \n",
+        "This section will walk you through the workflow to compute model explanations for an AutoML model on your remote compute.\n",
+        "\n",
+        "### Retrieve any AutoML Model for explanations\n",
+        "\n",
+        "Below we select the some AutoML pipeline from our iterations. The `get_output` method returns the a AutoML run and the fitted model for the last invocation. Overloads on `get_output` allow you to retrieve the best run and fitted model for *any* logged metric or for a particular *iteration*."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "automl_run, fitted_model = remote_run.get_output(metric='r2_score')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Setup model explanation run on the remote compute\n",
+        "The following section provides details on how to setup an AzureML experiment to run model explanations for an AutoML model on your remote compute."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Sample script used for computing explanations\n",
+        "View the sample script for computing the model explanations for your AutoML model on remote compute."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "with open('train_explainer.py', 'r') as cefr:\n",
+        "    print(cefr.read())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Substitute values in your sample script\n",
+        "The following cell shows how you change the values in the sample script so that you can change the sample script according to your experiment and dataset."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import shutil\n",
+        "import os\n",
+        "\n",
+        "# create script folder\n",
+        "script_folder = './sample_projects/automl-regression-hardware'\n",
+        "if not os.path.exists(script_folder):\n",
+        "    os.makedirs(script_folder)\n",
+        "\n",
+        "# Copy the sample script to script folder.\n",
+        "shutil.copy('train_explainer.py', script_folder)\n",
+        "\n",
+        "# Create the explainer script that will run on the remote compute.\n",
+        "script_file_name = script_folder + '/train_explainer.py'\n",
+        "\n",
+        "# Open the sample script for modification\n",
+        "with open(script_file_name, 'r') as cefr:\n",
+        "    content = cefr.read()\n",
+        "\n",
+        "# Replace the values in train_explainer.py file with the appropriate values\n",
+        "content = content.replace('<<experimnet_name>>', automl_run.experiment.name) # your experiment name.\n",
+        "content = content.replace('<<run_id>>', automl_run.id) # Run-id of the AutoML run for which you want to explain the model.\n",
+        "content = content.replace('<<target_column_name>>', 'ERP') # Your target column name\n",
+        "content = content.replace('<<task>>', 'regression') # Training task type\n",
+        "# Name of your training dataset register with your workspace\n",
+        "content = content.replace('<<train_dataset_name>>', 'machineData_train_dataset') \n",
+        "# Name of your test dataset register with your workspace\n",
+        "content = content.replace('<<test_dataset_name>>', 'machineData_test_dataset')\n",
+        "\n",
+        "# Write sample file into your script folder.\n",
+        "with open(script_file_name, 'w') as cefw:\n",
+        "    cefw.write(content)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Create conda configuration for model explanations experiment\n",
+        "We need `azureml-explain-model`, `azureml-train-automl` and `azureml-core` packages for computing model explanations for your AutoML model on remote compute."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.runconfig import RunConfiguration\n",
+        "from azureml.core.conda_dependencies import CondaDependencies\n",
+        "import pkg_resources\n",
+        "\n",
+        "# create a new RunConfig object\n",
+        "conda_run_config = RunConfiguration(framework=\"python\")\n",
+        "\n",
+        "# Set compute target to AmlCompute\n",
+        "conda_run_config.target = compute_target\n",
+        "conda_run_config.environment.docker.enabled = True\n",
+        "azureml_pip_packages = [\n",
+        "    'azureml-train-automl', 'azureml-core', 'azureml-explain-model'\n",
+        "]\n",
+        "\n",
+        "# specify CondaDependencies obj\n",
+        "conda_run_config.environment.python.conda_dependencies = CondaDependencies.create(\n",
+        "    conda_packages=['scikit-learn', 'numpy','py-xgboost<=0.80'],\n",
+        "    pip_packages=azureml_pip_packages)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Submit the experiment for model explanations\n",
+        "Submit the experiment with the above `run_config` and the sample script for computing explanations."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Now submit a run on AmlCompute for model explanations\n",
+        "from azureml.core.script_run_config import ScriptRunConfig\n",
+        "\n",
+        "script_run_config = ScriptRunConfig(source_directory=script_folder,\n",
+        "                                    script='train_explainer.py',\n",
+        "                                    run_config=conda_run_config)\n",
+        "\n",
+        "run = experiment.submit(script_run_config)\n",
+        "\n",
+        "# Show run details\n",
+        "run"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "%%time\n",
+        "# Shows output of the run on stdout.\n",
+        "run.wait_for_completion(show_output=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Feature importance  and  explanation dashboard\n",
+        "In this section we describe how you can download the explanation results from the explanations experiment and visualize the feature importance for your AutoML model. "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Setup for visualizing the model explanation results\n",
+        "For visualizing the explanation results for the *fitted_model* we need to perform the following steps:-\n",
+        "1. Featurize test data samples.\n",
+        "\n",
+        "The *automl_explainer_setup_obj* contains all the structures from above list. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "X_test = test_data.drop_columns([label]).to_pandas_dataframe()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.train.automl.runtime.automl_explain_utilities import AutoMLExplainerSetupClass, automl_setup_model_explanations\n",
+        "explainer_setup_class = automl_setup_model_explanations(fitted_model, 'regression', X_test=X_test)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Download engineered feature importance from artifact store\n",
+        "You can use *ExplanationClient* to download the engineered feature explanations from the artifact store of the *automl_run*. You can also use ExplanationDashboard to view the dash board visualization of the feature importance values of the engineered features."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.explain.model._internal.explanation_client import ExplanationClient\n",
+        "from interpret_community.widget import ExplanationDashboard\n",
+        "client = ExplanationClient.from_run(automl_run)\n",
+        "engineered_explanations = client.download_model_explanation(raw=False)\n",
+        "print(engineered_explanations.get_feature_importance_dict())\n",
+        "ExplanationDashboard(engineered_explanations, explainer_setup_class.automl_estimator, datasetX=explainer_setup_class.X_test_transform)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "#### Download raw feature importance from artifact store\n",
+        "You can use *ExplanationClient* to download the raw feature explanations from the artifact store of the *automl_run*. You can also use ExplanationDashboard to view the dash board visualization of the feature importance values of the raw features."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "raw_explanations = client.download_model_explanation(raw=True)\n",
+        "print(raw_explanations.get_feature_importance_dict())\n",
+        "ExplanationDashboard(raw_explanations, explainer_setup_class.automl_pipeline, datasetX=explainer_setup_class.X_test_raw)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Operationailze\n",
+        "In this section we will show how you can operationalize an AutoML model and the explainer which was used to compute the explanations in the previous section.\n",
+        "\n",
+        "### Register the AutoML model and the scoring explainer\n",
+        "We use the *TreeScoringExplainer* from *azureml.explain.model* package to create the scoring explainer which will be used to compute the raw and engineered feature importances at the inference time. \n",
+        "In the cell below, we register the AutoML model and the scoring explainer with the Model Management Service."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Register trained automl model present in the 'outputs' folder in the artifacts\n",
+        "original_model = automl_run.register_model(model_name='automl_model', \n",
+        "                                           model_path='outputs/model.pkl')\n",
+        "scoring_explainer_model = automl_run.register_model(model_name='scoring_explainer',\n",
+        "                                                    model_path='outputs/scoring_explainer.pkl')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Create the conda dependencies for setting up the service\n",
+        "We need to create the conda dependencies comprising of the *azureml-explain-model*, *azureml-train-automl* and *azureml-defaults* packages. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.conda_dependencies import CondaDependencies \n",
+        "\n",
+        "azureml_pip_packages = [\n",
+        "    'azureml-explain-model', 'azureml-train-automl', 'azureml-defaults'\n",
+        "]\n",
+        " \n",
+        "\n",
+        "# specify CondaDependencies obj\n",
+        "myenv = CondaDependencies.create(conda_packages=['scikit-learn', 'pandas', 'numpy', 'py-xgboost<=0.80'],\n",
+        "                                 pip_packages=azureml_pip_packages,\n",
+        "                                 pin_sdk_version=True)\n",
+        "\n",
+        "with open(\"myenv.yml\",\"w\") as f:\n",
+        "    f.write(myenv.serialize_to_string())\n",
+        "\n",
+        "with open(\"myenv.yml\",\"r\") as f:\n",
+        "    print(f.read())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### View your scoring file"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "with open(\"score_explain.py\",\"r\") as f:\n",
+        "    print(f.read())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Deploy the service\n",
+        "In the cell below, we deploy the service using the conda file and the scoring file from the previous steps. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from azureml.core.webservice import Webservice\n",
+        "from azureml.core.model import InferenceConfig\n",
+        "from azureml.core.webservice import AciWebservice\n",
+        "from azureml.core.model import Model\n",
+        "\n",
+        "aciconfig = AciWebservice.deploy_configuration(cpu_cores=1, \n",
+        "                                               memory_gb=1, \n",
+        "                                               tags={\"data\": \"Machine Data\",  \n",
+        "                                                     \"method\" : \"local_explanation\"}, \n",
+        "                                               description='Get local explanations for Machine test data')\n",
+        "\n",
+        "inference_config = InferenceConfig(runtime= \"python\", \n",
+        "                                   entry_script=\"score_explain.py\",\n",
+        "                                   conda_file=\"myenv.yml\")\n",
+        "\n",
+        "# Use configs and models generated above\n",
+        "service = Model.deploy(ws, 'model-scoring', [scoring_explainer_model, original_model], inference_config, aciconfig)\n",
+        "service.wait_for_deployment(show_output=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### View the service logs"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "service.get_logs()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Inference using some test data\n",
+        "Inference using some test data to see the predicted value from autml model, view the engineered feature importance for the predicted value and raw feature importance for the predicted value."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "if service.state == 'Healthy':\n",
+        "    # Serialize the first row of the test data into json\n",
+        "    X_test_json = X_test[:1].to_json(orient='records')\n",
+        "    print(X_test_json)\n",
+        "    # Call the service to get the predictions and the engineered and raw explanations\n",
+        "    output = service.run(X_test_json)\n",
+        "    # Print the predicted value\n",
+        "    print(output['predictions'])\n",
+        "    # Print the engineered feature importances for the predicted value\n",
+        "    print(output['engineered_local_importance_values'])\n",
+        "    # Print the raw feature importances for the predicted value\n",
+        "    print(output['raw_local_importance_values'])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Delete the service\n",
+        "Delete the service once you have finished inferencing."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "service.delete()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Test"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# preview the first 3 rows of the dataset\n",
+        "\n",
+        "test_data = test_data.to_pandas_dataframe()\n",
+        "y_test = test_data['ERP'].fillna(0)\n",
+        "test_data = test_data.drop('ERP', 1)\n",
+        "test_data = test_data.fillna(0)\n",
+        "\n",
+        "\n",
+        "train_data = train_data.to_pandas_dataframe()\n",
+        "y_train = train_data['ERP'].fillna(0)\n",
+        "train_data = train_data.drop('ERP', 1)\n",
+        "train_data = train_data.fillna(0)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "y_pred_train = fitted_model.predict(train_data)\n",
+        "y_residual_train = y_train - y_pred_train\n",
+        "\n",
+        "y_pred_test = fitted_model.predict(test_data)\n",
+        "y_residual_test = y_test - y_pred_test"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "%matplotlib inline\n",
+        "from sklearn.metrics import mean_squared_error, r2_score\n",
+        "\n",
+        "# Set up a multi-plot chart.\n",
+        "f, (a0, a1) = plt.subplots(1, 2, gridspec_kw = {'width_ratios':[1, 1], 'wspace':0, 'hspace': 0})\n",
+        "f.suptitle('Regression Residual Values', fontsize = 18)\n",
+        "f.set_figheight(6)\n",
+        "f.set_figwidth(16)\n",
+        "\n",
+        "# Plot residual values of training set.\n",
+        "a0.axis([0, 360, -100, 100])\n",
+        "a0.plot(y_residual_train, 'bo', alpha = 0.5)\n",
+        "a0.plot([-10,360],[0,0], 'r-', lw = 3)\n",
+        "a0.text(16,170,'RMSE = {0:.2f}'.format(np.sqrt(mean_squared_error(y_train, y_pred_train))), fontsize = 12)\n",
+        "a0.text(16,140,'R2 score = {0:.2f}'.format(r2_score(y_train, y_pred_train)),fontsize = 12)\n",
+        "a0.set_xlabel('Training samples', fontsize = 12)\n",
+        "a0.set_ylabel('Residual Values', fontsize = 12)\n",
+        "\n",
+        "# Plot residual values of test set.\n",
+        "a1.axis([0, 90, -100, 100])\n",
+        "a1.plot(y_residual_test, 'bo', alpha = 0.5)\n",
+        "a1.plot([-10,360],[0,0], 'r-', lw = 3)\n",
+        "a1.text(5,170,'RMSE = {0:.2f}'.format(np.sqrt(mean_squared_error(y_test, y_pred_test))), fontsize = 12)\n",
+        "a1.text(5,140,'R2 score = {0:.2f}'.format(r2_score(y_test, y_pred_test)),fontsize = 12)\n",
+        "a1.set_xlabel('Test samples', fontsize = 12)\n",
+        "a1.set_yticklabels([])\n",
+        "\n",
+        "plt.show()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "%matplotlib inline\n",
+        "test_pred = plt.scatter(y_test, y_pred_test, color='')\n",
+        "test_test = plt.scatter(y_test, y_test, color='g')\n",
+        "plt.legend((test_pred, test_test), ('prediction', 'truth'), loc='upper left', fontsize=8)\n",
+        "plt.show()"
+      ]
+    }
+  ],
+  "metadata": {
+    "authors": [
+      {
+        "name": "anumamah"
+      }
+    ],
+    "categories": [
+      "how-to-use-azureml",
+      "automated-machine-learning"
+    ],
+    "category": "tutorial",
+    "compute": [
+      "AML"
+    ],
+    "datasets": [
+      "MachineData"
+    ],
+    "deployment": [
+      "ACI"
+    ],
+    "exclude_from_index": false,
+    "framework": [
+      "None"
+    ],
+    "friendly_name": "Automated ML run with featurization and model explainability.",
+    "index_order": 5,
+    "kernelspec": {
+      "display_name": "Python 3.6",
+      "language": "python",
+      "name": "python36"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.6.7"
+    },
+    "tags": [
+      "featurization",
+      "explainability",
+      "remote_run",
+      "AutomatedML"
+    ],
+    "task": "Regression"
+  },
+  "nbformat": 4,
+  "nbformat_minor": 2
+}

how-to-use-azureml/automated-machine-learning/regression-hardware-performance-explanation-and-featurization/auto-ml-regression-hardware-performance-explanation-and-featurization.yml

@@ -0,0 +1,13 @@
+name: auto-ml-regression-hardware-performance-explanation-and-featurization
+dependencies:
+- pip:
+  - azureml-sdk
+  - interpret
+  - azureml-defaults
+  - azureml-explain-model
+  - azureml-train-automl
+  - azureml-widgets
+  - matplotlib
+  - pandas_ml
+  - azureml-explain-model
+  - azureml-contrib-interpret

how-to-use-azureml/automated-machine-learning/regression-hardware-performance-explanation-and-featurization/score_explain.py

@@ -0,0 +1,43 @@
+import json
+import numpy as np
+import pandas as pd
+import os
+import pickle
+import azureml.train.automl
+import azureml.explain.model
+from azureml.train.automl.runtime.automl_explain_utilities import AutoMLExplainerSetupClass, \
+    automl_setup_model_explanations
+from sklearn.externals import joblib
+from azureml.core.model import Model
+
+
+def init():
+
+    global automl_model
+    global scoring_explainer
+
+    # Retrieve the path to the model file using the model name
+    # Assume original model is named original_prediction_model
+    automl_model_path = Model.get_model_path('automl_model')
+    scoring_explainer_path = Model.get_model_path('scoring_explainer')
+
+    automl_model = joblib.load(automl_model_path)
+    scoring_explainer = joblib.load(scoring_explainer_path)
+
+
+def run(raw_data):
+    # Get predictions and explanations for each data point
+    data = pd.read_json(raw_data, orient='records')
+    # Make prediction
+    predictions = automl_model.predict(data)
+    # Setup for inferencing explanations
+    automl_explainer_setup_obj = automl_setup_model_explanations(automl_model,
+                                                                 X_test=data, task='regression')
+    # Retrieve model explanations for engineered explanations
+    engineered_local_importance_values = scoring_explainer.explain(automl_explainer_setup_obj.X_test_transform)
+    # Retrieve model explanations for raw explanations
+    raw_local_importance_values = scoring_explainer.explain(automl_explainer_setup_obj.X_test_transform, get_raw=True)
+    # You can return any data type as long as it is JSON-serializable
+    return {'predictions': predictions.tolist(),
+            'engineered_local_importance_values': engineered_local_importance_values,
+            'raw_local_importance_values': raw_local_importance_values}

how-to-use-azureml/automated-machine-learning/regression-hardware-performance-explanation-and-featurization/train_explainer.py

@@ -0,0 +1,76 @@
+# Copyright (c) Microsoft. All rights reserved.
+# Licensed under the MIT license.
+import os
+
+from azureml.core.run import Run
+from azureml.core.experiment import Experiment
+from sklearn.externals import joblib
+from azureml.core.dataset import Dataset
+from azureml.train.automl.runtime.automl_explain_utilities import AutoMLExplainerSetupClass, \
+    automl_setup_model_explanations
+from azureml.explain.model.mimic.models.lightgbm_model import LGBMExplainableModel
+from azureml.explain.model.mimic_wrapper import MimicWrapper
+from automl.client.core.common.constants import MODEL_PATH
+from azureml.explain.model.scoring.scoring_explainer import TreeScoringExplainer, save
+
+
+OUTPUT_DIR = './outputs/'
+os.makedirs(OUTPUT_DIR, exist_ok=True)
+
+# Get workspace from the run context
+run = Run.get_context()
+ws = run.experiment.workspace
+
+# Get the AutoML run object from the experiment name and the workspace
+experiment = Experiment(ws, '<<experimnet_name>>')
+automl_run = Run(experiment=experiment, run_id='<<run_id>>')
+
+# Download the best model from the artifact store
+automl_run.download_file(name=MODEL_PATH, output_file_path='model.pkl')
+
+# Load the AutoML model into memory
+fitted_model = joblib.load('model.pkl')
+
+# Get the train dataset from the workspace
+train_dataset = Dataset.get_by_name(workspace=ws, name='<<train_dataset_name>>')
+# Drop the lablled column to get the training set.
+X_train = train_dataset.drop_columns(columns=['<<target_column_name>>'])
+y_train = train_dataset.keep_columns(columns=['<<target_column_name>>'], validate=True)
+
+# Get the train dataset from the workspace
+test_dataset = Dataset.get_by_name(workspace=ws, name='<<test_dataset_name>>')
+# Drop the lablled column to get the testing set.
+X_test = test_dataset.drop_columns(columns=['<<target_column_name>>'])
+
+# Setup the class for explaining the AtuoML models
+automl_explainer_setup_obj = automl_setup_model_explanations(fitted_model, '<<task>>',
+                                                             X=X_train, X_test=X_test,
+                                                             y=y_train)
+
+# Initialize the Mimic Explainer
+explainer = MimicWrapper(ws, automl_explainer_setup_obj.automl_estimator, LGBMExplainableModel,
+                         init_dataset=automl_explainer_setup_obj.X_transform, run=automl_run,
+                         features=automl_explainer_setup_obj.engineered_feature_names,
+                         feature_maps=[automl_explainer_setup_obj.feature_map],
+                         classes=automl_explainer_setup_obj.classes)
+
+# Compute the engineered explanations
+engineered_explanations = explainer.explain(['local', 'global'],
+                                            eval_dataset=automl_explainer_setup_obj.X_test_transform)
+
+# Compute the raw explanations
+raw_explanations = explainer.explain(['local', 'global'], get_raw=True,
+                                     raw_feature_names=automl_explainer_setup_obj.raw_feature_names,
+                                     eval_dataset=automl_explainer_setup_obj.X_test_transform)
+
+print("Engineered and raw explanations computed successfully")
+
+
+# Initialize the ScoringExplainer
+scoring_explainer = TreeScoringExplainer(explainer.explainer, feature_maps=[automl_explainer_setup_obj.feature_map])
+
+# Pickle scoring explainer locally
+save(scoring_explainer, exist_ok=True)
+
+# Upload the scoring explainer to the automl run
+automl_run.upload_file('outputs/scoring_explainer.pkl', 'scoring_explainer.pkl')

how-to-use-azureml/automated-machine-learning/regression/auto-ml-regression.ipynb

@@ -9,12 +9,19 @@
         "Licensed under the MIT License."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/automated-machine-learning/regression/auto-ml-regression.png)"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
         "# Automated Machine Learning\n",
-        "_**Regression with Local Compute**_\n",
+        "_**Regression with Aml Compute**_\n",
         "\n",
         "## Contents\n",
         "1. [Introduction](#Introduction)\n",
@@ -22,7 +29,8 @@
         "1. [Data](#Data)\n",
         "1. [Train](#Train)\n",
         "1. [Results](#Results)\n",
-        "1. [Test](#Test)\n"
+        "1. [Test](#Test)\n",
+        "\n"
       ]
     },
     {
@@ -30,9 +38,9 @@
       "metadata": {},
       "source": [
         "## Introduction\n",
-        "In this example we use the scikit-learn's [diabetes dataset](http://scikit-learn.org/stable/datasets/index.html#diabetes-dataset) to showcase how you can use AutoML for a simple regression problem.\n",
+        "In this example we use the Hardware Performance Dataset to showcase how you can use AutoML for a simple regression problem. The Regression goal is to predict the performance of certain combinations of hardware parts.\n",
         "\n",
-        "Make sure you have executed the [configuration](../../../configuration.ipynb) before running this notebook.\n",
+        "If you are using an Azure Machine Learning Notebook VM, you are all set.  Otherwise, go through the [configuration](../../../configuration.ipynb)  notebook first if you haven't already to establish your connection to the AzureML Workspace. \n",
         "\n",
         "In this notebook you will learn how to:\n",
         "1. Create an `Experiment` in an existing `Workspace`.\n",
@@ -48,7 +56,7 @@
       "source": [
         "## Setup\n",
         "\n",
-        "As part of the setup you have already created an Azure ML `Workspace` object. For AutoML you will need to create an `Experiment` object, which is a named object in a `Workspace` used to run experiments."
+        "As part of the setup you have already created an Azure ML `Workspace` object. For Automated ML you will need to create an `Experiment` object, which is a named object in a `Workspace` used to run experiments."
       ]
     },
     {
@@ -62,10 +70,12 @@
         "from matplotlib import pyplot as plt\n",
         "import numpy as np\n",
         "import pandas as pd\n",
+        " \n",
         "\n",
         "import azureml.core\n",
         "from azureml.core.experiment import Experiment\n",
         "from azureml.core.workspace import Workspace\n",
+        "from azureml.core.dataset import Dataset\n",
         "from azureml.train.automl import AutoMLConfig"
       ]
     },
@@ -77,20 +87,18 @@
       "source": [
         "ws = Workspace.from_config()\n",
         "\n",
-        "# Choose a name for the experiment and specify the project folder.\n",
-        "experiment_name = 'automl-local-regression'\n",
-        "project_folder = './sample_projects/automl-local-regression'\n",
+        "# Choose a name for the experiment.\n",
+        "experiment_name = 'automl-regression'\n",
         "\n",
         "experiment = Experiment(ws, experiment_name)\n",
         "\n",
         "output = {}\n",
         "output['SDK version'] = azureml.core.VERSION\n",
         "output['Subscription ID'] = ws.subscription_id\n",
-        "output['Workspace Name'] = ws.name\n",
+        "output['Workspace'] = ws.name\n",
         "output['Resource Group'] = ws.resource_group\n",
         "output['Location'] = ws.location\n",
-        "output['Project Directory'] = project_folder\n",
-        "output['Experiment Name'] = experiment.name\n",
+        "output['Run History Name'] = experiment_name\n",
         "pd.set_option('display.max_colwidth', -1)\n",
         "outputDf = pd.DataFrame(data = output, index = [''])\n",
         "outputDf.T"
@@ -100,8 +108,8 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "## Data\n",
-        "This uses scikit-learn's [load_diabetes](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_diabetes.html) method."
+        "### Using AmlCompute\n",
+        "You will need to create a [compute target](https://docs.microsoft.com/azure/machine-learning/service/concept-azure-machine-learning-architecture#compute-target) for your AutoML run. In this tutorial, you use `AmlCompute` as your training compute resource."
       ]
     },
     {
@@ -110,15 +118,52 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "# Load the diabetes dataset, a well-known built-in small dataset that comes with scikit-learn.\n",
-        "from sklearn.datasets import load_diabetes\n",
-        "from sklearn.model_selection import train_test_split\n",
+        "from azureml.core.compute import ComputeTarget, AmlCompute\n",
+        "from azureml.core.compute_target import ComputeTargetException\n",
         "\n",
-        "X, y = load_diabetes(return_X_y = True)\n",
+        "# Choose a name for your CPU cluster\n",
+        "cpu_cluster_name = \"cpu-cluster-2\"\n",
         "\n",
-        "columns = ['age', 'gender', 'bmi', 'bp', 's1', 's2', 's3', 's4', 's5', 's6']\n",
+        "# Verify that cluster does not exist already\n",
+        "try:\n",
+        "    compute_target = ComputeTarget(workspace=ws, name=cpu_cluster_name)\n",
+        "    print('Found existing cluster, use it.')\n",
+        "except ComputeTargetException:\n",
+        "    compute_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_D2_V2',\n",
+        "                                                           max_nodes=4)\n",
+        "    compute_target = ComputeTarget.create(ws, cpu_cluster_name, compute_config)\n",
         "\n",
-        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)"
+        "compute_target.wait_for_completion(show_output=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Data\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Load Data\n",
+        "Load the hardware dataset from a csv file containing both training features and labels. The features are inputs to the model, while the training labels represent the expected output of the model. Next, we'll split the data using random_split and extract the training data for the model. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "data = \"https://automlsamplenotebookdata.blob.core.windows.net/automl-sample-notebook-data/machineData.csv\"\n",
+        "dataset = Dataset.Tabular.from_delimited_files(data)\n",
+        "\n",
+        "# Split the dataset into train and test datasets\n",
+        "train_data, test_data = dataset.random_split(percentage=0.8, seed=223)\n",
+        "\n",
+        "label = \"ERP\"\n"
       ]
     },
     {
@@ -131,14 +176,13 @@
         "\n",
         "|Property|Description|\n",
         "|-|-|\n",
-        "|**task**|classification or regression|\n",
+        "|**task**|classification, regression or forecasting|\n",
         "|**primary_metric**|This is the metric that you want to optimize. Regression supports the following primary metrics: <br><i>spearman_correlation</i><br><i>normalized_root_mean_squared_error</i><br><i>r2_score</i><br><i>normalized_mean_absolute_error</i>|\n",
-        "|**iteration_timeout_minutes**|Time limit in minutes for each iteration.|\n",
-        "|**iterations**|Number of iterations. In each iteration AutoML trains a specific pipeline with the data.|\n",
         "|**n_cross_validations**|Number of cross validation splits.|\n",
-        "|**X**|(sparse) array-like, shape = [n_samples, n_features]|\n",
-        "|**y**|(sparse) array-like, shape = [n_samples, ], targets values.|\n",
-        "|**path**|Relative path to the project folder. AutoML stores configuration files for the experiment under this folder. You can specify a new empty folder.|"
+        "|**training_data**|(sparse) array-like, shape = [n_samples, n_features]|\n",
+        "|**label_column_name**|(sparse) array-like, shape = [n_samples, ], targets values.|\n",
+        "\n",
+        "**_You can find more information about primary metrics_** [here](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-configure-auto-train#primary-metric)"
       ]
     },
     {
@@ -147,24 +191,30 @@
       "metadata": {},
       "outputs": [],
       "source": [
+        "automl_settings = {\n",
+        "    \"n_cross_validations\": 3,\n",
+        "    \"primary_metric\": 'r2_score',\n",
+        "    \"preprocess\": True,\n",
+        "    \"enable_early_stopping\": True, \n",
+        "    \"experiment_timeout_minutes\": 20, #for real scenarios we reccommend a timeout of at least one hour \n",
+        "    \"max_concurrent_iterations\": 4,\n",
+        "    \"max_cores_per_iteration\": -1,\n",
+        "    \"verbosity\": logging.INFO,\n",
+        "}\n",
+        "\n",
         "automl_config = AutoMLConfig(task = 'regression',\n",
-        "                             iteration_timeout_minutes = 10,\n",
-        "                             iterations = 10,\n",
-        "                             primary_metric = 'spearman_correlation',\n",
-        "                             n_cross_validations = 5,\n",
-        "                             debug_log = 'automl.log',\n",
-        "                             verbosity = logging.INFO,\n",
-        "                             X = X_train, \n",
-        "                             y = y_train,\n",
-        "                             path = project_folder)"
+        "                             compute_target = compute_target,\n",
+        "                             training_data = train_data,\n",
+        "                             label_column_name = label,\n",
+        "                             **automl_settings\n",
+        "                            )"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "Call the `submit` method on the experiment object and pass the run configuration. Execution of local runs is synchronous. Depending on the data and the number of iterations this can run for a while.\n",
-        "In this example, we specify `show_output = True` to print currently running iterations to the console."
+        "Call the `submit` method on the experiment object and pass the run configuration. Execution of remote runs is asynchronous. Depending on the data and the number of iterations this can run for a while."
       ]
     },
     {
@@ -173,7 +223,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "local_run = experiment.submit(automl_config, show_output = True)"
+        "remote_run = experiment.submit(automl_config, show_output = False)"
       ]
     },
     {
@@ -182,7 +232,18 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "local_run"
+        "# If you need to retrieve a run that already started, use the following code\n",
+        "#from azureml.train.automl.run import AutoMLRun\n",
+        "#remote_run = AutoMLRun(experiment = experiment, run_id = '<replace with your run id>')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "remote_run"
       ]
     },
     {
@@ -210,16 +271,7 @@
       "outputs": [],
       "source": [
         "from azureml.widgets import RunDetails\n",
-        "RunDetails(local_run).show() "
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "\n",
-        "#### Retrieve All Child Runs\n",
-        "You can also use SDK methods to fetch all the child runs and see individual metrics that we log."
+        "RunDetails(remote_run).show() "
       ]
     },
     {
@@ -228,15 +280,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "children = list(local_run.get_children())\n",
-        "metricslist = {}\n",
-        "for run in children:\n",
-        "    properties = run.get_properties()\n",
-        "    metrics = {k: v for k, v in run.get_metrics().items() if isinstance(v, float)}\n",
-        "    metricslist[int(properties['iteration'])] = metrics\n",
-        "\n",
-        "rundata = pd.DataFrame(metricslist).sort_index(1)\n",
-        "rundata"
+        "remote_run.wait_for_completion()"
       ]
     },
     {
@@ -254,7 +298,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "best_run, fitted_model = local_run.get_output()\n",
+        "best_run, fitted_model = remote_run.get_output()\n",
         "print(best_run)\n",
         "print(fitted_model)"
       ]
@@ -274,7 +318,7 @@
       "outputs": [],
       "source": [
         "lookup_metric = \"root_mean_squared_error\"\n",
-        "best_run, fitted_model = local_run.get_output(metric = lookup_metric)\n",
+        "best_run, fitted_model = remote_run.get_output(metric = lookup_metric)\n",
         "print(best_run)\n",
         "print(fitted_model)"
       ]
@@ -294,7 +338,7 @@
       "outputs": [],
       "source": [
         "iteration = 3\n",
-        "third_run, third_model = local_run.get_output(iteration = iteration)\n",
+        "third_run, third_model = remote_run.get_output(iteration = iteration)\n",
         "print(third_run)\n",
         "print(third_model)"
       ]
@@ -307,10 +351,23 @@
       ]
     },
     {
-      "cell_type": "markdown",
+      "cell_type": "code",
+      "execution_count": null,
       "metadata": {},
+      "outputs": [],
       "source": [
-        "Predict on training and test set, and calculate residual values."
+        "# preview the first 3 rows of the dataset\n",
+        "\n",
+        "test_data = test_data.to_pandas_dataframe()\n",
+        "y_test = test_data['ERP'].fillna(0)\n",
+        "test_data = test_data.drop('ERP', 1)\n",
+        "test_data = test_data.fillna(0)\n",
+        "\n",
+        "\n",
+        "train_data = train_data.to_pandas_dataframe()\n",
+        "y_train = train_data['ERP'].fillna(0)\n",
+        "train_data = train_data.drop('ERP', 1)\n",
+        "train_data = train_data.fillna(0)\n"
       ]
     },
     {
@@ -319,10 +376,10 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "y_pred_train = fitted_model.predict(X_train)\n",
+        "y_pred_train = fitted_model.predict(train_data)\n",
         "y_residual_train = y_train - y_pred_train\n",
         "\n",
-        "y_pred_test = fitted_model.predict(X_test)\n",
+        "y_pred_test = fitted_model.predict(test_data)\n",
         "y_residual_test = y_test - y_pred_test"
       ]
     },
@@ -342,41 +399,57 @@
         "f.set_figwidth(16)\n",
         "\n",
         "# Plot residual values of training set.\n",
-        "a0.axis([0, 360, -200, 200])\n",
+        "a0.axis([0, 360, -100, 100])\n",
         "a0.plot(y_residual_train, 'bo', alpha = 0.5)\n",
         "a0.plot([-10,360],[0,0], 'r-', lw = 3)\n",
         "a0.text(16,170,'RMSE = {0:.2f}'.format(np.sqrt(mean_squared_error(y_train, y_pred_train))), fontsize = 12)\n",
-        "a0.text(16,140,'R2 score = {0:.2f}'.format(r2_score(y_train, y_pred_train)), fontsize = 12)\n",
+        "a0.text(16,140,'R2 score = {0:.2f}'.format(r2_score(y_train, y_pred_train)),fontsize = 12)\n",
         "a0.set_xlabel('Training samples', fontsize = 12)\n",
         "a0.set_ylabel('Residual Values', fontsize = 12)\n",
         "\n",
-        "# Plot a histogram.\n",
-        "a0.hist(y_residual_train, orientation = 'horizontal', color = 'b', bins = 10, histtype = 'step')\n",
-        "a0.hist(y_residual_train, orientation = 'horizontal', color = 'b', alpha = 0.2, bins = 10)\n",
-        "\n",
         "# Plot residual values of test set.\n",
-        "a1.axis([0, 90, -200, 200])\n",
+        "a1.axis([0, 90, -100, 100])\n",
         "a1.plot(y_residual_test, 'bo', alpha = 0.5)\n",
         "a1.plot([-10,360],[0,0], 'r-', lw = 3)\n",
         "a1.text(5,170,'RMSE = {0:.2f}'.format(np.sqrt(mean_squared_error(y_test, y_pred_test))), fontsize = 12)\n",
-        "a1.text(5,140,'R2 score = {0:.2f}'.format(r2_score(y_test, y_pred_test)), fontsize = 12)\n",
+        "a1.text(5,140,'R2 score = {0:.2f}'.format(r2_score(y_test, y_pred_test)),fontsize = 12)\n",
         "a1.set_xlabel('Test samples', fontsize = 12)\n",
         "a1.set_yticklabels([])\n",
         "\n",
-        "# Plot a histogram.\n",
-        "a1.hist(y_residual_test, orientation = 'horizontal', color = 'b', bins = 10, histtype = 'step')\n",
-        "a1.hist(y_residual_test, orientation = 'horizontal', color = 'b', alpha = 0.2, bins = 10)\n",
-        "\n",
         "plt.show()"
       ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "%matplotlib inline\n",
+        "test_pred = plt.scatter(y_test, y_pred_test, color='')\n",
+        "test_test = plt.scatter(y_test, y_test, color='g')\n",
+        "plt.legend((test_pred, test_test), ('prediction', 'truth'), loc='upper left', fontsize=8)\n",
+        "plt.show()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": []
     }
   ],
   "metadata": {
     "authors": [
       {
-        "name": "savitam"
+        "name": "rakellam"
       }
     ],
+    "categories": [
+      "how-to-use-azureml",
+      "automated-machine-learning"
+    ],
     "kernelspec": {
       "display_name": "Python 3.6",
       "language": "python",
@@ -392,7 +465,7 @@
       "name": "python",
       "nbconvert_exporter": "python",
       "pygments_lexer": "ipython3",
-      "version": "3.6.6"
+      "version": "3.6.2"
     }
   },
   "nbformat": 4,

how-to-use-azureml/automated-machine-learning/regression/auto-ml-regression.yml

@@ -0,0 +1,9 @@
+name: auto-ml-regression
+dependencies:
+- pip:
+  - azureml-sdk
+  - azureml-train-automl
+  - azureml-widgets
+  - matplotlib
+  - pandas_ml
+  - paramiko<2.5.0

how-to-use-azureml/automated-machine-learning/remote-amlcompute/auto-ml-remote-amlcompute.ipynb

@@ -1,555 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "Copyright (c) Microsoft Corporation. All rights reserved.\n",
-        "\n",
-        "Licensed under the MIT License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "# Automated Machine Learning\n",
-        "_**Remote Execution using AmlCompute**_\n",
-        "\n",
-        "## Contents\n",
-        "1. [Introduction](#Introduction)\n",
-        "1. [Setup](#Setup)\n",
-        "1. [Data](#Data)\n",
-        "1. [Train](#Train)\n",
-        "1. [Results](#Results)\n",
-        "1. [Test](#Test)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Introduction\n",
-        "In this example we use the scikit-learn's [digit dataset](http://scikit-learn.org/stable/datasets/index.html#optical-recognition-of-handwritten-digits-dataset) to showcase how you can use AutoML for a simple classification problem.\n",
-        "\n",
-        "Make sure you have executed the [configuration](../../../configuration.ipynb) before running this notebook.\n",
-        "\n",
-        "In this notebook you would see\n",
-        "1. Create an `Experiment` in an existing `Workspace`.\n",
-        "2. Create or Attach existing AmlCompute to a workspace.\n",
-        "3. Configure AutoML using `AutoMLConfig`.\n",
-        "4. Train the model using AmlCompute\n",
-        "5. Explore the results.\n",
-        "6. Test the best fitted model.\n",
-        "\n",
-        "In addition this notebook showcases the following features\n",
-        "- **Parallel** executions for iterations\n",
-        "- **Asynchronous** tracking of progress\n",
-        "- **Cancellation** of individual iterations or the entire run\n",
-        "- Retrieving models for any iteration or logged metric\n",
-        "- Specifying AutoML settings as `**kwargs`"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Setup\n",
-        "\n",
-        "As part of the setup you have already created an Azure ML `Workspace` object. For AutoML you will need to create an `Experiment` object, which is a named object in a `Workspace` used to run experiments."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "import logging\n",
-        "import os\n",
-        "import csv\n",
-        "\n",
-        "from matplotlib import pyplot as plt\n",
-        "import numpy as np\n",
-        "import pandas as pd\n",
-        "from sklearn import datasets\n",
-        "\n",
-        "import azureml.core\n",
-        "from azureml.core.experiment import Experiment\n",
-        "from azureml.core.workspace import Workspace\n",
-        "from azureml.train.automl import AutoMLConfig"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "ws = Workspace.from_config()\n",
-        "\n",
-        "# Choose a name for the run history container in the workspace.\n",
-        "experiment_name = 'automl-remote-amlcompute'\n",
-        "project_folder = './project'\n",
-        "\n",
-        "experiment = Experiment(ws, experiment_name)\n",
-        "\n",
-        "output = {}\n",
-        "output['SDK version'] = azureml.core.VERSION\n",
-        "output['Subscription ID'] = ws.subscription_id\n",
-        "output['Workspace Name'] = ws.name\n",
-        "output['Resource Group'] = ws.resource_group\n",
-        "output['Location'] = ws.location\n",
-        "output['Project Directory'] = project_folder\n",
-        "output['Experiment Name'] = experiment.name\n",
-        "pd.set_option('display.max_colwidth', -1)\n",
-        "outputDf = pd.DataFrame(data = output, index = [''])\n",
-        "outputDf.T"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Create or Attach existing AmlCompute\n",
-        "You will need to create a [compute target](https://docs.microsoft.com/azure/machine-learning/service/concept-azure-machine-learning-architecture#compute-target) for your AutoML run. In this tutorial, you create `AmlCompute` as your training compute resource.\n",
-        "\n",
-        "**Creation of AmlCompute takes approximately 5 minutes.** If the AmlCompute with that name is already in your workspace this code will skip the creation process.\n",
-        "\n",
-        "As with other Azure services, there are limits on certain resources (e.g. AmlCompute) associated with the Azure Machine Learning service. Please read [this article](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-manage-quotas) on the default limits and how to request more quota."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "from azureml.core.compute import AmlCompute\n",
-        "from azureml.core.compute import ComputeTarget\n",
-        "\n",
-        "# Choose a name for your cluster.\n",
-        "amlcompute_cluster_name = \"automlcl\"\n",
-        "\n",
-        "found = False\n",
-        "# Check if this compute target already exists in the workspace.\n",
-        "cts = ws.compute_targets\n",
-        "if amlcompute_cluster_name in cts and cts[amlcompute_cluster_name].type == 'AmlCompute':\n",
-        "    found = True\n",
-        "    print('Found existing compute target.')\n",
-        "    compute_target = cts[amlcompute_cluster_name]\n",
-        "    \n",
-        "if not found:\n",
-        "    print('Creating a new compute target...')\n",
-        "    provisioning_config = AmlCompute.provisioning_configuration(vm_size = \"STANDARD_D2_V2\", # for GPU, use \"STANDARD_NC6\"\n",
-        "                                                                #vm_priority = 'lowpriority', # optional\n",
-        "                                                                max_nodes = 6)\n",
-        "\n",
-        "    # Create the cluster.\n",
-        "    compute_target = ComputeTarget.create(ws, amlcompute_cluster_name, provisioning_config)\n",
-        "    \n",
-        "    # Can poll for a minimum number of nodes and for a specific timeout.\n",
-        "    # If no min_node_count is provided, it will use the scale settings for the cluster.\n",
-        "    compute_target.wait_for_completion(show_output = True, min_node_count = None, timeout_in_minutes = 20)\n",
-        "    \n",
-        "     # For a more detailed view of current AmlCompute status, use get_status()."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Data\n",
-        "For remote executions, you need to make the data accessible from the remote compute.\n",
-        "This can be done by uploading the data to DataStore.\n",
-        "In this example, we upload scikit-learn's [load_digits](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_digits.html) data."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "data_train = datasets.load_digits()\n",
-        "\n",
-        "if not os.path.isdir('data'):\n",
-        "    os.mkdir('data')\n",
-        "    \n",
-        "if not os.path.exists(project_folder):\n",
-        "    os.makedirs(project_folder)\n",
-        "    \n",
-        "pd.DataFrame(data_train.data).to_csv(\"data/X_train.tsv\", index=False, header=False, quoting=csv.QUOTE_ALL, sep=\"\\t\")\n",
-        "pd.DataFrame(data_train.target).to_csv(\"data/y_train.tsv\", index=False, header=False, sep=\"\\t\")\n",
-        "\n",
-        "ds = ws.get_default_datastore()\n",
-        "ds.upload(src_dir='./data', target_path='bai_data', overwrite=True, show_progress=True)\n",
-        "\n",
-        "from azureml.core.runconfig import DataReferenceConfiguration\n",
-        "dr = DataReferenceConfiguration(datastore_name=ds.name, \n",
-        "                   path_on_datastore='bai_data', \n",
-        "                   path_on_compute='/tmp/azureml_runs',\n",
-        "                   mode='download', # download files from datastore to compute target\n",
-        "                   overwrite=False)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "from azureml.core.runconfig import RunConfiguration\n",
-        "from azureml.core.conda_dependencies import CondaDependencies\n",
-        "\n",
-        "# create a new RunConfig object\n",
-        "conda_run_config = RunConfiguration(framework=\"python\")\n",
-        "\n",
-        "# Set compute target to AmlCompute\n",
-        "conda_run_config.target = compute_target\n",
-        "conda_run_config.environment.docker.enabled = True\n",
-        "conda_run_config.environment.docker.base_image = azureml.core.runconfig.DEFAULT_CPU_IMAGE\n",
-        "\n",
-        "# set the data reference of the run coonfiguration\n",
-        "conda_run_config.data_references = {ds.name: dr}\n",
-        "\n",
-        "cd = CondaDependencies.create(pip_packages=['azureml-sdk[automl]'], conda_packages=['numpy','py-xgboost<=0.80'])\n",
-        "conda_run_config.environment.python.conda_dependencies = cd"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "%%writefile $project_folder/get_data.py\n",
-        "\n",
-        "import pandas as pd\n",
-        "\n",
-        "def get_data():\n",
-        "    X_train = pd.read_csv(\"/tmp/azureml_runs/bai_data/X_train.tsv\", delimiter=\"\\t\", header=None, quotechar='\"')\n",
-        "    y_train = pd.read_csv(\"/tmp/azureml_runs/bai_data/y_train.tsv\", delimiter=\"\\t\", header=None, quotechar='\"')\n",
-        "\n",
-        "    return { \"X\" : X_train.values, \"y\" : y_train[0].values }\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Train\n",
-        "\n",
-        "You can specify `automl_settings` as `**kwargs` as well. Also note that you can use a `get_data()` function for local excutions too.\n",
-        "\n",
-        "**Note:** When using AmlCompute, you can't pass Numpy arrays directly to the fit method.\n",
-        "\n",
-        "|Property|Description|\n",
-        "|-|-|\n",
-        "|**primary_metric**|This is the metric that you want to optimize. Classification supports the following primary metrics: <br><i>accuracy</i><br><i>AUC_weighted</i><br><i>average_precision_score_weighted</i><br><i>norm_macro_recall</i><br><i>precision_score_weighted</i>|\n",
-        "|**iteration_timeout_minutes**|Time limit in minutes for each iteration.|\n",
-        "|**iterations**|Number of iterations. In each iteration AutoML trains a specific pipeline with the data.|\n",
-        "|**n_cross_validations**|Number of cross validation splits.|\n",
-        "|**max_concurrent_iterations**|Maximum number of iterations that would be executed in parallel. This should be less than the number of cores on the DSVM.|"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "automl_settings = {\n",
-        "    \"iteration_timeout_minutes\": 10,\n",
-        "    \"iterations\": 20,\n",
-        "    \"n_cross_validations\": 5,\n",
-        "    \"primary_metric\": 'AUC_weighted',\n",
-        "    \"preprocess\": False,\n",
-        "    \"max_concurrent_iterations\": 5,\n",
-        "    \"verbosity\": logging.INFO\n",
-        "}\n",
-        "\n",
-        "automl_config = AutoMLConfig(task = 'classification',\n",
-        "                             debug_log = 'automl_errors.log',\n",
-        "                             path = project_folder,\n",
-        "                             run_configuration=conda_run_config,\n",
-        "                             data_script = project_folder + \"/get_data.py\",\n",
-        "                             **automl_settings\n",
-        "                            )\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "Call the `submit` method on the experiment object and pass the run configuration. For remote runs the execution is asynchronous, so you will see the iterations get populated as they complete. You can interact with the widgets and models even when the experiment is running to retrieve the best model up to that point. Once you are satisfied with the model, you can cancel a particular iteration or the whole run.\n",
-        "In this example, we specify `show_output = False` to suppress console output while the run is in progress."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "remote_run = experiment.submit(automl_config, show_output = False)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "remote_run"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Results\n",
-        "\n",
-        "#### Loading executed runs\n",
-        "In case you need to load a previously executed run, enable the cell below and replace the `run_id` value."
-      ]
-    },
-    {
-      "cell_type": "raw",
-      "metadata": {},
-      "source": [
-        "remote_run = AutoMLRun(experiment = experiment, run_id = 'AutoML_5db13491-c92a-4f1d-b622-8ab8d973a058')"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Widget for Monitoring Runs\n",
-        "\n",
-        "The widget will first report a \"loading\" status while running the first iteration. After completing the first iteration, an auto-updating graph and table will be shown. The widget will refresh once per minute, so you should see the graph update as child runs complete.\n",
-        "\n",
-        "You can click on a pipeline to see run properties and output logs.  Logs are also available on the DSVM under `/tmp/azureml_run/{iterationid}/azureml-logs`\n",
-        "\n",
-        "**Note:** The widget displays a link at the bottom. Use this link to open a web interface to explore the individual run details."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "remote_run"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "from azureml.widgets import RunDetails\n",
-        "RunDetails(remote_run).show() "
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "# Wait until the run finishes.\n",
-        "remote_run.wait_for_completion(show_output = True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "\n",
-        "#### Retrieve All Child Runs\n",
-        "You can also use SDK methods to fetch all the child runs and see individual metrics that we log."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "children = list(remote_run.get_children())\n",
-        "metricslist = {}\n",
-        "for run in children:\n",
-        "    properties = run.get_properties()\n",
-        "    metrics = {k: v for k, v in run.get_metrics().items() if isinstance(v, float)}\n",
-        "    metricslist[int(properties['iteration'])] = metrics\n",
-        "\n",
-        "rundata = pd.DataFrame(metricslist).sort_index(1)\n",
-        "rundata"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Cancelling Runs\n",
-        "\n",
-        "You can cancel ongoing remote runs using the `cancel` and `cancel_iteration` functions."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "# Cancel the ongoing experiment and stop scheduling new iterations.\n",
-        "# remote_run.cancel()\n",
-        "\n",
-        "# Cancel iteration 1 and move onto iteration 2.\n",
-        "# remote_run.cancel_iteration(1)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Retrieve the Best Model\n",
-        "\n",
-        "Below we select the best pipeline from our iterations. The `get_output` method returns the best run and the fitted model. The Model includes the pipeline and any pre-processing.  Overloads on `get_output` allow you to retrieve the best run and fitted model for *any* logged metric or for a particular *iteration*."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "best_run, fitted_model = remote_run.get_output()\n",
-        "print(best_run)\n",
-        "print(fitted_model)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Best Model Based on Any Other Metric\n",
-        "Show the run and the model which has the smallest `log_loss` value:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "lookup_metric = \"log_loss\"\n",
-        "best_run, fitted_model = remote_run.get_output(metric = lookup_metric)\n",
-        "print(best_run)\n",
-        "print(fitted_model)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Model from a Specific Iteration\n",
-        "Show the run and the model from the third iteration:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "iteration = 3\n",
-        "third_run, third_model = remote_run.get_output(iteration=iteration)\n",
-        "print(third_run)\n",
-        "print(third_model)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Test\n",
-        "\n",
-        "#### Load Test Data"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "digits = datasets.load_digits()\n",
-        "X_test = digits.data[:10, :]\n",
-        "y_test = digits.target[:10]\n",
-        "images = digits.images[:10]"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "#### Testing Our Best Fitted Model"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "# Randomly select digits and test.\n",
-        "for index in np.random.choice(len(y_test), 2, replace = False):\n",
-        "    print(index)\n",
-        "    predicted = fitted_model.predict(X_test[index:index + 1])[0]\n",
-        "    label = y_test[index]\n",
-        "    title = \"Label value = %d  Predicted value = %d \" % (label, predicted)\n",
-        "    fig = plt.figure(1, figsize=(3,3))\n",
-        "    ax1 = fig.add_axes((0,0,.8,.8))\n",
-        "    ax1.set_title(title)\n",
-        "    plt.imshow(images[index], cmap = plt.cm.gray_r, interpolation = 'nearest')\n",
-        "    plt.show()"
-      ]
-    }
-  ],
-  "metadata": {
-    "authors": [
-      {
-        "name": "savitam"
-      }
-    ],
-    "kernelspec": {
-      "display_name": "Python 3.6",
-      "language": "python",
-      "name": "python36"
-    },
-    "language_info": {
-      "codemirror_mode": {
-        "name": "ipython",
-        "version": 3
-      },
-      "file_extension": ".py",
-      "mimetype": "text/x-python",
-      "name": "python",
-      "nbconvert_exporter": "python",
-      "pygments_lexer": "ipython3",
-      "version": "3.6.6"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 2
-}