From 18cd152591f58161a47e18cbc1e98bb03faaa909 Mon Sep 17 00:00:00 2001 From: vizhur Date: Fri, 12 Jul 2019 22:51:17 +0000 Subject: [PATCH] update samples - test --- configuration.ipynb | 2 +- ...uto-ml-classification-bank-marketing.ipynb | 4 +- ...-ml-classification-credit-card-fraud.ipynb | 4 +- .../auto-ml-dataprep-remote-execution.ipynb | 5 ++- ...auto-ml-regression-concrete-strength.ipynb | 4 +- ...o-ml-regression-hardware-performance.ipynb | 4 +- .../auto-ml-remote-amlcompute.ipynb | 45 +++++++++---------- .../training/logging-api/logging-api.ipynb | 2 +- setup-environment/configuration.ipynb | 2 +- 9 files changed, 40 insertions(+), 32 deletions(-) diff --git a/configuration.ipynb b/configuration.ipynb index 023aab23..e7fb4aab 100644 --- a/configuration.ipynb +++ b/configuration.ipynb @@ -103,7 +103,7 @@ "source": [ "import azureml.core\n", "\n", - "print(\"This notebook was created using version 1.0.48 of the Azure ML SDK\")\n", + "print(\"This notebook was created using version 1.0.48\r\n of the Azure ML SDK\")\n", "print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")" ] }, diff --git a/how-to-use-azureml/automated-machine-learning/classification-bank-marketing/auto-ml-classification-bank-marketing.ipynb b/how-to-use-azureml/automated-machine-learning/classification-bank-marketing/auto-ml-classification-bank-marketing.ipynb index efaccf23..7a44c359 100644 --- a/how-to-use-azureml/automated-machine-learning/classification-bank-marketing/auto-ml-classification-bank-marketing.ipynb +++ b/how-to-use-azureml/automated-machine-learning/classification-bank-marketing/auto-ml-classification-bank-marketing.ipynb @@ -192,6 +192,7 @@ "source": [ "from azureml.core.runconfig import RunConfiguration\n", "from azureml.core.conda_dependencies import CondaDependencies\n", + "import pkg_resources\n", "\n", "# create a new RunConfig object\n", "conda_run_config = RunConfiguration(framework=\"python\")\n", @@ -201,8 +202,9 @@ "conda_run_config.environment.docker.enabled = True\n", "conda_run_config.environment.docker.base_image = azureml.core.runconfig.DEFAULT_CPU_IMAGE\n", "\n", + "dprep_dependency = 'azureml-dataprep==' + pkg_resources.get_distribution(\"azureml-dataprep\").version\n", "\n", - "cd = CondaDependencies.create(pip_packages=['azureml-sdk[automl]'], conda_packages=['numpy','py-xgboost<=0.80'])\n", + "cd = CondaDependencies.create(pip_packages=['azureml-sdk[automl]', dprep_dependency], conda_packages=['numpy','py-xgboost<=0.80'])\n", "conda_run_config.environment.python.conda_dependencies = cd" ] }, diff --git a/how-to-use-azureml/automated-machine-learning/classification-credit-card-fraud/auto-ml-classification-credit-card-fraud.ipynb b/how-to-use-azureml/automated-machine-learning/classification-credit-card-fraud/auto-ml-classification-credit-card-fraud.ipynb index 9bd1342a..00a03fa6 100644 --- a/how-to-use-azureml/automated-machine-learning/classification-credit-card-fraud/auto-ml-classification-credit-card-fraud.ipynb +++ b/how-to-use-azureml/automated-machine-learning/classification-credit-card-fraud/auto-ml-classification-credit-card-fraud.ipynb @@ -189,6 +189,7 @@ "source": [ "from azureml.core.runconfig import RunConfiguration\n", "from azureml.core.conda_dependencies import CondaDependencies\n", + "import pkg_resources\n", "\n", "# create a new RunConfig object\n", "conda_run_config = RunConfiguration(framework=\"python\")\n", @@ -198,8 +199,9 @@ "conda_run_config.environment.docker.enabled = True\n", "conda_run_config.environment.docker.base_image = azureml.core.runconfig.DEFAULT_CPU_IMAGE\n", "\n", + "dprep_dependency = 'azureml-dataprep==' + pkg_resources.get_distribution(\"azureml-dataprep\").version\n", "\n", - "cd = CondaDependencies.create(pip_packages=['azureml-sdk[automl]'], conda_packages=['numpy','py-xgboost<=0.80'])\n", + "cd = CondaDependencies.create(pip_packages=['azureml-sdk[automl]', dprep_dependency], conda_packages=['numpy','py-xgboost<=0.80'])\n", "conda_run_config.environment.python.conda_dependencies = cd" ] }, diff --git a/how-to-use-azureml/automated-machine-learning/dataprep-remote-execution/auto-ml-dataprep-remote-execution.ipynb b/how-to-use-azureml/automated-machine-learning/dataprep-remote-execution/auto-ml-dataprep-remote-execution.ipynb index abc53d82..aa85a399 100644 --- a/how-to-use-azureml/automated-machine-learning/dataprep-remote-execution/auto-ml-dataprep-remote-execution.ipynb +++ b/how-to-use-azureml/automated-machine-learning/dataprep-remote-execution/auto-ml-dataprep-remote-execution.ipynb @@ -241,6 +241,7 @@ "source": [ "from azureml.core.runconfig import RunConfiguration\n", "from azureml.core.conda_dependencies import CondaDependencies\n", + "import pkg_resources\n", "\n", "# create a new RunConfig object\n", "conda_run_config = RunConfiguration(framework=\"python\")\n", @@ -250,7 +251,9 @@ "conda_run_config.environment.docker.enabled = True\n", "conda_run_config.environment.docker.base_image = azureml.core.runconfig.DEFAULT_CPU_IMAGE\n", "\n", - "cd = CondaDependencies.create(pip_packages=['azureml-sdk[automl]'], conda_packages=['numpy','py-xgboost<=0.80'])\n", + "dprep_dependency = 'azureml-dataprep==' + pkg_resources.get_distribution(\"azureml-dataprep\").version\n", + "\n", + "cd = CondaDependencies.create(pip_packages=['azureml-sdk[automl]', dprep_dependency], conda_packages=['numpy','py-xgboost<=0.80'])\n", "conda_run_config.environment.python.conda_dependencies = cd" ] }, diff --git a/how-to-use-azureml/automated-machine-learning/regression-concrete-strength/auto-ml-regression-concrete-strength.ipynb b/how-to-use-azureml/automated-machine-learning/regression-concrete-strength/auto-ml-regression-concrete-strength.ipynb index 6b1fc201..88d8bba0 100644 --- a/how-to-use-azureml/automated-machine-learning/regression-concrete-strength/auto-ml-regression-concrete-strength.ipynb +++ b/how-to-use-azureml/automated-machine-learning/regression-concrete-strength/auto-ml-regression-concrete-strength.ipynb @@ -184,6 +184,7 @@ "source": [ "from azureml.core.runconfig import RunConfiguration\n", "from azureml.core.conda_dependencies import CondaDependencies\n", + "import pkg_resources\n", "\n", "# create a new RunConfig object\n", "conda_run_config = RunConfiguration(framework=\"python\")\n", @@ -193,8 +194,9 @@ "conda_run_config.environment.docker.enabled = True\n", "conda_run_config.environment.docker.base_image = azureml.core.runconfig.DEFAULT_CPU_IMAGE\n", "\n", + "dprep_dependency = 'azureml-dataprep==' + pkg_resources.get_distribution(\"azureml-dataprep\").version\n", "\n", - "cd = CondaDependencies.create(pip_packages=['azureml-sdk[automl]'], conda_packages=['numpy'])\n", + "cd = CondaDependencies.create(pip_packages=['azureml-sdk[automl]', dprep_dependency], conda_packages=['numpy'])\n", "conda_run_config.environment.python.conda_dependencies = cd" ] }, diff --git a/how-to-use-azureml/automated-machine-learning/regression-hardware-performance/auto-ml-regression-hardware-performance.ipynb b/how-to-use-azureml/automated-machine-learning/regression-hardware-performance/auto-ml-regression-hardware-performance.ipynb index 5376ac3d..0cebc885 100644 --- a/how-to-use-azureml/automated-machine-learning/regression-hardware-performance/auto-ml-regression-hardware-performance.ipynb +++ b/how-to-use-azureml/automated-machine-learning/regression-hardware-performance/auto-ml-regression-hardware-performance.ipynb @@ -184,6 +184,7 @@ "source": [ "from azureml.core.runconfig import RunConfiguration\n", "from azureml.core.conda_dependencies import CondaDependencies\n", + "import pkg_resources\n", "\n", "# create a new RunConfig object\n", "conda_run_config = RunConfiguration(framework=\"python\")\n", @@ -193,8 +194,9 @@ "conda_run_config.environment.docker.enabled = True\n", "conda_run_config.environment.docker.base_image = azureml.core.runconfig.DEFAULT_CPU_IMAGE\n", "\n", + "dprep_dependency = 'azureml-dataprep==' + pkg_resources.get_distribution(\"azureml-dataprep\").version\n", "\n", - "cd = CondaDependencies.create(pip_packages=['azureml-sdk[automl]'], conda_packages=['numpy'])\n", + "cd = CondaDependencies.create(pip_packages=['azureml-sdk[automl]', dprep_dependency], conda_packages=['numpy'])\n", "conda_run_config.environment.python.conda_dependencies = cd" ] }, diff --git a/how-to-use-azureml/automated-machine-learning/remote-amlcompute/auto-ml-remote-amlcompute.ipynb b/how-to-use-azureml/automated-machine-learning/remote-amlcompute/auto-ml-remote-amlcompute.ipynb index f00e51dc..71553abf 100644 --- a/how-to-use-azureml/automated-machine-learning/remote-amlcompute/auto-ml-remote-amlcompute.ipynb +++ b/how-to-use-azureml/automated-machine-learning/remote-amlcompute/auto-ml-remote-amlcompute.ipynb @@ -84,7 +84,8 @@ "import azureml.core\n", "from azureml.core.experiment import Experiment\n", "from azureml.core.workspace import Workspace\n", - "from azureml.train.automl import AutoMLConfig" + "from azureml.train.automl import AutoMLConfig\n", + "import azureml.dataprep as dprep" ] }, { @@ -186,18 +187,11 @@ "if not os.path.exists(project_folder):\n", " os.makedirs(project_folder)\n", " \n", - "pd.DataFrame(data_train.data).to_csv(\"data/X_train.tsv\", index=False, header=False, quoting=csv.QUOTE_ALL, sep=\"\\t\")\n", - "pd.DataFrame(data_train.target).to_csv(\"data/y_train.tsv\", index=False, header=False, sep=\"\\t\")\n", + "pd.DataFrame(data_train.data[100:,:]).to_csv(\"data/X_train.csv\", index=False)\n", + "pd.DataFrame(data_train.target[100:]).to_csv(\"data/y_train.csv\", index=False)\n", "\n", "ds = ws.get_default_datastore()\n", - "ds.upload(src_dir='./data', target_path='bai_data', overwrite=True, show_progress=True)\n", - "\n", - "from azureml.core.runconfig import DataReferenceConfiguration\n", - "dr = DataReferenceConfiguration(datastore_name=ds.name, \n", - " path_on_datastore='bai_data', \n", - " path_on_compute='/tmp/azureml_runs',\n", - " mode='download', # download files from datastore to compute target\n", - " overwrite=False)" + "ds.upload(src_dir='./data', target_path='digitsdata', overwrite=True, show_progress=True)" ] }, { @@ -208,6 +202,7 @@ "source": [ "from azureml.core.runconfig import RunConfiguration\n", "from azureml.core.conda_dependencies import CondaDependencies\n", + "import pkg_resources\n", "\n", "# create a new RunConfig object\n", "conda_run_config = RunConfiguration(framework=\"python\")\n", @@ -217,28 +212,29 @@ "conda_run_config.environment.docker.enabled = True\n", "conda_run_config.environment.docker.base_image = azureml.core.runconfig.DEFAULT_CPU_IMAGE\n", "\n", - "# set the data reference of the run coonfiguration\n", - "conda_run_config.data_references = {ds.name: dr}\n", + "dprep_dependency = 'azureml-dataprep==' + pkg_resources.get_distribution(\"azureml-dataprep\").version\n", "\n", - "cd = CondaDependencies.create(pip_packages=['azureml-sdk[automl]'], conda_packages=['numpy','py-xgboost<=0.80'])\n", + "cd = CondaDependencies.create(pip_packages=['azureml-sdk[automl]', dprep_dependency], conda_packages=['numpy','py-xgboost<=0.80'])\n", "conda_run_config.environment.python.conda_dependencies = cd" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Dprep reference\n", + "\n", + "Defined X and y as dprep references, which are passed to automated machine learning in the AutoMLConfig." + ] + }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "%%writefile $project_folder/get_data.py\n", - "\n", - "import pandas as pd\n", - "\n", - "def get_data():\n", - " X_train = pd.read_csv(\"/tmp/azureml_runs/bai_data/X_train.tsv\", delimiter=\"\\t\", header=None, quotechar='\"')\n", - " y_train = pd.read_csv(\"/tmp/azureml_runs/bai_data/y_train.tsv\", delimiter=\"\\t\", header=None, quotechar='\"')\n", - "\n", - " return { \"X\" : X_train.values, \"y\" : y_train[0].values }\n" + "X = dprep.auto_read_file(path=ds.path('digitsdata/X_train.csv'))\n", + "y = dprep.auto_read_file(path=ds.path('digitsdata/y_train.csv'))" ] }, { @@ -280,7 +276,8 @@ " debug_log = 'automl_errors.log',\n", " path = project_folder,\n", " run_configuration=conda_run_config,\n", - " data_script = project_folder + \"/get_data.py\",\n", + " X = X,\n", + " y = y,\n", " **automl_settings\n", " )\n" ] diff --git a/how-to-use-azureml/training/logging-api/logging-api.ipynb b/how-to-use-azureml/training/logging-api/logging-api.ipynb index 8e45a3ba..4d9401ae 100644 --- a/how-to-use-azureml/training/logging-api/logging-api.ipynb +++ b/how-to-use-azureml/training/logging-api/logging-api.ipynb @@ -100,7 +100,7 @@ "\n", "# Check core SDK version number\n", "\n", - "print(\"This notebook was created using SDK version 1.0.48, you are currently running version\", azureml.core.VERSION)" + "print(\"This notebook was created using SDK version 1.0.48\r\n, you are currently running version\", azureml.core.VERSION)" ] }, { diff --git a/setup-environment/configuration.ipynb b/setup-environment/configuration.ipynb index 2f623fdd..a197eafd 100644 --- a/setup-environment/configuration.ipynb +++ b/setup-environment/configuration.ipynb @@ -102,7 +102,7 @@ "source": [ "import azureml.core\n", "\n", - "print(\"This notebook was created using version 1.0.48 of the Azure ML SDK\")\n", + "print(\"This notebook was created using version 1.0.48\r\n of the Azure ML SDK\")\n", "print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")" ] },