jprdonnelly/MachineLearningNotebooks
1
0
Fork 0
mirror of https://github.com/Azure/MachineLearningNotebooks.git synced 2025-12-19 17:17:04 -05:00
Code Issues Projects Releases Wiki Activity

update samples from Release-61 as a part of SDK release

Browse Source
This commit is contained in:
amlrelsa-ms
2020-08-17 17:45:26 +00:00
parent 79739b5e1b
commit d0dc4836ae
46 changed files with 1757 additions and 1414 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
configuration.ipynb
View File

@@ -103,7 +103,7 @@
"source": [
"import azureml.core\n",
"\n",
"print(\"This notebook was created using version 1.11.0 of the Azure ML SDK\")\n",
"print(\"This notebook was created using version 1.12.0 of the Azure ML SDK\")\n",
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
]
},

89
how-to-use-azureml/automated-machine-learning/classification-bank-marketing-all-features/auto-ml-classification-bank-marketing-all-features.ipynb
View File

@@ -89,7 +89,7 @@
"from azureml.automl.core.featurization import FeaturizationConfig\n",
"from azureml.core.dataset import Dataset\n",
"from azureml.train.automl import AutoMLConfig\n",
"from azureml.explain.model._internal.explanation_client import ExplanationClient"
"from azureml.interpret._internal.explanation_client import ExplanationClient"
]
},
{
@@ -105,7 +105,7 @@
"metadata": {},
"outputs": [],
"source": [
"print(\"This notebook was created using version 1.11.0 of the Azure ML SDK\")\n",
"print(\"This notebook was created using version 1.12.0 of the Azure ML SDK\")\n",
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
]
},
@@ -733,24 +733,6 @@
"print(aci_service.state)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Delete a Web Service\n",
"\n",
"Deletes the specified web service."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#aci_service.delete()"
]
},
{
"cell_type": "markdown",
"metadata": {},
@@ -775,7 +757,9 @@
"source": [
"## Test\n",
"\n",
"Now that the model is trained, run the test data through the trained model to get the predicted values."
"Now that the model is trained, run the test data through the trained model to get the predicted values. This calls the ACI web service to do the prediction.\n",
"\n",
"Note that the JSON passed to the ACI web service is an array of rows of data. Each row should either be an array of values in the same order that was used for training or a dictionary where the keys are the same as the column names used for training. The example below uses dictionary rows."
]
},
{
@@ -815,10 +799,27 @@
"metadata": {},
"outputs": [],
"source": [
"y_pred = fitted_model.predict(X_test)\n",
"import json\n",
"import requests\n",
"\n",
"X_test_json = X_test.to_json(orient='records')\n",
"data = \"{\\\"data\\\": \" + X_test_json +\"}\"\n",
"headers = {'Content-Type': 'application/json'}\n",
"\n",
"resp = requests.post(aci_service.scoring_uri, data, headers=headers)\n",
"\n",
"y_pred = json.loads(json.loads(resp.text))['result']"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"actual = array(y_test)\n",
"actual = actual[:,0]\n",
"print(y_pred.shape, \" \", actual.shape)"
"print(len(y_pred), \" \", len(actual))"
]
},
{
@@ -827,8 +828,7 @@
"source": [
"### Calculate metrics for the prediction\n",
"\n",
"Now visualize the data on a scatter plot to show what our truth (actual) values are compared to the predicted values \n",
"from the trained model that was returned."
"Now visualize the data as a confusion matrix that compared the predicted values against the actual values.\n"
]
},
{
@@ -838,12 +838,45 @@
"outputs": [],
"source": [
"%matplotlib notebook\n",
"test_pred = plt.scatter(actual, y_pred, color='b')\n",
"test_test = plt.scatter(actual, actual, color='g')\n",
"plt.legend((test_pred, test_test), ('prediction', 'truth'), loc='upper left', fontsize=8)\n",
"from sklearn.metrics import confusion_matrix\n",
"import numpy as np\n",
"import itertools\n",
"\n",
"cf =confusion_matrix(actual,y_pred)\n",
"plt.imshow(cf,cmap=plt.cm.Blues,interpolation='nearest')\n",
"plt.colorbar()\n",
"plt.title('Confusion Matrix')\n",
"plt.xlabel('Predicted')\n",
"plt.ylabel('Actual')\n",
"class_labels = ['no','yes']\n",
"tick_marks = np.arange(len(class_labels))\n",
"plt.xticks(tick_marks,class_labels)\n",
"plt.yticks([-0.5,0,1,1.5],['','no','yes',''])\n",
"# plotting text value inside cells\n",
"thresh = cf.max() / 2.\n",
"for i,j in itertools.product(range(cf.shape[0]),range(cf.shape[1])):\n",
" plt.text(j,i,format(cf[i,j],'d'),horizontalalignment='center',color='white' if cf[i,j] >thresh else 'black')\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Delete a Web Service\n",
"\n",
"Deletes the specified web service."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"aci_service.delete()"
]
},
{
"cell_type": "markdown",
"metadata": {},

2
how-to-use-azureml/automated-machine-learning/classification-credit-card-fraud/auto-ml-classification-credit-card-fraud.ipynb
View File

@@ -93,7 +93,7 @@
"metadata": {},
"outputs": [],
"source": [
"print(\"This notebook was created using version 1.11.0 of the Azure ML SDK\")\n",
"print(\"This notebook was created using version 1.12.0 of the Azure ML SDK\")\n",
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
]
},

2
how-to-use-azureml/automated-machine-learning/classification-text-dnn/auto-ml-classification-text-dnn.ipynb
View File

@@ -97,7 +97,7 @@
"metadata": {},
"outputs": [],
"source": [
"print(\"This notebook was created using version 1.11.0 of the Azure ML SDK\")\n",
"print(\"This notebook was created using version 1.12.0 of the Azure ML SDK\")\n",
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
]
},

3
how-to-use-azureml/automated-machine-learning/continuous-retraining/auto-ml-continuous-retraining.ipynb
View File

@@ -88,7 +88,7 @@
"metadata": {},
"outputs": [],
"source": [
"print(\"This notebook was created using version 1.11.0 of the Azure ML SDK\")\n",
"print(\"This notebook was created using version 1.12.0 of the Azure ML SDK\")\n",
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
]
},
@@ -204,7 +204,6 @@
"cd = CondaDependencies.create(pip_packages=['azureml-sdk[automl]', 'applicationinsights', 'azureml-opendatasets', 'azureml-defaults'], \n",
" conda_packages=['numpy==1.16.2'], \n",
" pin_sdk_version=False)\n",
"#cd.add_pip_package('azureml-explain-model')\n",
"conda_run_config.environment.python.conda_dependencies = cd\n",
"\n",
"print('run config is ready')"

24
how-to-use-azureml/automated-machine-learning/forecasting-beer-remote/auto-ml-forecasting-beer-remote.ipynb
View File

@@ -114,7 +114,7 @@
"metadata": {},
"outputs": [],
"source": [
"print(\"This notebook was created using version 1.11.0 of the Azure ML SDK\")\n",
"print(\"This notebook was created using version 1.12.0 of the Azure ML SDK\")\n",
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
]
},
@@ -217,7 +217,7 @@
"\n",
"**Time column** is the time axis along which to predict.\n",
"\n",
"**Grain** is another word for an individual time series in your dataset. Grains are identified by values of the columns listed `grain_column_names`, for example \"store\" and \"item\" if your data has multiple time series of sales, one series for each combination of store and item sold.\n",
"**Time series identifier columns** are identified by values of the columns listed `time_series_id_column_names`, for example \"store\" and \"item\" if your data has multiple time series of sales, one series for each combination of store and item sold.\n",
"\n",
"This dataset has only one time series. Please see the [orange juice notebook](https://github.com/Azure/MachineLearningNotebooks/tree/master/how-to-use-azureml/automated-machine-learning/forecasting-orange-juice-sales) for an example of a multi-time series dataset."
]
@@ -269,7 +269,7 @@
"source": [
"target_column_name = 'BeerProduction'\n",
"time_column_name = 'DATE'\n",
"grain_column_names = []\n",
"time_series_id_column_names = []\n",
"freq = 'M' #Monthly data"
]
},
@@ -329,7 +329,7 @@
},
"outputs": [],
"source": [
"max_horizon = 12"
"forecast_horizon = 12"
]
},
{
@@ -364,11 +364,10 @@
},
"outputs": [],
"source": [
"automl_settings = {\n",
" 'time_column_name': time_column_name,\n",
" 'max_horizon': max_horizon,\n",
" 'enable_dnn' : True,\n",
"}\n",
"from azureml.automl.core.forecasting_parameters import ForecastingParameters\n",
"forecasting_parameters = ForecastingParameters(\n",
" time_column_name=time_column_name, forecast_horizon=forecast_horizon\n",
")\n",
"\n",
"automl_config = AutoMLConfig(task='forecasting', \n",
" primary_metric='normalized_root_mean_squared_error',\n",
@@ -380,7 +379,8 @@
" compute_target=compute_target,\n",
" max_concurrent_iterations=4,\n",
" max_cores_per_iteration=-1,\n",
" **automl_settings)"
" enable_dnn=True,\n",
" forecasting_parameters=forecasting_parameters)"
]
},
{
@@ -581,7 +581,7 @@
"source": [
"from helper import run_inference\n",
"\n",
"test_run = run_inference(test_experiment, compute_target, script_folder, best_dnn_run, test_dataset, valid_dataset, max_horizon,\n",
"test_run = run_inference(test_experiment, compute_target, script_folder, best_dnn_run, test_dataset, valid_dataset, forecast_horizon,\n",
" target_column_name, time_column_name, freq)"
]
},
@@ -603,7 +603,7 @@
"from helper import run_multiple_inferences\n",
"\n",
"summary_df = run_multiple_inferences(summary_df, experiment, test_experiment, compute_target, script_folder, test_dataset, \n",
" valid_dataset, max_horizon, target_column_name, time_column_name, freq)"
" valid_dataset, forecast_horizon, target_column_name, time_column_name, freq)"
]
},
{

63
how-to-use-azureml/automated-machine-learning/forecasting-bike-share/auto-ml-forecasting-bike-share.ipynb
View File

@@ -87,7 +87,7 @@
"metadata": {},
"outputs": [],
"source": [
"print(\"This notebook was created using version 1.11.0 of the Azure ML SDK\")\n",
"print(\"This notebook was created using version 1.12.0 of the Azure ML SDK\")\n",
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
]
},
@@ -238,6 +238,22 @@
"test.to_pandas_dataframe().head(5).reset_index(drop=True)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Forecasting Parameters\n",
"To define forecasting parameters for your experiment training, you can leverage the ForecastingParameters class. The table below details the forecasting parameter we will be passing into our experiment.\n",
"\n",
"|Property|Description|\n",
"|-|-|\n",
"|**time_column_name**|The name of your time column.|\n",
"|**forecast_horizon**|The forecast horizon is how many periods forward you would like to forecast. This integer horizon is in units of the timeseries frequency (e.g. daily, weekly).|\n",
"|**country_or_region_for_holidays**|The country/region used to generate holiday features. These should be ISO 3166 two-letter country/region codes (i.e. 'US', 'GB').|\n",
"|**target_lags**|The target_lags specifies how far back we will construct the lags of the target variable.|\n",
"|**drop_column_names**|Name(s) of columns to drop prior to modeling|"
]
},
{
"cell_type": "markdown",
"metadata": {},
@@ -257,11 +273,7 @@
"|**compute_target**|The remote compute for training.|\n",
"|**n_cross_validations**|Number of cross validation splits.|\n",
"|**enable_early_stopping**|If early stopping is on, training will stop when the primary metric is no longer improving.|\n",
"|**time_column_name**|Name of the datetime column in the input data|\n",
"|**max_horizon**|Maximum desired forecast horizon in units of time-series frequency|\n",
"|**country_or_region**|The country/region used to generate holiday features. These should be ISO 3166 two-letter country/region codes (i.e. 'US', 'GB').|\n",
"|**target_lags**|The target_lags specifies how far back we will construct the lags of the target variable.|\n",
"|**drop_column_names**|Name(s) of columns to drop prior to modeling|\n",
"|**forecasting_parameters**|A class that holds all the forecasting related parameters.|\n",
"\n",
"This notebook uses the blocked_models parameter to exclude some models that take a longer time to train on this dataset. You can choose to remove models from the blocked_models list but you may need to increase the experiment_timeout_hours parameter value to get results."
]
@@ -281,7 +293,7 @@
"metadata": {},
"outputs": [],
"source": [
"max_horizon = 14"
"forecast_horizon = 14"
]
},
{
@@ -297,13 +309,14 @@
"metadata": {},
"outputs": [],
"source": [
"time_series_settings = {\n",
" 'time_column_name': time_column_name,\n",
" 'max_horizon': max_horizon, \n",
" 'country_or_region': 'US', # set country_or_region will trigger holiday featurizer\n",
" 'target_lags': 'auto', # use heuristic based lag setting \n",
" 'drop_column_names': ['casual', 'registered'] # these columns are a breakdown of the total and therefore a leak\n",
"}\n",
"from azureml.automl.core.forecasting_parameters import ForecastingParameters\n",
"forecasting_parameters = ForecastingParameters(\n",
" time_column_name=time_column_name,\n",
" forecast_horizon=forecast_horizon,\n",
" country_or_region_for_holidays='US', # set country_or_region will trigger holiday featurizer\n",
" target_lags='auto', # use heuristic based lag setting \n",
" drop_column_names=['casual', 'registered'] # these columns are a breakdown of the total and therefore a leak\n",
")\n",
"\n",
"automl_config = AutoMLConfig(task='forecasting', \n",
" primary_metric='normalized_root_mean_squared_error',\n",
@@ -317,7 +330,7 @@
" max_concurrent_iterations=4,\n",
" max_cores_per_iteration=-1,\n",
" verbosity=logging.INFO,\n",
" **time_series_settings)"
" forecasting_parameters=forecasting_parameters)"
]
},
{
@@ -422,7 +435,7 @@
"source": [
"We now use the best fitted model from the AutoML Run to make forecasts for the test set. We will do batch scoring on the test dataset which should have the same schema as training dataset.\n",
"\n",
"The scoring will run on a remote compute. In this example, it will reuse the training compute.|"
"The scoring will run on a remote compute. In this example, it will reuse the training compute."
]
},
{
@@ -439,7 +452,7 @@
"metadata": {},
"source": [
"### Retrieving forecasts from the model\n",
"To run the forecast on the remote compute we will use two helper scripts: forecasting_script and forecasting_helper. These scripts contain the utility methods which will be used by the remote estimator. We copy these scripts to the project folder to upload them to remote compute."
"To run the forecast on the remote compute we will use a helper script: forecasting_script. This script contains the utility methods which will be used by the remote estimator. We copy the script to the project folder to upload it to remote compute."
]
},
{
@@ -453,15 +466,14 @@
"\n",
"script_folder = os.path.join(os.getcwd(), 'forecast')\n",
"os.makedirs(script_folder, exist_ok=True)\n",
"shutil.copy('forecasting_script.py', script_folder)\n",
"shutil.copy('forecasting_helper.py', script_folder)"
"shutil.copy('forecasting_script.py', script_folder)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"For brevity we have created the function called run_forecast. It submits the test data to the best model and run the estimation on the selected compute target. Validation errors and current status will be shown when setting `show_output=True` and the execution will be synchronous."
"For brevity, we have created a function called run_forecast that submits the test data to the best model determined during the training run and retrieves forecasts. The test set is longer than the forecast horizon specified at train time, so the forecasting script uses a so-called rolling evaluation to generate predictions over the whole test set. A rolling evaluation iterates the forecaster over the test set, using the actuals in the test set to make lag features as needed. "
]
},
{
@@ -472,8 +484,7 @@
"source": [
"from run_forecast import run_rolling_forecast\n",
"\n",
"remote_run = run_rolling_forecast(test_experiment, compute_target, best_run, test, max_horizon,\n",
" target_column_name, time_column_name)\n",
"remote_run = run_rolling_forecast(test_experiment, compute_target, best_run, test, target_column_name)\n",
"remote_run"
]
},
@@ -537,7 +548,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"The MAPE seems high; it is being skewed by an actual with a small absolute value. For a more informative evaluation, we can calculate the metrics by forecast horizon:"
"Since we did a rolling evaluation on the test set, we can analyze the predictions by their forecast horizon relative to the rolling origin. The model was initially trained at a forecast horizon of 14, so each prediction from the model is associated with a horizon value from 1 to 14. The horizon values are in a column named, \"horizon_origin,\" in the prediction set. For example, we can calculate some of the error metrics grouped by the horizon:"
]
},
{
@@ -557,7 +568,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"It's also interesting to see the distributions of APE (absolute percentage error) by horizon. On a log scale, the outlying APE in the horizon-3 group is clear."
"To drill down more, we can look at the distributions of APE (absolute percentage error) by horizon. From the chart, it is clear that the overall MAPE is being skewed by one particular point where the actual value is of small absolute value."
]
},
{
@@ -567,7 +578,7 @@
"outputs": [],
"source": [
"df_all_APE = df_all.assign(APE=APE(df_all[target_column_name], df_all['predicted']))\n",
"APEs = [df_all_APE[df_all['horizon_origin'] == h].APE.values for h in range(1, max_horizon + 1)]\n",
"APEs = [df_all_APE[df_all['horizon_origin'] == h].APE.values for h in range(1, forecast_horizon + 1)]\n",
"\n",
"%matplotlib inline\n",
"plt.boxplot(APEs)\n",
@@ -631,5 +642,5 @@
"version": 3
},
"nbformat": 4,
"nbformat_minor": 2
"nbformat_minor": 4
}

99
how-to-use-azureml/automated-machine-learning/forecasting-bike-share/forecasting_helper.py
View File

@@ -1,99 +0,0 @@
import pandas as pd
import numpy as np
from pandas.tseries.frequencies import to_offset
def align_outputs(y_predicted, X_trans, X_test, y_test, target_column_name,
predicted_column_name='predicted',
horizon_colname='horizon_origin'):
"""
Demonstrates how to get the output aligned to the inputs
using pandas indexes. Helps understand what happened if
the output's shape differs from the input shape, or if
the data got re-sorted by time and grain during forecasting.
Typical causes of misalignment are:
* we predicted some periods that were missing in actuals -> drop from eval
* model was asked to predict past max_horizon -> increase max horizon
* data at start of X_test was needed for lags -> provide previous periods
"""
if (horizon_colname in X_trans):
df_fcst = pd.DataFrame({predicted_column_name: y_predicted,
horizon_colname: X_trans[horizon_colname]})
else:
df_fcst = pd.DataFrame({predicted_column_name: y_predicted})
# y and X outputs are aligned by forecast() function contract
df_fcst.index = X_trans.index
# align original X_test to y_test
X_test_full = X_test.copy()
X_test_full[target_column_name] = y_test
# X_test_full's index does not include origin, so reset for merge
df_fcst.reset_index(inplace=True)
X_test_full = X_test_full.reset_index().drop(columns='index')
together = df_fcst.merge(X_test_full, how='right')
# drop rows where prediction or actuals are nan
# happens because of missing actuals
# or at edges of time due to lags/rolling windows
clean = together[together[[target_column_name,
predicted_column_name]].notnull().all(axis=1)]
return(clean)
def do_rolling_forecast(fitted_model, X_test, y_test, target_column_name,
time_column_name, max_horizon, freq='D'):
"""
Produce forecasts on a rolling origin over the given test set.
Each iteration makes a forecast for the next 'max_horizon' periods
with respect to the current origin, then advances the origin by the
horizon time duration. The prediction context for each forecast is set so
that the forecaster uses the actual target values prior to the current
origin time for constructing lag features.
This function returns a concatenated DataFrame of rolling forecasts.
"""
df_list = []
origin_time = X_test[time_column_name].min()
while origin_time <= X_test[time_column_name].max():
# Set the horizon time - end date of the forecast
horizon_time = origin_time + max_horizon * to_offset(freq)
# Extract test data from an expanding window up-to the horizon
expand_wind = (X_test[time_column_name] < horizon_time)
X_test_expand = X_test[expand_wind]
y_query_expand = np.zeros(len(X_test_expand)).astype(np.float)
y_query_expand.fill(np.NaN)
if origin_time != X_test[time_column_name].min():
# Set the context by including actuals up-to the origin time
test_context_expand_wind = (X_test[time_column_name] < origin_time)
context_expand_wind = (
X_test_expand[time_column_name] < origin_time)
y_query_expand[context_expand_wind] = y_test[
test_context_expand_wind]
# Make a forecast out to the maximum horizon
y_fcst, X_trans = fitted_model.forecast(X_test_expand, y_query_expand)
# Align forecast with test set for dates within the
# current rolling window
trans_tindex = X_trans.index.get_level_values(time_column_name)
trans_roll_wind = (trans_tindex >= origin_time) & (
trans_tindex < horizon_time)
test_roll_wind = expand_wind & (
X_test[time_column_name] >= origin_time)
df_list.append(align_outputs(y_fcst[trans_roll_wind],
X_trans[trans_roll_wind],
X_test[test_roll_wind],
y_test[test_roll_wind],
target_column_name))
# Advance the origin time
origin_time = horizon_time
return pd.concat(df_list, ignore_index=True)

30
how-to-use-azureml/automated-machine-learning/forecasting-bike-share/forecasting_script.py
View File

@@ -1,37 +1,21 @@
import argparse
import azureml.train.automl
from azureml.automl.runtime.shared import forecasting_models
from azureml.core import Run
from sklearn.externals import joblib
import forecasting_helper
parser = argparse.ArgumentParser()
parser.add_argument(
'--max_horizon', type=int, dest='max_horizon',
default=10, help='Max Horizon for forecasting')
parser.add_argument(
'--target_column_name', type=str, dest='target_column_name',
help='Target Column Name')
parser.add_argument(
'--time_column_name', type=str, dest='time_column_name',
help='Time Column Name')
parser.add_argument(
'--frequency', type=str, dest='freq',
help='Frequency of prediction')
args = parser.parse_args()
max_horizon = args.max_horizon
target_column_name = args.target_column_name
time_column_name = args.time_column_name
freq = args.freq
run = Run.get_context()
# get input dataset by name
test_dataset = run.input_datasets['test_data']
grain_column_names = []
df = test_dataset.to_pandas_dataframe().reset_index(drop=True)
X_test_df = test_dataset.drop_columns(columns=[target_column_name]).to_pandas_dataframe().reset_index(drop=True)
@@ -39,14 +23,12 @@ y_test_df = test_dataset.with_timestamp_columns(None).keep_columns(columns=[targ
fitted_model = joblib.load('model.pkl')
df_all = forecasting_helper.do_rolling_forecast(
fitted_model,
X_test_df,
y_test_df.values.T[0],
target_column_name,
time_column_name,
max_horizon,
freq)
y_pred, X_trans = fitted_model.rolling_evaluation(X_test_df, y_test_df.values)
# Add predictions, actuals, and horizon relative to rolling origin to the test feature data
assign_dict = {'horizon_origin': X_trans['horizon_origin'].values, 'predicted': y_pred,
target_column_name: y_test_df[target_column_name].values}
df_all = X_test_df.assign(**assign_dict)
file_name = 'outputs/predictions.csv'
export_csv = df_all.to_csv(file_name, header=True)

8
how-to-use-azureml/automated-machine-learning/forecasting-bike-share/run_forecast.py
View File

@@ -5,8 +5,7 @@ from azureml.core.run import Run
def run_rolling_forecast(test_experiment, compute_target, train_run, test_dataset,
max_horizon, target_column_name, time_column_name,
freq='D', inference_folder='./forecast'):
target_column_name, inference_folder='./forecast'):
condafile = inference_folder + '/condafile.yml'
train_run.download_file('outputs/model.pkl',
inference_folder + '/model.pkl')
@@ -20,10 +19,7 @@ def run_rolling_forecast(test_experiment, compute_target, train_run, test_datase
est = Estimator(source_directory=inference_folder,
entry_script='forecasting_script.py',
script_params={
'--max_horizon': max_horizon,
'--target_column_name': target_column_name,
'--time_column_name': time_column_name,
'--frequency': freq
'--target_column_name': target_column_name
},
inputs=[test_dataset.as_named_input('test_data')],
compute_target=compute_target,

50
how-to-use-azureml/automated-machine-learning/forecasting-energy-demand/auto-ml-forecasting-energy-demand.ipynb
View File

@@ -97,7 +97,7 @@
"metadata": {},
"outputs": [],
"source": [
"print(\"This notebook was created using version 1.11.0 of the Azure ML SDK\")\n",
"print(\"This notebook was created using version 1.12.0 of the Azure ML SDK\")\n",
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
]
},
@@ -288,7 +288,20 @@
"metadata": {},
"outputs": [],
"source": [
"max_horizon = 48"
"forecast_horizon = 48"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Forecasting Parameters\n",
"To define forecasting parameters for your experiment training, you can leverage the ForecastingParameters class. The table below details the forecasting parameter we will be passing into our experiment.\n",
"\n",
"|Property|Description|\n",
"|-|-|\n",
"|**time_column_name**|The name of your time column.|\n",
"|**forecast_horizon**|The forecast horizon is how many periods forward you would like to forecast. This integer horizon is in units of the timeseries frequency (e.g. daily, weekly).|"
]
},
{
@@ -297,7 +310,7 @@
"source": [
"## Train\n",
"\n",
"Instantiate an AutoMLConfig object. This config defines the settings and data used to run the experiment. We can provide extra configurations within 'automl_settings', for this forecasting task we add the name of the time column and the maximum forecast horizon.\n",
"Instantiate an AutoMLConfig object. This config defines the settings and data used to run the experiment. We can provide extra configurations within 'automl_settings', for this forecasting task we add the forecasting parameters to hold all the additional forecasting parameters.\n",
"\n",
"|Property|Description|\n",
"|-|-|\n",
@@ -310,8 +323,7 @@
"|**compute_target**|The remote compute for training.|\n",
"|**n_cross_validations**|Number of cross validation splits. Rolling Origin Validation is used to split time-series in a temporally consistent way.|\n",
"|**enable_early_stopping**|Flag to enble early termination if the score is not improving in the short term.|\n",
"|**time_column_name**|The name of your time column.|\n",
"|**max_horizon**|The number of periods out you would like to predict past your training data. Periods are inferred from your data.|\n"
"|**forecasting_parameters**|A class holds all the forecasting related parameters.|\n"
]
},
{
@@ -327,10 +339,10 @@
"metadata": {},
"outputs": [],
"source": [
"automl_settings = {\n",
" 'time_column_name': time_column_name,\n",
" 'max_horizon': max_horizon,\n",
"}\n",
"from azureml.automl.core.forecasting_parameters import ForecastingParameters\n",
"forecasting_parameters = ForecastingParameters(\n",
" time_column_name=time_column_name, forecast_horizon=forecast_horizon\n",
")\n",
"\n",
"automl_config = AutoMLConfig(task='forecasting', \n",
" primary_metric='normalized_root_mean_squared_error',\n",
@@ -342,7 +354,7 @@
" enable_early_stopping=True,\n",
" n_cross_validations=3, \n",
" verbosity=logging.INFO,\n",
" **automl_settings)"
" forecasting_parameters=forecasting_parameters)"
]
},
{
@@ -550,7 +562,7 @@
"metadata": {},
"source": [
"## Advanced Training <a id=\"advanced_training\"></a>\n",
"We did not use lags in the previous model specification. In effect, the prediction was the result of a simple regression on date, grain and any additional features. This is often a very good prediction as common time series patterns like seasonality and trends can be captured in this manner. Such simple regression is horizon-less: it doesn't matter how far into the future we are predicting, because we are not using past data. In the previous example, the horizon was only used to split the data for cross-validation."
"We did not use lags in the previous model specification. In effect, the prediction was the result of a simple regression on date, time series identifier columns and any additional features. This is often a very good prediction as common time series patterns like seasonality and trends can be captured in this manner. Such simple regression is horizon-less: it doesn't matter how far into the future we are predicting, because we are not using past data. In the previous example, the horizon was only used to split the data for cross-validation."
]
},
{
@@ -558,7 +570,7 @@
"metadata": {},
"source": [
"### Using lags and rolling window features\n",
"Now we will configure the target lags, that is the previous values of the target variables, meaning the prediction is no longer horizon-less. We therefore must still specify the `max_horizon` that the model will learn to forecast. The `target_lags` keyword specifies how far back we will construct the lags of the target variable, and the `target_rolling_window_size` specifies the size of the rolling window over which we will generate the `max`, `min` and `sum` features.\n",
"Now we will configure the target lags, that is the previous values of the target variables, meaning the prediction is no longer horizon-less. We therefore must still specify the `forecast_horizon` that the model will learn to forecast. The `target_lags` keyword specifies how far back we will construct the lags of the target variable, and the `target_rolling_window_size` specifies the size of the rolling window over which we will generate the `max`, `min` and `sum` features.\n",
"\n",
"This notebook uses the blocked_models parameter to exclude some models that take a longer time to train on this dataset. You can choose to remove models from the blocked_models list but you may need to increase the iteration_timeout_minutes parameter value to get results."
]
@@ -569,12 +581,10 @@
"metadata": {},
"outputs": [],
"source": [
"automl_advanced_settings = {\n",
" 'time_column_name': time_column_name,\n",
" 'max_horizon': max_horizon,\n",
" 'target_lags': 12,\n",
" 'target_rolling_window_size': 4,\n",
"}\n",
"advanced_forecasting_parameters = ForecastingParameters(\n",
" time_column_name=time_column_name, forecast_horizon=forecast_horizon,\n",
" target_lags=12, target_rolling_window_size=4\n",
")\n",
"\n",
"automl_config = AutoMLConfig(task='forecasting', \n",
" primary_metric='normalized_root_mean_squared_error',\n",
@@ -586,7 +596,7 @@
" enable_early_stopping = True,\n",
" n_cross_validations=3, \n",
" verbosity=logging.INFO,\n",
" **automl_advanced_settings)"
" forecasting_parameters=advanced_forecasting_parameters)"
]
},
{
@@ -635,7 +645,7 @@
"metadata": {},
"source": [
"## Advanced Results<a id=\"advanced_results\"></a>\n",
"We did not use lags in the previous model specification. In effect, the prediction was the result of a simple regression on date, grain and any additional features. This is often a very good prediction as common time series patterns like seasonality and trends can be captured in this manner. Such simple regression is horizon-less: it doesn't matter how far into the future we are predicting, because we are not using past data. In the previous example, the horizon was only used to split the data for cross-validation."
"We did not use lags in the previous model specification. In effect, the prediction was the result of a simple regression on date, time series identifier columns and any additional features. This is often a very good prediction as common time series patterns like seasonality and trends can be captured in this manner. Such simple regression is horizon-less: it doesn't matter how far into the future we are predicting, because we are not using past data. In the previous example, the horizon was only used to split the data for cross-validation."
]
},
{

17
how-to-use-azureml/automated-machine-learning/forecasting-forecast-function/auto-ml-forecasting-function.ipynb
View File

@@ -94,7 +94,7 @@
"metadata": {},
"outputs": [],
"source": [
"print(\"This notebook was created using version 1.11.0 of the Azure ML SDK\")\n",
"print(\"This notebook was created using version 1.12.0 of the Azure ML SDK\")\n",
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
]
},
@@ -311,14 +311,15 @@
"metadata": {},
"outputs": [],
"source": [
"from azureml.automl.core.forecasting_parameters import ForecastingParameters\n",
"lags = [1,2,3]\n",
"forecast_horizon = n_test_periods\n",
"time_series_settings = { \n",
" 'time_column_name': TIME_COLUMN_NAME,\n",
" 'time_series_id_column_names': [ TIME_SERIES_ID_COLUMN_NAME ],\n",
" 'forecast_horizon': forecast_horizon ,\n",
" 'target_lags': lags\n",
"}"
"forecasting_parameters = ForecastingParameters(\n",
" time_column_name=TIME_COLUMN_NAME,\n",
" forecast_horizon=forecast_horizon,\n",
" time_series_id_column_names=[ TIME_SERIES_ID_COLUMN_NAME ],\n",
" target_lags=lags\n",
")"
]
},
{
@@ -351,7 +352,7 @@
" max_concurrent_iterations=4,\n",
" max_cores_per_iteration=-1,\n",
" label_column_name=target_label,\n",
" **time_series_settings)\n",
" forecasting_parameters=forecasting_parameters)\n",
"\n",
"remote_run = experiment.submit(automl_config, show_output=False)"
]

73
how-to-use-azureml/automated-machine-learning/forecasting-orange-juice-sales/auto-ml-forecasting-orange-juice-sales.ipynb
View File

@@ -82,7 +82,7 @@
"metadata": {},
"outputs": [],
"source": [
"print(\"This notebook was created using version 1.11.0 of the Azure ML SDK\")\n",
"print(\"This notebook was created using version 1.12.0 of the Azure ML SDK\")\n",
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
]
},
@@ -178,7 +178,7 @@
"source": [
"Each row in the DataFrame holds a quantity of weekly sales for an OJ brand at a single store. The data also includes the sales price, a flag indicating if the OJ brand was advertised in the store that week, and some customer demographic information based on the store location. For historical reasons, the data also include the logarithm of the sales quantity. The Dominick's grocery data is commonly used to illustrate econometric modeling techniques where logarithms of quantities are generally preferred. \n",
"\n",
"The task is now to build a time-series model for the _Quantity_ column. It is important to note that this dataset is comprised of many individual time-series - one for each unique combination of _Store_ and _Brand_. To distinguish the individual time-series, we thus define the **grain** - the columns whose values determine the boundaries between time-series: "
"The task is now to build a time-series model for the _Quantity_ column. It is important to note that this dataset is comprised of many individual time-series - one for each unique combination of _Store_ and _Brand_. To distinguish the individual time-series, we define the **time_series_id_column_names** - the columns whose values determine the boundaries between time-series: "
]
},
{
@@ -187,8 +187,8 @@
"metadata": {},
"outputs": [],
"source": [
"grain_column_names = ['Store', 'Brand']\n",
"nseries = data.groupby(grain_column_names).ngroups\n",
"time_series_id_column_names = ['Store', 'Brand']\n",
"nseries = data.groupby(time_series_id_column_names).ngroups\n",
"print('Data contains {0} individual time-series.'.format(nseries))"
]
},
@@ -207,7 +207,7 @@
"source": [
"use_stores = [2, 5, 8]\n",
"data_subset = data[data.Store.isin(use_stores)]\n",
"nseries = data_subset.groupby(grain_column_names).ngroups\n",
"nseries = data_subset.groupby(time_series_id_column_names).ngroups\n",
"print('Data subset contains {0} individual time-series.'.format(nseries))"
]
},
@@ -216,7 +216,7 @@
"metadata": {},
"source": [
"### Data Splitting\n",
"We now split the data into a training and a testing set for later forecast evaluation. The test set will contain the final 20 weeks of observed sales for each time-series. The splits should be stratified by series, so we use a group-by statement on the grain columns."
"We now split the data into a training and a testing set for later forecast evaluation. The test set will contain the final 20 weeks of observed sales for each time-series. The splits should be stratified by series, so we use a group-by statement on the time series identifier columns."
]
},
{
@@ -227,15 +227,15 @@
"source": [
"n_test_periods = 20\n",
"\n",
"def split_last_n_by_grain(df, n):\n",
" \"\"\"Group df by grain and split on last n rows for each group.\"\"\"\n",
"def split_last_n_by_series_id(df, n):\n",
" \"\"\"Group df by series identifiers and split on last n rows for each group.\"\"\"\n",
" df_grouped = (df.sort_values(time_column_name) # Sort by ascending time\n",
" .groupby(grain_column_names, group_keys=False))\n",
" .groupby(time_series_id_column_names, group_keys=False))\n",
" df_head = df_grouped.apply(lambda dfg: dfg.iloc[:-n])\n",
" df_tail = df_grouped.apply(lambda dfg: dfg.iloc[-n:])\n",
" return df_head, df_tail\n",
"\n",
"train, test = split_last_n_by_grain(data_subset, n_test_periods)"
"train, test = split_last_n_by_series_id(data_subset, n_test_periods)"
]
},
{
@@ -301,11 +301,11 @@
"For forecasting tasks, AutoML uses pre-processing and estimation steps that are specific to time-series. AutoML will undertake the following pre-processing steps:\n",
"* Detect time-series sample frequency (e.g. hourly, daily, weekly) and create new records for absent time points to make the series regular. A regular time series has a well-defined frequency and has a value at every sample point in a contiguous time span \n",
"* Impute missing values in the target (via forward-fill) and feature columns (using median column values) \n",
"* Create grain-based features to enable fixed effects across different series\n",
"* Create features based on time series identifiers to enable fixed effects across different series\n",
"* Create time-based features to assist in learning seasonal patterns\n",
"* Encode categorical variables to numeric quantities\n",
"\n",
"In this notebook, AutoML will train a single, regression-type model across **all** time-series in a given training set. This allows the model to generalize across related series. If you're looking for training multiple models for different time-series, please check out the forecasting grouping notebook. \n",
"In this notebook, AutoML will train a single, regression-type model across **all** time-series in a given training set. This allows the model to generalize across related series. If you're looking for training multiple models for different time-series, please see the many-models notebook.\n",
"\n",
"You are almost ready to start an AutoML training job. First, we need to separate the target column from the rest of the DataFrame: "
]
@@ -353,6 +353,21 @@
"featurization_config.add_transformer_params('Imputer', ['INCOME'], {\"strategy\": \"median\"})"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Forecasting Parameters\n",
"To define forecasting parameters for your experiment training, you can leverage the ForecastingParameters class. The table below details the forecasting parameter we will be passing into our experiment.\n",
"\n",
"\n",
"|Property|Description|\n",
"|-|-|\n",
"|**time_column_name**|The name of your time column.|\n",
"|**forecast_horizon**|The forecast horizon is how many periods forward you would like to forecast. This integer horizon is in units of the timeseries frequency (e.g. daily, weekly).|\n",
"|**time_series_id_column_names**|The column names used to uniquely identify the time series in data that has multiple rows with the same timestamp. If the time series identifiers are not defined, the data set is assumed to be one time series.|"
]
},
{
"cell_type": "markdown",
"metadata": {},
@@ -361,9 +376,9 @@
"\n",
"The [AutoMLConfig](https://docs.microsoft.com/en-us/python/api/azureml-train-automl-client/azureml.train.automl.automlconfig.automlconfig?view=azure-ml-py) object defines the settings and data for an AutoML training job. Here, we set necessary inputs like the task type, the number of AutoML iterations to try, the training data, and cross-validation parameters.\n",
"\n",
"For forecasting tasks, there are some additional parameters that can be set: the name of the column holding the date/time, the grain column names, and the maximum forecast horizon. A time column is required for forecasting, while the grain is optional. If grain columns are not given, AutoML assumes that the whole dataset is a single time-series. We also pass a list of columns to drop prior to modeling. The _logQuantity_ column is completely correlated with the target quantity, so it must be removed to prevent a target leak.\n",
"For forecasting tasks, there are some additional parameters that can be set in the `ForecastingParameters` class: the name of the column holding the date/time, the timeseries id column names, and the maximum forecast horizon. A time column is required for forecasting, while the time_series_id is optional. If time_series_id columns are not given, AutoML assumes that the whole dataset is a single time-series. We also pass a list of columns to drop prior to modeling. The _logQuantity_ column is completely correlated with the target quantity, so it must be removed to prevent a target leak.\n",
"\n",
"The forecast horizon is given in units of the time-series frequency; for instance, the OJ series frequency is weekly, so a horizon of 20 means that a trained model will estimate sales up to 20 weeks beyond the latest date in the training data for each series. In this example, we set the maximum horizon to the number of samples per series in the test set (n_test_periods). Generally, the value of this parameter will be dictated by business needs. For example, a demand planning application that estimates the next month of sales should set the horizon according to suitable planning time-scales. Please see the [energy_demand notebook](https://github.com/Azure/MachineLearningNotebooks/tree/master/how-to-use-azureml/automated-machine-learning/forecasting-energy-demand) for more discussion of forecast horizon.\n",
"The forecast horizon is given in units of the time-series frequency; for instance, the OJ series frequency is weekly, so a horizon of 20 means that a trained model will estimate sales up to 20 weeks beyond the latest date in the training data for each series. In this example, we set the forecast horizon to the number of samples per series in the test set (n_test_periods). Generally, the value of this parameter will be dictated by business needs. For example, a demand planning application that estimates the next month of sales should set the horizon according to suitable planning time-scales. Please see the [energy_demand notebook](https://github.com/Azure/MachineLearningNotebooks/tree/master/how-to-use-azureml/automated-machine-learning/forecasting-energy-demand) for more discussion of forecast horizon.\n",
"\n",
"We note here that AutoML can sweep over two types of time-series models:\n",
"* Models that are trained for each series such as ARIMA and Facebook's Prophet. Note that these models are only available for [Enterprise Edition Workspaces](https://docs.microsoft.com/en-us/azure/machine-learning/how-to-manage-workspace#upgrade).\n",
@@ -389,11 +404,8 @@
"|**enable_voting_ensemble**|Allow AutoML to create a Voting ensemble of the best performing models|\n",
"|**enable_stack_ensemble**|Allow AutoML to create a Stack ensemble of the best performing models|\n",
"|**debug_log**|Log file path for writing debugging information|\n",
"|**time_column_name**|Name of the datetime column in the input data|\n",
"|**grain_column_names**|Name(s) of the columns defining individual series in the input data|\n",
"|**max_horizon**|Maximum desired forecast horizon in units of time-series frequency|\n",
"|**featurization**| 'auto' / 'off' / FeaturizationConfig Indicator for whether featurization step should be done automatically or not, or whether customized featurization should be used. Setting this enables AutoML to perform featurization on the input to handle *missing data*, and to perform some common *feature extraction*.|\n",
"|**max_cores_per_iteration**|Maximum number of cores to utilize per iteration. A value of -1 indicates all available cores should be used.|"
"|**max_cores_per_iteration**|Maximum number of cores to utilize per iteration. A value of -1 indicates all available cores should be used"
]
},
{
@@ -402,11 +414,12 @@
"metadata": {},
"outputs": [],
"source": [
"time_series_settings = {\n",
" 'time_column_name': time_column_name,\n",
" 'grain_column_names': grain_column_names,\n",
" 'max_horizon': n_test_periods\n",
"}\n",
"from azureml.automl.core.forecasting_parameters import ForecastingParameters\n",
"forecasting_parameters = ForecastingParameters(\n",
" time_column_name=time_column_name,\n",
" forecast_horizon=n_test_periods,\n",
" time_series_id_column_names=time_series_id_column_names\n",
")\n",
"\n",
"automl_config = AutoMLConfig(task='forecasting',\n",
" debug_log='automl_oj_sales_errors.log',\n",
@@ -420,7 +433,7 @@
" n_cross_validations=3,\n",
" verbosity=logging.INFO,\n",
" max_cores_per_iteration=-1,\n",
" **time_series_settings)"
" forecasting_parameters=forecasting_parameters)"
]
},
{
@@ -537,9 +550,8 @@
"metadata": {},
"outputs": [],
"source": [
"# The featurized data, aligned to y, will also be returned.\n",
"# forecast returns the predictions and the featurized data, aligned to X_test.\n",
"# This contains the assumptions that were made in the forecast\n",
"# and helps align the forecast to the original data\n",
"y_predictions, X_trans = fitted_model.forecast(X_test)"
]
},
@@ -560,7 +572,7 @@
"\n",
"To evaluate the accuracy of the forecast, we'll compare against the actual sales quantities for some select metrics, included the mean absolute percentage error (MAPE). \n",
"\n",
"It is a good practice to always align the output explicitly to the input, as the count and order of the rows may have changed during transformations that span multiple rows."
"We'll add predictions and actuals into a single dataframe for convenience in calculating the metrics."
]
},
{
@@ -569,9 +581,8 @@
"metadata": {},
"outputs": [],
"source": [
"from forecasting_helper import align_outputs\n",
"\n",
"df_all = align_outputs(y_predictions, X_trans, X_test, y_test, target_column_name)"
"assign_dict = {'predicted': y_predictions, target_column_name: y_test}\n",
"df_all = X_test.assign(**assign_dict)"
]
},
{
@@ -794,5 +805,5 @@
"task": "Forecasting"
},
"nbformat": 4,
"nbformat_minor": 2
"nbformat_minor": 4
}

98
how-to-use-azureml/automated-machine-learning/forecasting-orange-juice-sales/forecasting_helper.py
View File

@@ -1,98 +0,0 @@
import pandas as pd
import numpy as np
from pandas.tseries.frequencies import to_offset
def align_outputs(y_predicted, X_trans, X_test, y_test, target_column_name,
predicted_column_name='predicted',
horizon_colname='horizon_origin'):
"""
Demonstrates how to get the output aligned to the inputs
using pandas indexes. Helps understand what happened if
the output's shape differs from the input shape, or if
the data got re-sorted by time and grain during forecasting.
Typical causes of misalignment are:
* we predicted some periods that were missing in actuals -> drop from eval
* model was asked to predict past max_horizon -> increase max horizon
* data at start of X_test was needed for lags -> provide previous periods
"""
if (horizon_colname in X_trans):
df_fcst = pd.DataFrame({predicted_column_name: y_predicted,
horizon_colname: X_trans[horizon_colname]})
else:
df_fcst = pd.DataFrame({predicted_column_name: y_predicted})
# y and X outputs are aligned by forecast() function contract
df_fcst.index = X_trans.index
# align original X_test to y_test
X_test_full = X_test.copy()
X_test_full[target_column_name] = y_test
# X_test_full's index does not include origin, so reset for merge
df_fcst.reset_index(inplace=True)
X_test_full = X_test_full.reset_index().drop(columns='index')
together = df_fcst.merge(X_test_full, how='right')
# drop rows where prediction or actuals are nan
# happens because of missing actuals
# or at edges of time due to lags/rolling windows
clean = together[together[[target_column_name,
predicted_column_name]].notnull().all(axis=1)]
return(clean)
def do_rolling_forecast(fitted_model, X_test, y_test, target_column_name, time_column_name, max_horizon, freq='D'):
"""
Produce forecasts on a rolling origin over the given test set.
Each iteration makes a forecast for the next 'max_horizon' periods
with respect to the current origin, then advances the origin by the
horizon time duration. The prediction context for each forecast is set so
that the forecaster uses the actual target values prior to the current
origin time for constructing lag features.
This function returns a concatenated DataFrame of rolling forecasts.
"""
df_list = []
origin_time = X_test[time_column_name].min()
while origin_time <= X_test[time_column_name].max():
# Set the horizon time - end date of the forecast
horizon_time = origin_time + max_horizon * to_offset(freq)
# Extract test data from an expanding window up-to the horizon
expand_wind = (X_test[time_column_name] < horizon_time)
X_test_expand = X_test[expand_wind]
y_query_expand = np.zeros(len(X_test_expand)).astype(np.float)
y_query_expand.fill(np.NaN)
if origin_time != X_test[time_column_name].min():
# Set the context by including actuals up-to the origin time
test_context_expand_wind = (X_test[time_column_name] < origin_time)
context_expand_wind = (
X_test_expand[time_column_name] < origin_time)
y_query_expand[context_expand_wind] = y_test[
test_context_expand_wind]
# Make a forecast out to the maximum horizon
y_fcst, X_trans = fitted_model.forecast(X_test_expand, y_query_expand)
# Align forecast with test set for dates within the
# current rolling window
trans_tindex = X_trans.index.get_level_values(time_column_name)
trans_roll_wind = (trans_tindex >= origin_time) & (
trans_tindex < horizon_time)
test_roll_wind = expand_wind & (
X_test[time_column_name] >= origin_time)
df_list.append(align_outputs(y_fcst[trans_roll_wind],
X_trans[trans_roll_wind],
X_test[test_roll_wind],
y_test[test_roll_wind],
target_column_name))
# Advance the origin time
origin_time = horizon_time
return pd.concat(df_list, ignore_index=True)

22
how-to-use-azureml/automated-machine-learning/forecasting-orange-juice-sales/metrics_helper.py
View File

@@ -1,22 +0,0 @@
import pandas as pd
import numpy as np
def APE(actual, pred):
"""
Calculate absolute percentage error.
Returns a vector of APE values with same length as actual/pred.
"""
return 100 * np.abs((actual - pred) / actual)
def MAPE(actual, pred):
"""
Calculate mean absolute percentage error.
Remove NA and values where actual is close to zero
"""
not_na = ~(np.isnan(actual) | np.isnan(pred))
not_zero = ~np.isclose(actual, 0.0)
actual_safe = actual[not_na & not_zero]
pred_safe = pred[not_na & not_zero]
return np.mean(APE(actual_safe, pred_safe))

19
how-to-use-azureml/automated-machine-learning/local-run-classification-credit-card-fraud/auto-ml-classification-credit-card-fraud-local.ipynb
View File

@@ -80,7 +80,7 @@
"from azureml.core.workspace import Workspace\n",
"from azureml.core.dataset import Dataset\n",
"from azureml.train.automl import AutoMLConfig\n",
"from azureml.explain.model._internal.explanation_client import ExplanationClient"
"from azureml.interpret._internal.explanation_client import ExplanationClient"
]
},
{
@@ -96,7 +96,7 @@
"metadata": {},
"outputs": [],
"source": [
"print(\"This notebook was created using version 1.11.0 of the Azure ML SDK\")\n",
"print(\"This notebook was created using version 1.12.0 of the Azure ML SDK\")\n",
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
]
},
@@ -354,7 +354,7 @@
"metadata": {},
"source": [
"## Explanation\n",
"In this section, we will show how to compute model explanations and visualize the explanations using azureml-explain-model package. We will also show how to run the automl model and the explainer model through deploying an AKS web service.\n",
"In this section, we will show how to compute model explanations and visualize the explanations using azureml-interpret package. We will also show how to run the automl model and the explainer model through deploying an AKS web service.\n",
"\n",
"Besides retrieving an existing model explanation for an AutoML model, you can also explain your AutoML model with different test data. The following steps will allow you to compute and visualize engineered feature importance based on your test data.\n",
"\n",
@@ -434,7 +434,7 @@
"metadata": {},
"source": [
"#### Initialize the Mimic Explainer for feature importance\n",
"For explaining the AutoML models, use the MimicWrapper from azureml.explain.model package. The MimicWrapper can be initialized with fields in automl_explainer_setup_obj, your workspace and a surrogate model to explain the AutoML model (fitted_model here). The MimicWrapper also takes the automl_run object where engineered explanations will be uploaded."
"For explaining the AutoML models, use the MimicWrapper from azureml-interpret package. The MimicWrapper can be initialized with fields in automl_explainer_setup_obj, your workspace and a surrogate model to explain the AutoML model (fitted_model here). The MimicWrapper also takes the automl_run object where engineered explanations will be uploaded."
]
},
{
@@ -443,7 +443,8 @@
"metadata": {},
"outputs": [],
"source": [
"from azureml.explain.model.mimic_wrapper import MimicWrapper\n",
"from interpret.ext.glassbox import LGBMExplainableModel\n",
"from azureml.interpret.mimic_wrapper import MimicWrapper\n",
"explainer = MimicWrapper(ws, automl_explainer_setup_obj.automl_estimator,\n",
" explainable_model=automl_explainer_setup_obj.surrogate_model, \n",
" init_dataset=automl_explainer_setup_obj.X_transform, run=automl_run,\n",
@@ -486,7 +487,7 @@
"metadata": {},
"outputs": [],
"source": [
"from azureml.explain.model.scoring.scoring_explainer import TreeScoringExplainer\n",
"from azureml.interpret.scoring.scoring_explainer import TreeScoringExplainer\n",
"import joblib\n",
"\n",
"# Initialize the ScoringExplainer\n",
@@ -507,7 +508,7 @@
"source": [
"### Deploying the scoring and explainer models to a web service to Azure Kubernetes Service (AKS)\n",
"\n",
"We use the TreeScoringExplainer from azureml.explain.model package to create the scoring explainer which will be used to compute the raw and engineered feature importances at the inference time. In the cell below, we register the AutoML model and the scoring explainer with the Model Management Service."
"We use the TreeScoringExplainer from azureml.interpret package to create the scoring explainer which will be used to compute the raw and engineered feature importances at the inference time. In the cell below, we register the AutoML model and the scoring explainer with the Model Management Service."
]
},
{
@@ -529,7 +530,7 @@
"source": [
"#### Create the conda dependencies for setting up the service\n",
"\n",
"We need to create the conda dependencies comprising of the azureml-explain-model, azureml-train-automl and azureml-defaults packages."
"We need to download the conda dependencies using the automl_run object."
]
},
{
@@ -566,7 +567,7 @@
"import os\n",
"import pickle\n",
"import azureml.train.automl\n",
"import azureml.explain.model\n",
"import azureml.interpret\n",
"from azureml.train.automl.runtime.automl_explain_utilities import AutoMLExplainerSetupClass, \\\n",
" automl_setup_model_explanations\n",
"import joblib\n",

8
how-to-use-azureml/automated-machine-learning/regression-explanation-featurization/auto-ml-regression-explanation-featurization.ipynb
View File

@@ -98,7 +98,7 @@
"metadata": {},
"outputs": [],
"source": [
"print(\"This notebook was created using version 1.11.0 of the Azure ML SDK\")\n",
"print(\"This notebook was created using version 1.12.0 of the Azure ML SDK\")\n",
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
]
},
@@ -625,7 +625,7 @@
"metadata": {},
"outputs": [],
"source": [
"from azureml.explain.model._internal.explanation_client import ExplanationClient\n",
"from azureml.interpret._internal.explanation_client import ExplanationClient\n",
"client = ExplanationClient.from_run(automl_run)\n",
"engineered_explanations = client.download_model_explanation(raw=False, comment='engineered explanations')\n",
"print(engineered_explanations.get_feature_importance_dict())\n",
@@ -659,7 +659,7 @@
"In this section we will show how you can operationalize an AutoML model and the explainer which was used to compute the explanations in the previous section.\n",
"\n",
"### Register the AutoML model and the scoring explainer\n",
"We use the *TreeScoringExplainer* from *azureml.explain.model* package to create the scoring explainer which will be used to compute the raw and engineered feature importances at the inference time. \n",
"We use the *TreeScoringExplainer* from *azureml-interpret* package to create the scoring explainer which will be used to compute the raw and engineered feature importances at the inference time. \n",
"In the cell below, we register the AutoML model and the scoring explainer with the Model Management Service."
]
},
@@ -681,7 +681,7 @@
"metadata": {},
"source": [
"### Create the conda dependencies for setting up the service\n",
"We need to create the conda dependencies comprising of the *azureml-explain-model*, *azureml-train-automl* and *azureml-defaults* packages. "
"We need to create the conda dependencies comprising of the *azureml* packages using the training environment from the *automl_run*."
]
},
{

2
how-to-use-azureml/automated-machine-learning/regression-explanation-featurization/score_explain.py
View File

@@ -4,7 +4,7 @@ import pandas as pd
import os
import pickle
import azureml.train.automl
import azureml.explain.model
import azureml.interpret
from azureml.train.automl.runtime.automl_explain_utilities import AutoMLExplainerSetupClass, \
automl_setup_model_explanations
import joblib

9
how-to-use-azureml/automated-machine-learning/regression-explanation-featurization/train_explainer.py
View File

@@ -7,12 +7,13 @@ from azureml.core.experiment import Experiment
from azureml.core.dataset import Dataset
from azureml.train.automl.runtime.automl_explain_utilities import AutoMLExplainerSetupClass, \
automl_setup_model_explanations, automl_check_model_if_explainable
from azureml.explain.model.mimic.models.lightgbm_model import LGBMExplainableModel
from azureml.explain.model.mimic_wrapper import MimicWrapper
from azureml.automl.core.shared.constants import MODEL_PATH
from azureml.explain.model.scoring.scoring_explainer import TreeScoringExplainer
from interpret.ext.glassbox import LGBMExplainableModel
from azureml.interpret.mimic_wrapper import MimicWrapper
from automl.client.core.common.constants import MODEL_PATH
from azureml.interpret.scoring.scoring_explainer import TreeScoringExplainer
import joblib
OUTPUT_DIR = './outputs/'
os.makedirs(OUTPUT_DIR, exist_ok=True)

2
how-to-use-azureml/automated-machine-learning/regression/auto-ml-regression.ipynb
View File

@@ -92,7 +92,7 @@
"metadata": {},
"outputs": [],
"source": [
"print(\"This notebook was created using version 1.11.0 of the Azure ML SDK\")\n",
"print(\"This notebook was created using version 1.12.0 of the Azure ML SDK\")\n",
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
]
},

10
how-to-use-azureml/machine-learning-pipelines/nyc-taxi-data-regression-model-building/nyc-taxi-data-regression-model-building.ipynb
View File

@@ -1,5 +1,13 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Copyright (c) Microsoft Corporation. All rights reserved. \n",
"Licensed under the MIT License."
]
},
{
"cell_type": "markdown",
"metadata": {},
@@ -1062,7 +1070,7 @@
"metadata": {
"authors": [
{
"name": "sanpil"
"name": "anshirga"
}
],
"kernelspec": {

8
how-to-use-azureml/machine-learning-pipelines/parallel-run/Code/iris_score.py
View File

@@ -6,10 +6,15 @@ import numpy as np
from azureml.core.model import Model
from sklearn.linear_model import LogisticRegression
from azureml_user.parallel_run import EntryScript
def init():
global iris_model
logger = EntryScript().logger
logger.info("init() is called.")
parser = argparse.ArgumentParser(description="Iris model serving")
parser.add_argument('--model_name', dest="model_name", required=True)
args, unknown_args = parser.parse_known_args()
@@ -20,6 +25,9 @@ def init():
def run(input_data):
logger = EntryScript().logger
logger.info("run() is called with: {}.".format(input_data))
# make inference
num_rows, num_cols = input_data.shape
pred = iris_model.predict(input_data).reshape((num_rows, 1))

86
how-to-use-azureml/machine-learning-pipelines/parallel-run/file-dataset-image-inference-mnist.ipynb
View File

@@ -51,7 +51,23 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Check core SDK version number\n",
"import azureml.core\n",
"\n",
"print(\"SDK version:\", azureml.core.VERSION)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"from azureml.core import Workspace\n",
@@ -397,7 +413,7 @@
" parallel_run_config=parallel_run_config,\n",
" inputs=[ input_mnist_ds_consumption ],\n",
" output=output_dir,\n",
" allow_reuse=True\n",
" allow_reuse=False\n",
")"
]
},
@@ -426,7 +442,9 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"### Monitor the run"
"### Monitor the run\n",
"\n",
"The pipeline run status could be checked in Azure Machine Learning portal (https://ml.azure.com). The link to the pipeline run could be retrieved by inspecting the `pipeline_run` object."
]
},
{
@@ -435,8 +453,8 @@
"metadata": {},
"outputs": [],
"source": [
"from azureml.widgets import RunDetails\n",
"RunDetails(pipeline_run).show()"
"# This will output information of the pipeline run, including the link to the details page of portal.\n",
"pipeline_run"
]
},
{
@@ -452,9 +470,40 @@
"metadata": {},
"outputs": [],
"source": [
"# Wait the run for completion and show output log to console\n",
"pipeline_run.wait_for_completion(show_output=True)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### View the prediction results per input image\n",
"In the digit_identification.py file above you can see that the ResultList with the filename and the prediction result gets returned. These are written to the DataStore specified in the PipelineData object as the output data, which in this case is called *inferences*. This containers the outputs from all of the worker nodes used in the compute cluster. You can download this data to view the results ... below just filters to the first 10 rows"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import tempfile\n",
"\n",
"batch_run = pipeline_run.find_step_run(parallelrun_step.name)[0]\n",
"batch_output = batch_run.get_output_data(output_dir.name)\n",
"\n",
"target_dir = tempfile.mkdtemp()\n",
"batch_output.download(local_path=target_dir)\n",
"result_file = os.path.join(target_dir, batch_output.path_on_datastore, parallel_run_config.append_row_file_name)\n",
"\n",
"df = pd.read_csv(result_file, delimiter=\":\", header=None)\n",
"df.columns = [\"Filename\", \"Prediction\"]\n",
"print(\"Prediction has \", df.shape[0], \" rows\")\n",
"df.head(10) "
]
},
{
"cell_type": "markdown",
"metadata": {},
@@ -492,15 +541,8 @@
"metadata": {},
"outputs": [],
"source": [
"pipeline_run_2.wait_for_completion(show_output=True)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### View the prediction results per input image\n",
"In the digit_identification.py file above you can see that the ResultList with the filename and the prediction result gets returned. These are written to the DataStore specified in the PipelineData object as the output data, which in this case is called *inferences*. This containers the outputs from all of the worker nodes used in the compute cluster. You can download this data to view the results ... below just filters to the first 10 rows"
"# This will output information of the pipeline run, including the link to the details page of portal.\n",
"pipeline_run_2"
]
},
{
@@ -509,20 +551,8 @@
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import tempfile\n",
"\n",
"batch_run = pipeline_run.find_step_run(parallelrun_step.name)[0]\n",
"batch_output = batch_run.get_output_data(output_dir.name)\n",
"\n",
"target_dir = tempfile.mkdtemp()\n",
"batch_output.download(local_path=target_dir)\n",
"result_file = os.path.join(target_dir, batch_output.path_on_datastore, parallel_run_config.append_row_file_name)\n",
"\n",
"df = pd.read_csv(result_file, delimiter=\":\", header=None)\n",
"df.columns = [\"Filename\", \"Prediction\"]\n",
"print(\"Prediction has \", df.shape[0], \" rows\")\n",
"df.head(10) "
"# Wait the run for completion and show output log to console\n",
"pipeline_run_2.wait_for_completion(show_output=True)"
]
},
{

54
how-to-use-azureml/machine-learning-pipelines/parallel-run/tabular-dataset-inference-iris.ipynb
View File

@@ -49,7 +49,23 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Check core SDK version number\n",
"import azureml.core\n",
"\n",
"print(\"SDK version:\", azureml.core.VERSION)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"from azureml.core import Workspace\n",
@@ -318,7 +334,7 @@
"parallel_run_config = ParallelRunConfig(\n",
" source_directory=scripts_folder,\n",
" entry_script=script_file, # the user script to run against each input\n",
" mini_batch_size='5MB',\n",
" mini_batch_size='1KB',\n",
" error_threshold=5,\n",
" output_action='append_row',\n",
" append_row_file_name=\"iris_outputs.txt\",\n",
@@ -349,7 +365,7 @@
" output=output_folder,\n",
" parallel_run_config=parallel_run_config,\n",
" arguments=['--model_name', 'iris-prs'],\n",
" allow_reuse=True\n",
" allow_reuse=False\n",
")"
]
},
@@ -375,13 +391,22 @@
"pipeline_run = Experiment(ws, 'iris-prs').submit(pipeline)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## View progress of Pipeline run\n",
"\n",
"The pipeline run status could be checked in Azure Machine Learning portal (https://ml.azure.com). The link to the pipeline run could be retrieved by inspecting the `pipeline_run` object."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# this will output a table with link to the run details in azure portal\n",
"# This will output information of the pipeline run, including the link to the details page of portal.\n",
"pipeline_run"
]
},
@@ -389,29 +414,18 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"## View progress of Pipeline run\n",
"\n",
"The progress of the pipeline is able to be viewed either through azureml.widgets or a console feed from PipelineRun.wait_for_completion()."
"### Optional: View detailed logs (streaming) "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# GUI\n",
"from azureml.widgets import RunDetails\n",
"RunDetails(pipeline_run).show() "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Console logs\n",
"## Wait the run for completion and show output log to console\n",
"pipeline_run.wait_for_completion(show_output=True)"
]
},

43
how-to-use-azureml/machine-learning-pipelines/pipeline-style-transfer/pipeline-style-transfer-parallel-run.ipynb
View File

@@ -461,7 +461,7 @@
" output=processed_images, # Output file share/blob container\n",
" arguments=[\"--style\", style_param],\n",
" parallel_run_config=parallel_run_config,\n",
" allow_reuse=True #[optional - default value True]\n",
" allow_reuse=False #[optional - default value True]\n",
")"
]
},
@@ -497,7 +497,10 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"# Monitor using widget"
"# Monitor pipeline run\n",
"\n",
"The pipeline run status could be checked in Azure Machine Learning portal (https://ml.azure.com). The link to the pipeline run could be retrieved by inspecting the `pipeline_run` object.\n",
"\n"
]
},
{
@@ -506,25 +509,25 @@
"metadata": {},
"outputs": [],
"source": [
"# Track pipeline run progress\n",
"from azureml.widgets import RunDetails\n",
"RunDetails(pipeline_run).show()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"pipeline_run.wait_for_completion()"
"# This will output information of the pipeline run, including the link to the details page of portal.\n",
"pipeline_run"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Downloads the video in `output_video` folder"
"### Optional: View detailed logs (streaming) "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Wait the run for completion and show output log to console\n",
"pipeline_run.wait_for_completion(show_output=True)"
]
},
{
@@ -534,6 +537,13 @@
"# Download output video"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Downloads the video in `output_video` folder"
]
},
{
"cell_type": "code",
"execution_count": null,
@@ -674,7 +684,8 @@
"from azureml.pipeline.core.run import PipelineRun\n",
"published_pipeline_run_candy = PipelineRun(ws.experiments[experiment_name], run_id)\n",
"\n",
"RunDetails(published_pipeline_run_candy).show()"
"# Show detail information of run\n",
"published_pipeline_run_candy"
]
},
{

38
how-to-use-azureml/ml-frameworks/tensorflow/deployment/train-hyperparameter-tune-deploy-with-tensorflow/train-hyperparameter-tune-deploy-with-tensorflow.ipynb
View File

@@ -630,52 +630,44 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"## Predict on the test set\n",
"## Predict on the test set (Optional)\n",
"Now load the saved TensorFlow graph, and list all operations under the `network` scope. This way we can discover the input tensor `network/X:0` and the output tensor `network/output/MatMul:0`, and use them in the scoring script in the next step.\n",
"\n",
"Note: if your local TensorFlow version is different than the version running in the cluster where the model is trained, you might see a \"compiletime version mismatch\" warning. You can ignore it."
]
},
{
"cell_type": "code",
"execution_count": null,
"cell_type": "markdown",
"metadata": {},
"outputs": [],
"source": [
"import tensorflow as tf\n",
"imported_model = tf.saved_model.load('./model')"
" import tensorflow as tf\n",
" imported_model = tf.saved_model.load('./model')"
]
},
{
"cell_type": "code",
"execution_count": null,
"cell_type": "markdown",
"metadata": {},
"outputs": [],
"source": [
"pred =imported_model(X_test)\n",
"y_hat = np.argmax(pred, axis=1)\n",
" pred = imported_model(X_test)\n",
" y_hat = np.argmax(pred, axis=1)\n",
"\n",
"# print the first 30 labels and predictions\n",
"print('labels: \\t', y_test[:30])\n",
"print('predictions:\\t', y_hat[:30])"
" # print the first 30 labels and predictions\n",
" print('labels: \\t', y_test[:30])\n",
" print('predictions:\\t', y_hat[:30])"
]
},
{
"cell_type": "code",
"execution_count": null,
"cell_type": "markdown",
"metadata": {},
"outputs": [],
"source": [
"print(\"Accuracy on the test set:\", np.average(y_hat == y_test))"
" print(\"Accuracy on the test set:\", np.average(y_hat == y_test))"
]
},
{
"cell_type": "code",
"execution_count": null,
"cell_type": "markdown",
"metadata": {},
"outputs": [],
"source": [
"print(\"Accuracy on the test set:\", np.average(y_hat == y_test))"
" print(\"Accuracy on the test set:\", np.average(y_hat == y_test))"
]
},
{
@@ -1055,7 +1047,7 @@
" font_color = 'red' if y_test[s] != result[i] else 'black'\n",
" clr_map = plt.cm.gray if y_test[s] != result[i] else plt.cm.Greys\n",
" \n",
" plt.text(x=10, y=-10, s=y_hat[s], fontsize=18, color=font_color)\n",
" plt.text(x=10, y=-10, s=result[i], fontsize=18, color=font_color)\n",
" plt.imshow(X_test[s].reshape(28, 28), cmap=clr_map)\n",
" \n",
" i = i + 1\n",

169
how-to-use-azureml/monitor-models/data-drift/drift-on-aks.ipynb
View File

@@ -264,6 +264,58 @@
"print(prediction)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Wait 15 minutes for scoring data to be uploaded\n",
"\n",
"From the Model Data Collector, it can take up to (but usually less than) 15 minutes for data to arrive in your blob storage account. \n",
"\n",
"Wait 15 minutes to ensure cells below will run."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import time\n",
"\n",
"time.sleep(900)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Get scoring dataset thats been created\n",
"\n",
"Scoring dataset will be created automatically for each model/version/service that has been deployed and registered with name in the format of inference-data-elevation-{model_name}-{model_version}-{service_name}\n",
"\n",
"Wait 15 minutes to ensure cells below will run."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"scoring_dataset_name = \"inference-data-{0}-{1}-{2}\".format(model.name, model.version, service_name)\n",
"scoring_dataset = Dataset.get_by_name(ws, scoring_dataset_name)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Create datadset monitor for scoring dataset against training dataset\n",
"\n",
"Check out [datadrift on dataset notebook](../../work-with-data/datadrift-tutorial/datadrift-tutorial.ipynb) for more details"
]
},
{
"cell_type": "markdown",
"metadata": {},
@@ -302,26 +354,6 @@
" print(compute_target.get_status().serialize())"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Wait 10 minutes\n",
"\n",
"From the Model Data Collector, it can take up to (but usually less than) 10 minutes for data to arrive in your blob storage account. Wait 10 minutes to ensure cells below will run."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import time\n",
"\n",
"time.sleep(600)"
]
},
{
"cell_type": "markdown",
"metadata": {},
@@ -335,22 +367,23 @@
"metadata": {},
"outputs": [],
"source": [
"from datetime import datetime, timedelta\n",
"from azureml.datadrift import DataDriftDetector, AlertConfiguration\n",
"\n",
"services = [service_name]\n",
"start = datetime.now() - timedelta(days=2)\n",
"feature_list = X_features\n",
"alert_config = AlertConfiguration(['user@contoso.com'])\n",
"alert_config = AlertConfiguration(['user@contoso.com']) # replace with your email to recieve alerts from the scheduled pipeline after enabling\n",
"monitor_name = \"monitor_model_demo\"\n",
"baseline = dset # training dataset\n",
"target = scoring_dataset # scording dataset\n",
"\n",
"try:\n",
" monitor = DataDriftDetector.create_from_model(ws, model.name, model.version, services, \n",
" frequency='Day', \n",
" schedule_start=datetime.utcnow() + timedelta(days=1), \n",
" alert_config=alert_config, \n",
" compute_target='cpu-cluster')\n",
" monitor = DataDriftDetector.create_from_datasets(ws, monitor_name, baseline, target, \n",
" compute_target='cpu-cluster', # compute target for scheduled pipeline and backfills \n",
" frequency='Day', # how often to analyze target data\n",
" feature_list=None, # list of features to detect drift on\n",
" drift_threshold=None, # threshold from 0 to 1 for email alerting\n",
" latency=0, # SLA in hours for target data to arrive in the dataset\n",
" alert_config=alert_config) # email addresses to send alert\n",
"except KeyError:\n",
" monitor = DataDriftDetector.get(ws, model.name, model.version)\n",
" monitor = DataDriftDetector.get_by_name(ws, monitor_name)\n",
" \n",
"monitor"
]
@@ -362,7 +395,7 @@
"outputs": [],
"source": [
"# many monitor settings can be updated \n",
"monitor = monitor.update(drift_threshold = 0.1)\n",
"monitor = monitor.update(drift_threshold = 0.1, feature_list = X_features)\n",
"\n",
"monitor"
]
@@ -371,7 +404,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"## Run the monitor on today's scoring data\n",
"## Analyze today's scoring data\n",
"\n",
"Perform a data drift run on the data sent to the service earlier in this notebook. If you set your email address in the alert configuration and the drift threshold <=0.1 you should recieve an email alert to drift from this run.\n",
"\n",
@@ -384,16 +417,37 @@
"metadata": {},
"outputs": [],
"source": [
"from datetime import datetime\n",
"\n",
"now = datetime.utcnow()\n",
"target_date = datetime(now.year, now.month, now.day)\n",
"run = monitor.run(target_date, services, feature_list=feature_list, compute_target='cpu-cluster')"
"analysis_run = monitor.backfill(target_date, target_date)\n",
"analysis_run"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Get and view results and metrics\n",
"## Query metrics and show results in Python\n",
"\n",
"The below cell will plot some key data drift metrics, and can be used to query the results. Run `help(monitor.get_output)` for specifics on the object returned."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"analysis_run.wait_for_completion(wait_post_processing=True)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Get and view results and metrics\n",
"\n",
"For enterprise workspaces, the UI in the Azure Machine Learning studio can be used. Otherwise, the metrics can be queried in Python and plotted. "
]
@@ -404,11 +458,8 @@
"metadata": {},
"outputs": [],
"source": [
"# The run() API initiates a pipeline run for each service in the services list. \n",
"# Here we retrieve the individual service run to get its output results and metrics. \n",
"\n",
"child_run = list(run.get_children())[0]\n",
"child_run"
"# get results from Python SDK after the analysis run finishes\n",
"results, metrics = monitor.get_output(start_time=target_date, end_time=target_date)"
]
},
{
@@ -417,25 +468,8 @@
"metadata": {},
"outputs": [],
"source": [
"child_run.wait_for_completion(wait_post_processing=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"results, metrics = monitor.get_output(run_id=child_run.id)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"drift_plots = monitor.show()"
"# plot the results from Python SDK \n",
"monitor.show(start_time=target_date, end_time=target_date)"
]
},
{
@@ -453,7 +487,11 @@
"metadata": {},
"outputs": [],
"source": [
"monitor.enable_schedule()"
"# enable the pipeline schedule and recieve email alerts\n",
"monitor.enable_schedule()\n",
"\n",
"# disable the pipeline schedule \n",
"#monitor.disable_schedule()"
]
},
{
@@ -484,13 +522,6 @@
" * [Send requests or feedback](mailto:driftfeedback@microsoft.com) on data drift directly to the team\n",
" * Please open issues with data drift here on GitHub or on StackOverflow if others are likely to run into the same issue"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
@@ -514,7 +545,7 @@
"Azure ML"
],
"friendly_name": "Data drift on aks",
"index_order": 1.0,
"index_order": 1,
"kernelspec": {
"display_name": "Python 3.6",
"language": "python",
@@ -530,7 +561,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.4"
"version": "3.6.10"
},
"star_tag": [
"featured"

27
how-to-use-azureml/reinforcement-learning/cartpole-on-single-compute/cartpole_sc.ipynb
View File

@@ -254,6 +254,7 @@
" dockerfile=f.read()\n",
"\n",
" xvfb_env = Environment(name='xvfb-vdisplay')\n",
" xvfb_env.docker.enabled = True\n",
" xvfb_env.docker.base_image = None\n",
" xvfb_env.docker.base_dockerfile = dockerfile\n",
" \n",
@@ -547,29 +548,18 @@
"source": [
"import shutil\n",
"\n",
"# A helper function to download (copy) movies from a dataset to local directory\n",
"# A helper function to download movies from a dataset to local directory\n",
"def download_movies(artifacts_ds, movies, destination):\n",
" # Create the local destination directory \n",
" if path.exists(destination):\n",
" dir_util.remove_tree(destination)\n",
" dir_util.mkpath(destination)\n",
" \n",
" try:\n",
" print(\"Trying mounting dataset and copying movies.\")\n",
" # Note: We assume movie paths start with '\\'\n",
" mount_context = artifacts_ds.mount()\n",
" mount_context.start()\n",
" for movie in movies:\n",
" print('Copying {} ...'.format(movie))\n",
" shutil.copy2(path.join(mount_context.mount_point, movie[1:]), destination)\n",
" mount_context.stop()\n",
" except OSError as e:\n",
" print(\"Mounting failed with error '{0}'. Going with dataset download.\".format(e))\n",
" for i, artifact in enumerate(artifacts_ds.to_path()):\n",
" if artifact in movies:\n",
" print('Downloading {} ...'.format(artifact))\n",
" artifacts_ds.skip(i).take(1).download(target_path=destination, overwrite=True)\n",
" \n",
"\n",
" for i, artifact in enumerate(artifacts_ds.to_path()):\n",
" if artifact in movies:\n",
" print('Downloading {} ...'.format(artifact))\n",
" artifacts_ds.skip(i).take(1).download(target_path=destination, overwrite=True)\n",
"\n",
" print('Downloading movies completed!')\n",
"\n",
"\n",
@@ -782,6 +772,7 @@
" dockerfile=f.read()\n",
"\n",
"xvfb_env = Environment(name='xvfb-vdisplay')\n",
"xvfb_env.docker.enabled = True\n",
"xvfb_env.docker.base_image = None\n",
"xvfb_env.docker.base_dockerfile = dockerfile\n",
" \n",

4
how-to-use-azureml/reinforcement-learning/cartpole-on-single-compute/files/docker/Dockerfile
View File

@@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
rm -rf /var/lib/apt/lists/* && \
rm -rf /usr/share/man/*
RUN conda install -y conda=4.7.12 python=3.6.2 && conda clean -ay && \
RUN conda install -y conda=4.7.12 python=3.7 && conda clean -ay && \
pip install --no-cache-dir \
azureml-defaults \
azureml-dataset-runtime[fuse,pandas] \
@@ -26,4 +26,4 @@ RUN conda install -y conda=4.7.12 python=3.6.2 && conda clean -ay && \
setproctitle \
gym[atari] && \
conda install -y -c conda-forge x264='1!152.20180717' ffmpeg=4.0.2 && \
conda install opencv
conda install -c anaconda opencv

2
how-to-use-azureml/track-and-monitor-experiments/README.md
View File

@@ -12,6 +12,8 @@
Try out the sample notebooks:
1. [Use MLflow with Azure Machine Learning for Local Training Run](./using-mlflow/train-local/train-local.ipynb)
1. [Use MLflow with Azure Machine Learning for Remote Training Run](./using-mlflow/train-remote/train-remote.ipynb)
1. [Train and Deploy PyTorch Image Classifier](./using-mlflow/train-and-deploy-pytorch/train-and-deploy-pytorch.ipynb)
1. [Train and Deploy Keras Image Classifier with MLflow auto logging](./using-mlflow/train-and-deploy-keras-auto-logging/train-and-deploy-keras-auto-logging.ipynb)
![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/track-and-monitor-experiments/README.png)

2
how-to-use-azureml/track-and-monitor-experiments/logging-api/logging-api.ipynb
View File

@@ -100,7 +100,7 @@
"\n",
"# Check core SDK version number\n",
"\n",
"print(\"This notebook was created using SDK version 1.11.0, you are currently running version\", azureml.core.VERSION)"
"print(\"This notebook was created using SDK version 1.12.0, you are currently running version\", azureml.core.VERSION)"
]
},
{

78
how-to-use-azureml/track-and-monitor-experiments/using-mlflow/train-and-deploy-keras-auto-logging/scripts/train.py Normal file
View File

@@ -0,0 +1,78 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
import mlflow
import mlflow.keras
import numpy as np
import warnings
import keras
from keras.models import Sequential
from keras.layers import Dense
from keras.optimizers import RMSprop
print("Keras version:", keras.__version__)
# Enable auto-logging to MLflow to capture Keras metrics.
mlflow.keras.autolog()
# Model / data parameters
n_inputs = 28 * 28
n_h1 = 300
n_h2 = 100
n_outputs = 10
learning_rate = 0.001
# the data, split between train and test sets
(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()
# Scale images to the [0, 1] range
x_train = x_train.astype("float32") / 255
x_test = x_test.astype("float32") / 255
# Flatten image to be (n, 28 * 28)
x_train = x_train.reshape(len(x_train), -1)
x_test = x_test.reshape(len(x_test), -1)
print("x_train shape:", x_train.shape)
print(x_train.shape[0], "train samples")
print(x_test.shape[0], "test samples")
# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, n_outputs)
y_test = keras.utils.to_categorical(y_test, n_outputs)
def driver():
warnings.filterwarnings("ignore")
with mlflow.start_run() as run:
# Build a simple MLP model
model = Sequential()
# first hidden layer
model.add(Dense(n_h1, activation='relu', input_shape=(n_inputs,)))
# second hidden layer
model.add(Dense(n_h2, activation='relu'))
# output layer
model.add(Dense(n_outputs, activation='softmax'))
model.summary()
batch_size = 128
epochs = 5
model.compile(loss='categorical_crossentropy',
optimizer=RMSprop(lr=learning_rate),
metrics=['accuracy'])
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_split=0.1)
score = model.evaluate(x_test, y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
return run
if __name__ == "__main__":
driver()

455
how-to-use-azureml/track-and-monitor-experiments/using-mlflow/train-and-deploy-keras-auto-logging/train-and-deploy-keras-auto-logging.ipynb Normal file
View File

@@ -0,0 +1,455 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Copyright (c) Microsoft Corporation. All rights reserved.\n",
"\n",
"Licensed under the MIT License."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/using-mlflow/train-and-deploy-keras-auto-logging/train-and-deploy-keras-auto-logging.png)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Use MLflow with Azure Machine Learning to Train and Deploy Keras Image Classifier\n",
"\n",
"This example shows you how to use MLflow together with Azure Machine Learning services for tracking the metrics and artifacts while training a Keras model to classify MNIST digit images and deploy the model as a web service. You'll learn how to:\n",
"\n",
" 1. Set up MLflow tracking URI so as to use Azure ML\n",
" 2. Create experiment\n",
" 3. Instrument your model with MLflow tracking\n",
" 4. Train a Keras model locally with MLflow auto logging\n",
" 5. Train a model on GPU compute on Azure with MLflow auto logging\n",
" 6. View your experiment within your Azure ML Workspace in Azure Portal\n",
" 7. Deploy the model as a web service on Azure Container Instance\n",
" 8. Call the model to make predictions\n",
" \n",
"### Pre-requisites\n",
" \n",
"If you are using a Notebook VM, you are all set. Otherwise, go through the [Configuration](../../../../configuration.ipnyb) notebook to set up your Azure Machine Learning workspace and ensure other common prerequisites are met.\n",
"\n",
"Install TensorFlow and Keras, this notebook has been tested with TensorFlow version 2.1.0 and Keras version 2.3.1.\n",
"\n",
"Also, install azureml-mlflow package using ```pip install azureml-mlflow```. Note that azureml-mlflow installs mlflow package itself as a dependency if you haven't done so previously.\n",
"\n",
"### Set-up\n",
"\n",
"Import packages and check versions of Azure ML SDK and MLflow installed on your computer. Then connect to your Workspace."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import sys, os\n",
"import mlflow\n",
"import mlflow.azureml\n",
"\n",
"import azureml.core\n",
"from azureml.core import Workspace\n",
"\n",
"\n",
"print(\"SDK version:\", azureml.core.VERSION)\n",
"print(\"MLflow version:\", mlflow.version.VERSION)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"ws = Workspace.from_config()\n",
"ws.get_details()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Set tracking URI\n",
"\n",
"Set the MLflow tracking URI to point to your Azure ML Workspace. The subsequent logging calls from MLflow APIs will go to Azure ML services and will be tracked under your Workspace."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"mlflow.set_tracking_uri(ws.get_mlflow_tracking_uri())"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Create Experiment\n",
"\n",
"In both MLflow and Azure ML, training runs are grouped into experiments. Let's create one for our experimentation."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"experiment_name = \"keras-with-mlflow\"\n",
"mlflow.set_experiment(experiment_name)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Train model locally while logging metrics and artifacts\n",
"\n",
"The ```scripts/train.py``` program contains the code to load the image dataset, train and test the model. Within this program, the train.driver function wraps the end-to-end workflow.\n",
"\n",
"Within the driver, the ```mlflow.start_run``` starts MLflow tracking. Then, MLflow's automatic logging is used to log metrics, parameters and model for the Keras run.\n",
"\n",
"Let's add the program to search path, import it as a module and invoke the driver function. Note that the training can take few minutes."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"lib_path = os.path.abspath(\"scripts\")\n",
"sys.path.append(lib_path)\n",
"\n",
"import train"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"run = train.driver()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Train model on GPU compute on Azure\n",
"\n",
"Next, let's run the same script on GPU-enabled compute for faster training. If you've completed the the [Configuration](../../../configuration.ipnyb) notebook, you should have a GPU cluster named \"gpu-cluster\" available in your workspace. Otherwise, follow the instructions in the notebook to create one. For simplicity, this example uses single process on single VM to train the model.\n",
"\n",
"Clone an environment object from the Tensorflow 2.1 Azure ML curated environment. Azure ML curated environments are pre-configured environments to simplify ML setup, reference [this doc](https://docs.microsoft.com/en-us/azure/machine-learning/how-to-use-environments#use-a-curated-environment) for more information. To enable MLflow tracking, add ```azureml-mlflow``` as pip package."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from azureml.core import Environment\n",
"\n",
"env = Environment.get(workspace=ws, name=\"AzureML-TensorFlow-2.1-GPU\").clone(\"mlflow-env\")\n",
"\n",
"env.python.conda_dependencies.add_pip_package(\"azureml-mlflow\")\n",
"env.python.conda_dependencies.add_pip_package(\"keras==2.3.1\")\n",
"env.python.conda_dependencies.add_pip_package(\"numpy\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Create a ScriptRunConfig to specify the training configuration: script, compute as well as environment."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from azureml.core import ScriptRunConfig\n",
"\n",
"src = ScriptRunConfig(source_directory=\"./scripts\", script=\"train.py\")\n",
"src.run_config.environment = env\n",
"src.run_config.target = \"gpu-cluster\""
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Get a reference to the experiment you created previously, but this time, as an Azure Machine Learning experiment object.\n",
"\n",
"Then, use the ```Experiment.submit``` method to start the remote training run. Note that the first training run often takes longer as Azure Machine Learning service builds the Docker image for executing the script. Subsequent runs will be faster as the cached image is used."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from azureml.core import Experiment\n",
"\n",
"exp = Experiment(ws, experiment_name)\n",
"run = exp.submit(src)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"You can monitor the run and its metrics on Azure Portal."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"run"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Also, you can wait for run to complete."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"run.wait_for_completion(show_output=True)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Deploy model as web service\n",
"\n",
"The ```mlflow.azureml.deploy``` function registers the logged Keras+Tensorflow model and deploys the model in a framework-aware manner. It automatically creates the Tensorflow-specific inferencing wrapper code and specifies package dependencies for you. See [this doc](https://mlflow.org/docs/latest/models.html#id34) for more information on deploying models on Azure ML using MLflow.\n",
"\n",
"In this example, we deploy the Docker image to Azure Container Instance: a serverless compute capable of running a single container. You can tag and add descriptions to help keep track of your web service. \n",
"\n",
"[Other inferencing compute choices](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-deploy-and-where) include Azure Kubernetes Service which provides scalable endpoint suitable for production use.\n",
"\n",
"Note that the service deployment can take several minutes."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from azureml.core.webservice import AciWebservice, Webservice\n",
"\n",
"model_path = \"model\"\n",
"\n",
"aci_config = AciWebservice.deploy_configuration(cpu_cores=2, \n",
" memory_gb=5, \n",
" tags={\"data\": \"MNIST\", \"method\" : \"keras\"}, \n",
" description=\"Predict using webservice\")\n",
"\n",
"webservice, azure_model = mlflow.azureml.deploy(model_uri='runs:/{}/{}'.format(run.id, model_path),\n",
" workspace=ws,\n",
" deployment_config=aci_config,\n",
" service_name=\"keras-mnist-1\",\n",
" model_name=\"keras_mnist\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Once the deployment has completed you can check the scoring URI of the web service."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(\"Scoring URI is: {}\".format(webservice.scoring_uri))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"In case of a service creation issue, you can use ```webservice.get_logs()``` to get logs to debug."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Make predictions using a web service\n",
"\n",
"To make the web service, create a test data set as normalized NumPy array. \n",
"\n",
"Then, let's define a utility function that takes a random image and converts it into a format and shape suitable for input to the Keras inferencing end-point. The conversion is done by: \n",
"\n",
" 1. Select a random (image, label) tuple\n",
" 2. Take the image and converting to to NumPy array \n",
" 3. Reshape array into 1 x 1 x N array\n",
" * 1 image in batch, 1 color channel, N = 784 pixels for MNIST images\n",
" * Note also ```x = x.view(-1, 1, 28, 28)``` in net definition in ```train.py``` program to shape incoming scoring requests.\n",
" 4. Convert the NumPy array to list to make it into a built-in type.\n",
" 5. Create a dictionary {\"data\", &lt;list&gt;} that can be converted to JSON string for web service requests."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import keras\n",
"import random\n",
"import numpy as np\n",
"\n",
"# the data, split between train and test sets\n",
"(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()\n",
"\n",
"# Scale images to the [0, 1] range\n",
"x_test = x_test.astype(\"float32\") / 255\n",
"x_test = x_test.reshape(len(x_test), -1)\n",
"\n",
"# convert class vectors to binary class matrices\n",
"y_test = keras.utils.to_categorical(y_test, 10)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%matplotlib inline\n",
"\n",
"import json\n",
"import matplotlib.pyplot as plt\n",
"\n",
"# send a random row from the test set to score\n",
"random_index = np.random.randint(0, len(x_test)-1)\n",
"input_data = \"{\\\"data\\\": [\" + str(list(x_test[random_index])) + \"]}\"\n",
"\n",
"response = webservice.run(input_data)\n",
"\n",
"response = sorted(response[0].items(), key = lambda x: x[1], reverse = True)\n",
"\n",
"print(\"Predicted label:\", response[0][0])\n",
"plt.imshow(x_test[random_index].reshape(28,28), cmap = \"gray\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"You can also call the web service using a raw POST method against the web service"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import requests\n",
"\n",
"response = requests.post(url=webservice.scoring_uri, data=input_data,headers={\"Content-type\": \"application/json\"})\n",
"print(response.text)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Clean up\n",
"You can delete the ACI deployment with a delete API call."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"webservice.delete()"
]
}
],
"metadata": {
"authors": [
{
"name": "hancwang"
}
],
"category": "tutorial",
"celltoolbar": "Edit Metadata",
"compute": [
"Local",
"AML Compute"
],
"datasets": [
"MNIST"
],
"deployment": [
"Azure Container Instance"
],
"exclude_from_index": false,
"framework": [
"Keras"
],
"friendly_name": "Use MLflow with Azure Machine Learning to Train and Deploy Keras Image Classifier",
"kernelspec": {
"display_name": "Python 3.6",
"language": "python",
"name": "python36"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.7"
},
"tags": [
"mlflow",
"keras"
],
"task": "Use MLflow with Azure Machine Learning to Train and Deploy Keras Image Classifier, leveraging MLflow auto logging"
},
"nbformat": 4,
"nbformat_minor": 4
}

9
how-to-use-azureml/track-and-monitor-experiments/using-mlflow/train-and-deploy-keras-auto-logging/train-and-deploy-keras-auto-logging.yml Normal file
View File

@@ -0,0 +1,9 @@
name: train-and-deploy-keras-auto-logging
dependencies:
- pip:
- azureml-sdk
- numpy
- azureml-mlflow
- matplotlib
- tensorflow==2.1
- keras

150
how-to-use-azureml/track-and-monitor-experiments/using-mlflow/train-and-deploy-pytorch/scripts/train.py Normal file
View File

@@ -0,0 +1,150 @@
# Copyright (c) 2017, PyTorch Team
# All rights reserved
# Licensed under BSD 3-Clause License.
# This example is based on PyTorch MNIST example:
# https://github.com/pytorch/examples/blob/master/mnist/main.py
import mlflow
import mlflow.pytorch
from mlflow.utils.environment import _mlflow_conda_env
import warnings
import cloudpickle
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torchvision
from torchvision import datasets, transforms
class Net(nn.Module):
def __init__(self):
super(Net, self).__init__()
self.conv1 = nn.Conv2d(1, 20, 5, 1)
self.conv2 = nn.Conv2d(20, 50, 5, 1)
self.fc1 = nn.Linear(4 * 4 * 50, 500)
self.fc2 = nn.Linear(500, 10)
def forward(self, x):
# Added the view for reshaping score requests
x = x.view(-1, 1, 28, 28)
x = F.relu(self.conv1(x))
x = F.max_pool2d(x, 2, 2)
x = F.relu(self.conv2(x))
x = F.max_pool2d(x, 2, 2)
x = x.view(-1, 4 * 4 * 50)
x = F.relu(self.fc1(x))
x = self.fc2(x)
return F.log_softmax(x, dim=1)
def train(args, model, device, train_loader, optimizer, epoch):
model.train()
for batch_idx, (data, target) in enumerate(train_loader):
data, target = data.to(device), target.to(device)
optimizer.zero_grad()
output = model(data)
loss = F.nll_loss(output, target)
loss.backward()
optimizer.step()
if batch_idx % args.log_interval == 0:
print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
epoch, batch_idx * len(data), len(train_loader.dataset),
100. * batch_idx / len(train_loader), loss.item()))
# Use MLflow logging
mlflow.log_metric("epoch_loss", loss.item())
def test(args, model, device, test_loader):
model.eval()
test_loss = 0
correct = 0
with torch.no_grad():
for data, target in test_loader:
data, target = data.to(device), target.to(device)
output = model(data)
# sum up batch loss
test_loss += F.nll_loss(output, target, reduction="sum").item()
# get the index of the max log-probability
pred = output.argmax(dim=1, keepdim=True)
correct += pred.eq(target.view_as(pred)).sum().item()
test_loss /= len(test_loader.dataset)
print("\n")
print("Test set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n".format(
test_loss, correct, len(test_loader.dataset),
100. * correct / len(test_loader.dataset)))
# Use MLflow logging
mlflow.log_metric("average_loss", test_loss)
class Args(object):
pass
# Training settings
args = Args()
setattr(args, 'batch_size', 64)
setattr(args, 'test_batch_size', 1000)
setattr(args, 'epochs', 3) # Higher number for better convergence
setattr(args, 'lr', 0.01)
setattr(args, 'momentum', 0.5)
setattr(args, 'no_cuda', True)
setattr(args, 'seed', 1)
setattr(args, 'log_interval', 10)
setattr(args, 'save_model', True)
use_cuda = not args.no_cuda and torch.cuda.is_available()
torch.manual_seed(args.seed)
device = torch.device("cuda" if use_cuda else "cpu")
kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
train_loader = torch.utils.data.DataLoader(
datasets.MNIST('../data', train=True, download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])),
batch_size=args.batch_size, shuffle=True, **kwargs)
test_loader = torch.utils.data.DataLoader(
datasets.MNIST(
'../data',
train=False,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))])),
batch_size=args.test_batch_size, shuffle=True, **kwargs)
def driver():
warnings.filterwarnings("ignore")
# Dependencies for deploying the model
pytorch_index = "https://download.pytorch.org/whl/"
pytorch_version = "cpu/torch-1.1.0-cp36-cp36m-linux_x86_64.whl"
deps = [
"cloudpickle=={}".format(cloudpickle.__version__),
pytorch_index + pytorch_version,
"torchvision=={}".format(torchvision.__version__),
"Pillow=={}".format("6.0.0")
]
with mlflow.start_run() as run:
model = Net().to(device)
optimizer = optim.SGD(
model.parameters(),
lr=args.lr,
momentum=args.momentum)
for epoch in range(1, args.epochs + 1):
train(args, model, device, train_loader, optimizer, epoch)
test(args, model, device, test_loader)
# Log model to run history using MLflow
if args.save_model:
model_env = _mlflow_conda_env(additional_pip_deps=deps)
mlflow.pytorch.log_model(model, "model", conda_env=model_env)
return run
if __name__ == "__main__":
driver()

464
how-to-use-azureml/track-and-monitor-experiments/using-mlflow/train-and-deploy-pytorch/train-and-deploy-pytorch.ipynb Normal file
View File

@@ -0,0 +1,464 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Copyright (c) Microsoft Corporation. All rights reserved.\n",
"\n",
"Licensed under the MIT License."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/using-mlflow/train-and-deploy-pytorch/train-and-deploy-pytorch.png)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Use MLflow with Azure Machine Learning to Train and Deploy PyTorch Image Classifier\n",
"\n",
"This example shows you how to use MLflow together with Azure Machine Learning services for tracking the metrics and artifacts while training a PyTorch model to classify MNIST digit images and deploy the model as a web service. You'll learn how to:\n",
"\n",
" 1. Set up MLflow tracking URI so as to use Azure ML\n",
" 2. Create experiment\n",
" 3. Instrument your model with MLflow tracking\n",
" 4. Train a PyTorch model locally\n",
" 5. Train a model on GPU compute on Azure\n",
" 6. View your experiment within your Azure ML Workspace in Azure Portal\n",
" 7. Deploy the model as a web service on Azure Container Instance\n",
" 8. Call the model to make predictions\n",
" \n",
"## Pre-requisites\n",
" \n",
"If you are using a Notebook VM, you are all set. Otherwise, go through the [Configuration](../../../../configuration.ipnyb) notebook to set up your Azure Machine Learning workspace and ensure other common prerequisites are met.\n",
"\n",
"Install PyTorch, this notebook has been tested with torch==1.4\n",
"\n",
"Also, install azureml-mlflow package using ```pip install azureml-mlflow```. Note that azureml-mlflow installs mlflow package itself as a dependency if you haven't done so previously.\n",
"\n",
"## Set-up\n",
"\n",
"Import packages and check versions of Azure ML SDK and MLflow installed on your computer. Then connect to your Workspace."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import sys, os\n",
"import mlflow\n",
"import mlflow.azureml\n",
"\n",
"import azureml.core\n",
"from azureml.core import Workspace\n",
"\n",
"\n",
"print(\"SDK version:\", azureml.core.VERSION)\n",
"print(\"MLflow version:\", mlflow.version.VERSION)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"ws = Workspace.from_config()\n",
"ws.get_details()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Set tracking URI\n",
"\n",
"Set the MLflow tracking URI to point to your Azure ML Workspace. The subsequent logging calls from MLflow APIs will go to Azure ML services and will be tracked under your Workspace."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"mlflow.set_tracking_uri(ws.get_mlflow_tracking_uri())"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Create Experiment\n",
"\n",
"In both MLflow and Azure ML, training runs are grouped into experiments. Let's create one for our experimentation."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"experiment_name = \"pytorch-with-mlflow\"\n",
"mlflow.set_experiment(experiment_name)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Train model locally while logging metrics and artifacts\n",
"\n",
"The ```scripts/train.py``` program contains the code to load the image dataset, train and test the model. Within this program, the train.driver function wraps the end-to-end workflow.\n",
"\n",
"Within the driver, the ```mlflow.start_run``` starts MLflow tracking. Then, ```mlflow.log_metric``` functions are used to track the convergence of the neural network training iterations. Finally ```mlflow.pytorch.save_model``` is used to save the trained model in framework-aware manner.\n",
"\n",
"Let's add the program to search path, import it as a module and invoke the driver function. Note that the training can take few minutes."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"lib_path = os.path.abspath(\"scripts\")\n",
"sys.path.append(lib_path)\n",
"\n",
"import train"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"run = train.driver()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Train model on GPU compute on Azure\n",
"\n",
"Next, let's run the same script on GPU-enabled compute for faster training. If you've completed the the [Configuration](../../../configuration.ipnyb) notebook, you should have a GPU cluster named \"gpu-cluster\" available in your workspace. Otherwise, follow the instructions in the notebook to create one. For simplicity, this example uses single process on single VM to train the model.\n",
"\n",
"Clone an environment object from the PyTorch 1.4 Azure ML curated environment. Azure ML curated environments are pre-configured environments to simplify ML setup, reference [this doc](https://docs.microsoft.com/en-us/azure/machine-learning/how-to-use-environments#use-a-curated-environment) for more information. To enable MLflow tracking, add ```azureml-mlflow``` as pip package."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from azureml.core import Environment\n",
"\n",
"env = Environment.get(workspace=ws, name=\"AzureML-PyTorch-1.4-GPU\").clone(\"mlflow-env\")\n",
"\n",
"env.python.conda_dependencies.add_pip_package(\"azureml-mlflow\")\n",
"env.python.conda_dependencies.add_pip_package(\"Pillow==6.0.0\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Create a ScriptRunConfig to specify the training configuration: script, compute as well as environment."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from azureml.core import ScriptRunConfig\n",
"\n",
"src = ScriptRunConfig(source_directory=\"./scripts\", script=\"train.py\")\n",
"src.run_config.environment = env\n",
"src.run_config.target = \"gpu-cluster\""
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Get a reference to the experiment you created previously, but this time, as an Azure Machine Learning experiment object.\n",
"\n",
"Then, use the ```Experiment.submit``` method to start the remote training run. Note that the first training run often takes longer as Azure Machine Learning service builds the Docker image for executing the script. Subsequent runs will be faster as the cached image is used."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from azureml.core import Experiment\n",
"\n",
"exp = Experiment(ws, experiment_name)\n",
"run = exp.submit(src)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"You can monitor the run and its metrics on Azure Portal."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"run"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Also, you can wait for run to complete."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"run.wait_for_completion(show_output=True)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Deploy model as web service\n",
"\n",
"The ```mlflow.azureml.deploy``` function registers the logged PyTorch model and deploys the model in a framework-aware manner. It automatically creates the PyTorch-specific inferencing wrapper code and specifies package dependencies for you. See [this doc](https://mlflow.org/docs/latest/models.html#id34) for more information on deploying models on Azure ML using MLflow.\n",
"\n",
"In this example, we deploy the Docker image to Azure Container Instance: a serverless compute capable of running a single container. You can tag and add descriptions to help keep track of your web service. \n",
"\n",
"[Other inferencing compute choices](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-deploy-and-where) include Azure Kubernetes Service which provides scalable endpoint suitable for production use.\n",
"\n",
"Note that the service deployment can take several minutes."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from azureml.core.webservice import AciWebservice, Webservice\n",
"\n",
"model_path = \"model\"\n",
"\n",
"aci_config = AciWebservice.deploy_configuration(cpu_cores=2, \n",
" memory_gb=5, \n",
" tags={\"data\": \"MNIST\", \"method\" : \"pytorch\"}, \n",
" description=\"Predict using webservice\")\n",
"\n",
"webservice, azure_model = mlflow.azureml.deploy(model_uri='runs:/{}/{}'.format(run.id, model_path),\n",
" workspace=ws,\n",
" deployment_config=aci_config,\n",
" service_name=\"pytorch-mnist-1\",\n",
" model_name=\"pytorch_mnist\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Once the deployment has completed you can check the scoring URI of the web service."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(\"Scoring URI is: {}\".format(webservice.scoring_uri))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"In case of a service creation issue, you can use ```webservice.get_logs()``` to get logs to debug."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Make predictions using a web service\n",
"\n",
"To make the web service, create a test data set as normalized PyTorch tensors. \n",
"\n",
"Then, let's define a utility function that takes a random image and converts it into a format and shape suitable for input to the PyTorch inferencing end-point. The conversion is done by: \n",
"\n",
" 1. Select a random (image, label) tuple\n",
" 2. Take the image and converting the tensor to NumPy array \n",
" 3. Reshape array into 1 x 1 x N array\n",
" * 1 image in batch, 1 color channel, N = 784 pixels for MNIST images\n",
" * Note also ```x = x.view(-1, 1, 28, 28)``` in net definition in ```train.py``` program to shape incoming scoring requests.\n",
" 4. Convert the NumPy array to list to make it into a built-in type.\n",
" 5. Create a dictionary {\"data\", &lt;list&gt;} that can be converted to JSON string for web service requests."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from torchvision import datasets, transforms\n",
"import random\n",
"import numpy as np\n",
"\n",
"test_data = datasets.MNIST('../data', train=False, transform=transforms.Compose([\n",
" transforms.ToTensor(),\n",
" transforms.Normalize((0.1307,), (0.3081,))]))\n",
"\n",
"\n",
"def get_random_image():\n",
" image_idx = random.randint(0,len(test_data))\n",
" image_as_tensor = test_data[image_idx][0]\n",
" return {\"data\": elem for elem in image_as_tensor.numpy().reshape(1,1,-1).tolist()}"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Then, invoke the web service using a random test image. Convert the dictionary containing the image to JSON string before passing it to web service.\n",
"\n",
"The response contains the raw scores for each label, with greater value indicating higher probability. Sort the labels and select the one with greatest score to get the prediction. Let's also plot the image sent to web service for comparison purposes."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%matplotlib inline\n",
"\n",
"import json\n",
"import matplotlib.pyplot as plt\n",
"\n",
"test_image = get_random_image()\n",
"\n",
"response = webservice.run(json.dumps(test_image))\n",
"\n",
"response = sorted(response[0].items(), key = lambda x: x[1], reverse = True)\n",
"\n",
"\n",
"print(\"Predicted label:\", response[0][0])\n",
"plt.imshow(np.array(test_image[\"data\"]).reshape(28,28), cmap = \"gray\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"You can also call the web service using a raw POST method against the web service"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import requests\n",
"\n",
"response = requests.post(url=webservice.scoring_uri, data=json.dumps(test_image),headers={\"Content-type\": \"application/json\"})\n",
"print(response.text)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Clean up\n",
"You can delete the ACI deployment with a delete API call."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"webservice.delete()"
]
}
],
"metadata": {
"authors": [
{
"name": "shipatel"
}
],
"category": "tutorial",
"celltoolbar": "Edit Metadata",
"compute": [
"Local",
"AML Compute"
],
"datasets": [
"MNIST"
],
"deployment": [
"Azure Container Instance"
],
"exclude_from_index": false,
"framework": [
"PyTorch"
],
"friendly_name": "Use MLflow with Azure Machine Learning to Train and Deploy PyTorch Image Classifier",
"kernelspec": {
"display_name": "Python 3.6",
"language": "python",
"name": "python36"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.7"
},
"name": "mlflow-sparksummit-pytorch",
"notebookId": 2495374963457641,
"tags": [
"mlflow",
"pytorch"
],
"task": "Use MLflow with Azure Machine Learning to train and deploy PyTorch image classifier model"
},
"nbformat": 4,
"nbformat_minor": 4
}

9
how-to-use-azureml/track-and-monitor-experiments/using-mlflow/train-and-deploy-pytorch/train-and-deploy-pytorch.yml Normal file
View File

@@ -0,0 +1,9 @@
name: train-and-deploy-pytorch
dependencies:
- pytorch==1.4.0 -c pytorch
- torchvision -c pytorch
- pip:
- azureml-sdk
- numpy
- azureml-mlflow
- matplotlib

62
how-to-use-azureml/training-with-deep-learning/train-hyperparameter-tune-deploy-with-keras/train-hyperparameter-tune-deploy-with-keras.ipynb
View File

@@ -604,21 +604,19 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"## Predict on the test set\n",
"## Predict on the test set (Optional)\n",
"Let's check the version of the local Keras. Make sure it matches with the version number printed out in the training script. Otherwise you might not be able to load the model properly."
]
},
{
"cell_type": "code",
"execution_count": null,
"cell_type": "markdown",
"metadata": {},
"outputs": [],
"source": [
"import keras\n",
"import tensorflow as tf\n",
" import keras\n",
" import tensorflow as tf\n",
"\n",
"print(\"Keras version:\", keras.__version__)\n",
"print(\"Tensorflow version:\", tf.__version__)"
" print(\"Keras version:\", keras.__version__)\n",
" print(\"Tensorflow version:\", tf.__version__)"
]
},
{
@@ -629,21 +627,19 @@
]
},
{
"cell_type": "code",
"execution_count": null,
"cell_type": "markdown",
"metadata": {},
"outputs": [],
"source": [
"from keras.models import model_from_json\n",
" from keras.models import model_from_json\n",
"\n",
"# load json and create model\n",
"json_file = open('model/model.json', 'r')\n",
"loaded_model_json = json_file.read()\n",
"json_file.close()\n",
"loaded_model = model_from_json(loaded_model_json)\n",
"# load weights into new model\n",
"loaded_model.load_weights(\"model/model.h5\")\n",
"print(\"Model loaded from disk.\")"
" # load json and create model\n",
" json_file = open('model/model.json', 'r')\n",
" loaded_model_json = json_file.read()\n",
" json_file.close()\n",
" loaded_model = model_from_json(loaded_model_json)\n",
" # load weights into new model\n",
" loaded_model.load_weights(\"model/model.h5\")\n",
" print(\"Model loaded from disk.\")"
]
},
{
@@ -654,19 +650,17 @@
]
},
{
"cell_type": "code",
"execution_count": null,
"cell_type": "markdown",
"metadata": {},
"outputs": [],
"source": [
"# evaluate loaded model on test data\n",
"loaded_model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy'])\n",
"y_test_ohe = one_hot_encode(y_test, 10)\n",
"y_hat = np.argmax(loaded_model.predict(X_test), axis=1)\n",
" # evaluate loaded model on test data\n",
" loaded_model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy'])\n",
" y_test_ohe = one_hot_encode(y_test, 10)\n",
" y_hat = np.argmax(loaded_model.predict(X_test), axis=1)\n",
"\n",
"# print the first 30 labels and predictions\n",
"print('labels: \\t', y_test[:30])\n",
"print('predictions:\\t', y_hat[:30])"
" # print the first 30 labels and predictions\n",
" print('labels: \\t', y_test[:30])\n",
" print('predictions:\\t', y_hat[:30])"
]
},
{
@@ -677,12 +671,10 @@
]
},
{
"cell_type": "code",
"execution_count": null,
"cell_type": "markdown",
"metadata": {},
"outputs": [],
"source": [
"print(\"Accuracy on the test set:\", np.average(y_hat == y_test))"
" print(\"Accuracy on the test set:\", np.average(y_hat == y_test))"
]
},
{
@@ -1056,7 +1048,7 @@
" font_color = 'red' if y_test[s] != result[i] else 'black'\n",
" clr_map = plt.cm.gray if y_test[s] != result[i] else plt.cm.Greys\n",
" \n",
" plt.text(x=10, y=-10, s=y_test[s], fontsize=18, color=font_color)\n",
" plt.text(x=10, y=-10, s=result[i], fontsize=18, color=font_color)\n",
" plt.imshow(X_test[s].reshape(28, 28), cmap=clr_map)\n",
" \n",
" i = i + 1\n",

33
how-to-use-azureml/training/train-within-notebook/score.py
View File

@@ -1,33 +0,0 @@
import pickle
import json
import numpy as np
from sklearn.linear_model import Ridge
from azureml.core.model import Model
# sklearn.externals.joblib is removed in 0.23
try:
from sklearn.externals import joblib
except ImportError:
import joblib
def init():
global model
# note here "best_model" is the name of the model registered under the workspace
# this call should return the path to the model.pkl file on the local disk.
model_path = Model.get_model_path(model_name='best_model')
# deserialize the model file back into a sklearn model
model = joblib.load(model_path)
# note you can pass in multiple rows for scoring
def run(raw_data):
try:
data = json.loads(raw_data)['data']
data = np.array(data)
result = model.predict(data)
# you can return any data type as long as it is JSON-serializable
return result.tolist()
except Exception as e:
result = str(e)
return result

721
how-to-use-azureml/training/train-within-notebook/train-within-notebook.ipynb
View File

@@ -1,721 +0,0 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Copyright (c) Microsoft Corporation. All rights reserved.\n",
"\n",
"Licensed under the MIT License."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/training/train-within-notebook/train-within-notebook.png)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Train and deploy a model\n",
"_**Create and deploy a model directly from a notebook**_\n",
"\n",
"---\n",
"---\n",
"\n",
"## Contents\n",
"1. [Introduction](#Introduction)\n",
"1. [Setup](#Setup)\n",
"1. [Data](#Data)\n",
"1. [Train](#Train)\n",
" 1. Viewing run results\n",
" 1. Simple parameter sweep\n",
" 1. Viewing experiment results\n",
" 1. Select the best model\n",
"1. [Deploy](#Deploy)\n",
" 1. Register the model\n",
" 1. Create a scoring file\n",
" 1. Describe your environment\n",
" 1. Descrice your target compute\n",
" 1. Deploy your webservice\n",
" 1. Test your webservice\n",
" 1. Clean up\n",
"1. [Next Steps](#nextsteps)\n",
"\n",
"---\n",
"\n",
"## Introduction\n",
"Azure Machine Learning provides capabilities to control all aspects of model training and deployment directly from a notebook using the AML Python SDK. In this notebook we will\n",
"* connect to our AML Workspace\n",
"* create an experiment that contains multiple runs with tracked metrics\n",
"* choose the best model created across all runs\n",
"* deploy that model as a service\n",
"\n",
"In the end we will have a model deployed as a web service which we can call from an HTTP endpoint"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"---\n",
"\n",
"## Setup\n",
"If you are using an Azure Machine Learning Notebook VM, you are all set. Otherwise, go through the [configuration](../../../configuration.ipynb) Notebook first if you haven't already to establish your connection to the AzureML Workspace. From the configuration, the important sections are the workspace configuration and ACI regristration.\n",
"\n",
"We will also need the following libraries install to our conda environment. If these are not installed, use the following command to do so and restart the notebook.\n",
"```shell\n",
"(myenv) $ conda install -y matplotlib tqdm scikit-learn\n",
"```\n",
"\n",
"For this notebook we need the Azure ML SDK and access to our workspace. The following cell imports the SDK, checks the version, and accesses our already configured AzureML workspace."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": [
"install"
],
"name": "load_ws",
"msdoc": "how-to-track-experiments.md"
},
"outputs": [],
"source": [
"import azureml.core\n",
"from azureml.core import Experiment, Workspace\n",
"\n",
"# Check core SDK version number\n",
"print(\"This notebook was created using version 1.0.2 of the Azure ML SDK\")\n",
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")\n",
"print(\"\")\n",
"\n",
"\n",
"ws = Workspace.from_config()\n",
"print('Workspace name: ' + ws.name, \n",
" 'Azure region: ' + ws.location, \n",
" 'Subscription id: ' + ws.subscription_id, \n",
" 'Resource group: ' + ws.resource_group, sep='\\n')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"---\n",
"\n",
"## Data\n",
"We will use the diabetes dataset for this experiement, a well-known small dataset that comes with scikit-learn. This cell loads the dataset and splits it into random training and testing sets.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"name": "load_data",
"msdoc": "how-to-track-experiments.md"
},
"outputs": [],
"source": [
"from sklearn.datasets import load_diabetes\n",
"from sklearn.linear_model import Ridge\n",
"from sklearn.metrics import mean_squared_error\n",
"from sklearn.model_selection import train_test_split\n",
"from sklearn.externals import joblib\n",
"\n",
"X, y = load_diabetes(return_X_y = True)\n",
"columns = ['age', 'gender', 'bmi', 'bp', 's1', 's2', 's3', 's4', 's5', 's6']\n",
"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)\n",
"data = {\n",
" \"train\":{\"X\": X_train, \"y\": y_train}, \n",
" \"test\":{\"X\": X_test, \"y\": y_test}\n",
"}\n",
"\n",
"print (\"Data contains\", len(data['train']['X']), \"training samples and\",len(data['test']['X']), \"test samples\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"---\n",
"## Train\n",
"\n",
"Let's use scikit-learn to train a simple Ridge regression model. We use AML to record interesting information about the model in an Experiment. An Experiment contains a series of trials called Runs. During this trial we use AML in the following way:\n",
"* We access an experiment from our AML workspace by name, which will be created if it doesn't exist\n",
"* We use `start_logging` to create a new run in this experiment\n",
"* We use `run.log()` to record a parameter, alpha, and an accuracy measure - the Mean Squared Error (MSE) to the run. We will be able to review and compare these measures in the Azure Portal at a later time.\n",
"* We store the resulting model in the **outputs** directory, which is automatically captured by AML when the run is complete.\n",
"* We use `run.complete()` to indicate that the run is over and results can be captured and finalized"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": [
"local run",
"outputs upload"
],
"name": "create_experiment",
"msdoc": "how-to-track-experiments.md"
},
"outputs": [],
"source": [
"# Get an experiment object from Azure Machine Learning\n",
"experiment = Experiment(workspace=ws, name=\"train-within-notebook\")\n",
"\n",
"# Create a run object in the experiment\n",
"run = experiment.start_logging()\n",
"# Log the algorithm parameter alpha to the run\n",
"run.log('alpha', 0.03)\n",
"\n",
"# Create, fit, and test the scikit-learn Ridge regression model\n",
"regression_model = Ridge(alpha=0.03)\n",
"regression_model.fit(data['train']['X'], data['train']['y'])\n",
"preds = regression_model.predict(data['test']['X'])\n",
"\n",
"# Output the Mean Squared Error to the notebook and to the run\n",
"print('Mean Squared Error is', mean_squared_error(data['test']['y'], preds))\n",
"run.log('mse', mean_squared_error(data['test']['y'], preds))\n",
"\n",
"# Save the model to the outputs directory for capture\n",
"model_file_name = 'outputs/model.pkl'\n",
"\n",
"joblib.dump(value = regression_model, filename = model_file_name)\n",
"\n",
"# upload the model file explicitly into artifacts \n",
"run.upload_file(name = model_file_name, path_or_stream = model_file_name)\n",
"\n",
"# Complete the run\n",
"run.complete()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Viewing run results\n",
"Azure Machine Learning stores all the details about the run in the Azure cloud. Let's access those details by retrieving a link to the run using the default run output. Clicking on the resulting link will take you to an interactive page presenting all run information."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"run"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Simple parameter sweep\n",
"Now let's take the same concept from above and modify the **alpha** parameter. For each value of alpha we will create a run that will store metrics and the resulting model. In the end we can use the captured run history to determine which model was the best for us to deploy. \n",
"\n",
"Note that by using `with experiment.start_logging() as run` AML will automatically call `run.complete()` at the end of each loop.\n",
"\n",
"This example also uses the **tqdm** library to provide a thermometer feedback"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"from tqdm import tqdm\n",
"\n",
"# list of numbers from 0 to 1.0 with a 0.05 interval\n",
"alphas = np.arange(0.0, 1.0, 0.05)\n",
"\n",
"# try a bunch of alpha values in a Linear Regression (Ridge) model\n",
"for alpha in tqdm(alphas):\n",
" # create a bunch of runs, each train a model with a different alpha value\n",
" with experiment.start_logging() as run:\n",
" # Use Ridge algorithm to build a regression model\n",
" regression_model = Ridge(alpha=alpha)\n",
" regression_model.fit(X=data[\"train\"][\"X\"], y=data[\"train\"][\"y\"])\n",
" preds = regression_model.predict(X=data[\"test\"][\"X\"])\n",
" mse = mean_squared_error(y_true=data[\"test\"][\"y\"], y_pred=preds)\n",
"\n",
" # log alpha, mean_squared_error and feature names in run history\n",
" run.log(name=\"alpha\", value=alpha)\n",
" run.log(name=\"mse\", value=mse)\n",
"\n",
" # Save the model to the outputs directory for capture\n",
" joblib.dump(value=regression_model, filename='outputs/model.pkl')\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Viewing experiment results\n",
"Similar to viewing the run, we can also view the entire experiment. The experiment report view in the Azure portal lets us view all the runs in a table, and also allows us to customize charts. This way, we can see how the alpha parameter impacts the quality of the model"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# now let's take a look at the experiment in Azure portal.\n",
"experiment"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Select the best model \n",
"Now that we've created many runs with different parameters, we need to determine which model is the best for deployment. For this, we will iterate over the set of runs. From each run we will take the *run id* using the `id` property, and examine the metrics by calling `run.get_metrics()`. \n",
"\n",
"Since each run may be different, we do need to check if the run has the metric that we are looking for, in this case, **mse**. To find the best run, we create a dictionary mapping the run id's to the metrics.\n",
"\n",
"Finally, we use the `tag` method to mark the best run to make it easier to find later. "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"runs = {}\n",
"run_metrics = {}\n",
"\n",
"# Create dictionaries containing the runs and the metrics for all runs containing the 'mse' metric\n",
"for r in tqdm(experiment.get_runs()):\n",
" metrics = r.get_metrics()\n",
" if 'mse' in metrics.keys():\n",
" runs[r.id] = r\n",
" run_metrics[r.id] = metrics\n",
"\n",
"# Find the run with the best (lowest) mean squared error and display the id and metrics\n",
"best_run_id = min(run_metrics, key = lambda k: run_metrics[k]['mse'])\n",
"best_run = runs[best_run_id]\n",
"print('Best run is:', best_run_id)\n",
"print('Metrics:', run_metrics[best_run_id])\n",
"\n",
"# Tag the best run for identification later\n",
"best_run.tag(\"Best Run\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"---\n",
"## Deploy\n",
"Now that we have trained a set of models and identified the run containing the best model, we want to deploy the model for real time inference. The process of deploying a model involves\n",
"* registering a model in your workspace\n",
"* creating a scoring file containing init and run methods\n",
"* creating an environment dependency file describing packages necessary for your scoring file\n",
"* deploying the model and packages as a web service"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Register a model\n",
"We have already identified which run contains the \"best model\" by our evaluation criteria. Each run has a file structure associated with it that contains various files collected during the run. Since a run can have many outputs we need to tell AML which file from those outputs represents the model that we want to use for our deployment. We can use the `run.get_file_names()` method to list the files associated with the run, and then use the `run.register_model()` method to place the model in the workspace's model registry.\n",
"\n",
"When using `run.register_model()` we supply a `model_name` that is meaningful for our scenario and the `model_path` of the model relative to the run. In this case, the model path is what is returned from `run.get_file_names()`"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": [
"query history"
]
},
"outputs": [],
"source": [
"# View the files in the run\n",
"for f in best_run.get_file_names():\n",
" print(f)\n",
" \n",
"# Register the model with the workspace\n",
"model = best_run.register_model(model_name='best_model', model_path='outputs/model.pkl')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Once a model is registered, it is accessible from the list of models on the AML workspace. If you register models with the same name multiple times, AML keeps a version history of those models for you. The `Model.list()` lists all models in a workspace, and can be filtered by name, tags, or model properties. "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": [
"register model from history"
]
},
"outputs": [],
"source": [
"# Find all models called \"best_model\" and display their version numbers\n",
"from azureml.core.model import Model\n",
"models = Model.list(ws, name='best_model')\n",
"for m in models:\n",
" print(m.name, m.version)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Create a scoring file\n",
"\n",
"Since your model file can essentially be anything you want it to be, you need to supply a scoring script that can load your model and then apply the model to new data. This script is your 'scoring file'. This scoring file is a python program containing, at a minimum, two methods `init()` and `run()`. The `init()` method is called once when your deployment is started so you can load your model and any other required objects. This method uses the `get_model_path` function to locate the registered model inside the docker container. The `run()` method is called interactively when the web service is called with one or more data samples to predict.\n",
"\n",
"The scoring file used for this exercise is [here](score.py). \n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Describe your environment\n",
"\n",
"Each modelling process may require a unique set of packages. Therefore we need to create an environment object describing the dependencies. \n",
"\n",
"Next we create an inference configuration using this environment object and the scoring script that we created previously."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from azureml.core.conda_dependencies import CondaDependencies\n",
"from azureml.core.environment import Environment\n",
"from azureml.core.model import InferenceConfig\n",
"\n",
"env = Environment('deploytocloudenv')\n",
"env.python.conda_dependencies = CondaDependencies.create(conda_packages=['scikit-learn'],pip_packages=['azureml-defaults'])\n",
"inference_config = InferenceConfig(entry_script=\"score.py\", environment=env)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Describe your target compute\n",
"In addition to the inference configuration, we also need to describe the type of compute we want to allocate for our webservice. In in this example we are using an [Azure Container Instance](https://azure.microsoft.com/en-us/services/container-instances/) which is a good choice for quick and cost-effective dev/test deployment scenarios. ACI instances require the number of cores you want to run and memory you need. Tags and descriptions are available for you to identify the instances in AML when viewing the Compute tab in the AML Portal.\n",
"\n",
"For production workloads, it is better to use [Azure Kubernentes Service (AKS)](https://azure.microsoft.com/en-us/services/kubernetes-service/) instead. Try [this notebook](https://github.com/Azure/MachineLearningNotebooks/blob/master/how-to-use-azureml/deployment/production-deploy-to-aks/production-deploy-to-aks.ipynb) to see how that can be done from Azure ML.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": [
"deploy service",
"aci"
]
},
"outputs": [],
"source": [
"from azureml.core.webservice import AciWebservice\n",
"\n",
"aciconfig = AciWebservice.deploy_configuration(cpu_cores=1, \n",
" memory_gb=1, \n",
" tags={'sample name': 'AML 101'}, \n",
" description='This is a great example.')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Deploy your webservice\n",
"The final step to deploying your webservice is to call `Model.deploy()`. This function uses the deployment and inference configurations created above to perform the following:\n",
"* Build a docker image\n",
"* Deploy to the docker image to an Azure Container Instance\n",
"* Copy your model files to the Azure Container Instance\n",
"* Call the `init()` function in your scoring file\n",
"* Provide an HTTP endpoint for scoring calls\n",
"\n",
"The `Model.deploy` method requires the following parameters\n",
"* `workspace` - the workspace containing the service\n",
"* `name` - a unique named used to identify the service in the workspace\n",
"* `models` - an array of models to be deployed into the container\n",
"* `inference_config` - a configuration object describing the image environment\n",
"* `deployment_config` - a configuration object describing the compute type\n",
" \n",
"**Note:** The web service creation can take several minutes. "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": [
"deploy service",
"aci"
]
},
"outputs": [],
"source": [
"%%time\n",
"from azureml.core.model import Model\n",
"from azureml.core.webservice import Webservice\n",
"\n",
"# Create the webservice using all of the precreated configurations and our best model\n",
"service = Model.deploy(workspace=ws,\n",
" name='my-aci-svc',\n",
" models=[model],\n",
" inference_config=inference_config,\n",
" deployment_config=aciconfig)\n",
"\n",
"# Wait for the service deployment to complete while displaying log output\n",
"service.wait_for_deployment(show_output=True)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"\n",
"### Test your webservice"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Now that your web service is runing you can send JSON data directly to the service using the `run` method. This cell pulls the first test sample from the original dataset into JSON and then sends it to the service."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": [
"deploy service",
"aci"
]
},
"outputs": [],
"source": [
"import json\n",
"\n",
"service = ws.webservices['my-aci-svc']\n",
"\n",
"# scrape the first row from the test set.\n",
"test_samples = json.dumps({\"data\": X_test[0:1, :].tolist()})\n",
"\n",
"#score on our service\n",
"service.run(input_data = test_samples)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"This cell shows how you can send multiple rows to the webservice at once. It then calculates the residuals - that is, the errors - by subtracting out the actual values from the results. These residuals are used later to show a plotted result."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": [
"deploy service",
"aci"
]
},
"outputs": [],
"source": [
"# score the entire test set.\n",
"test_samples = json.dumps({'data': X_test.tolist()})\n",
"\n",
"result = service.run(input_data = test_samples)\n",
"residual = result - y_test"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"This cell shows how you can use the `service.scoring_uri` property to access the HTTP endpoint of the service and call it using standard POST operations."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": [
"deploy service",
"aci"
]
},
"outputs": [],
"source": [
"import requests\n",
"\n",
"# use the first row from the test set again\n",
"test_samples = json.dumps({\"data\": X_test[0:1, :].tolist()})\n",
"\n",
"# create the required header\n",
"headers = {'Content-Type':'application/json'}\n",
"\n",
"# post the request to the service and display the result\n",
"resp = requests.post(service.scoring_uri, test_samples, headers = headers)\n",
"print(resp.text)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Residual graph\n",
"One way to understand the behavior of your model is to see how the data performs against data with known results. This cell uses matplotlib to create a histogram of the residual values, or errors, created from scoring the test samples.\n",
"\n",
"A good model should have residual values that cluster around 0 - that is, no error. Observing the resulting histogram can also show you if the model is skewed in any particular direction."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%matplotlib inline\n",
"import matplotlib.pyplot as plt\n",
"\n",
"f, (a0, a1) = plt.subplots(1, 2, gridspec_kw={'width_ratios':[3, 1], 'wspace':0, 'hspace': 0})\n",
"f.suptitle('Residual Values', fontsize = 18)\n",
"\n",
"f.set_figheight(6)\n",
"f.set_figwidth(14)\n",
"\n",
"a0.plot(residual, 'bo', alpha=0.4)\n",
"a0.plot([0,90], [0,0], 'r', lw=2)\n",
"a0.set_ylabel('residue values', fontsize=14)\n",
"a0.set_xlabel('test data set', fontsize=14)\n",
"\n",
"a1.hist(residual, orientation='horizontal', color='blue', bins=10, histtype='step')\n",
"a1.hist(residual, orientation='horizontal', color='blue', alpha=0.2, bins=10)\n",
"a1.set_yticklabels([])\n",
"\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Clean up"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Delete the ACI instance to stop the compute and any associated billing."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": [
"deploy service",
"aci"
]
},
"outputs": [],
"source": [
"%%time\n",
"service.delete()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<a id='nextsteps'></a>\n",
"## Next Steps"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"In this example, you created a series of models inside the notebook using local data, stored them inside an AML experiment, found the best one and deployed it as a live service! From here you can continue to use Azure Machine Learning in this regard to run your own experiments and deploy your own models, or you can expand into further capabilities of AML!\n",
"\n",
"If you have a model that is difficult to process locally, either because the data is remote or the model is large, try the [train-on-remote-vm](../train-on-remote-vm) notebook to learn about submitting remote jobs.\n",
"\n",
"If you want to take advantage of multiple cloud machines to perform large parameter sweeps try the [train-hyperparameter-tune-deploy-with-pytorch](../../training-with-deep-learning/train-hyperparameter-tune-deploy-with-pytorch\n",
") sample.\n",
"\n",
"If you want to deploy models to a production cluster try the [production-deploy-to-aks](../../deployment/production-deploy-to-aks\n",
") notebook."
]
}
],
"metadata": {
"authors": [
{
"name": "roastala"
}
],
"category": "tutorial",
"compute": [
"Local"
],
"datasets": [
"Diabetes"
],
"deployment": [
"Azure Container Instance"
],
"exclude_from_index": false,
"framework": [
"None"
],
"friendly_name": "Train and deploy a model using Python SDK",
"index_order": 1,
"kernelspec": {
"display_name": "Python 3.6",
"language": "python",
"name": "python36"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.5"
},
"tags": [
"None"
],
"task": "Training and deploying a model from a notebook"
},
"nbformat": 4,
"nbformat_minor": 2
}

8
how-to-use-azureml/training/train-within-notebook/train-within-notebook.yml
View File

@@ -1,8 +0,0 @@
name: train-within-notebook
dependencies:
- tqdm
- scikit-learn
- matplotlib
- pip:
- azureml-sdk
- azureml-widgets

56
how-to-use-azureml/work-with-data/datasets-tutorial/timeseries-datasets/tabular-timeseries-dataset-filtering.ipynb
View File

@@ -84,14 +84,9 @@
"outputs": [],
"source": [
"# import packages\n",
"import os\n",
"\n",
"import pandas as pd\n",
"\n",
"from calendar import monthrange\n",
"from datetime import datetime, timedelta\n",
"\n",
"from azureml.core import Dataset, Datastore, Workspace, Run"
"from azureml.core import Dataset, Workspace"
]
},
{
@@ -174,7 +169,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"Assign timestamp column for Tabular Dataset to activate Time Series related APIs. The column to be assigned should be a Date type, otherwise the assigning will fail."
"Assign \"datetime\" column as timestamp and \"partition_time\" from folder path as partition_timestamp for Tabular Dataset to activate Time Series related APIs. The column to be assigned should be a Date type, otherwise the assigning will fail."
]
},
{
@@ -183,8 +178,7 @@
"metadata": {},
"outputs": [],
"source": [
"# for this demo, leave out partition_time so timestamp is used\n",
"tsd = dataset.with_timestamp_columns(timestamp='datetime') # partition_timestamp='partition_time')"
"tsd = dataset.with_timestamp_columns(timestamp='datetime', partition_timestamp='partition_time')"
]
},
{
@@ -228,7 +222,9 @@
"source": [
"## Filter Data by Time Windows\n",
"\n",
"Once your data has been loaded into the notebook, you can query by time using the time_before(), time_after(), time_between(), and time_recent() functions. You can also choose to drop or keep certain columns. "
"Once your data has been loaded into the notebook, you can query by time using the time_before(), time_after(), time_between(), and time_recent() functions.The filter is optimized to only load those data files within the partition_timestamp range when partition_timestamp is specified.\n",
"\n",
"include_boundary is default to be true for all the time series related filters, please pass include_boundary=False to exclude boundary."
]
},
{
@@ -276,13 +272,6 @@
"You can chain time functions together."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**NOTE:** You must set the partition_timestamp to None to filter on the timestamp. The below cell will fail unless the second line is uncommented "
]
},
{
"cell_type": "code",
"execution_count": null,
@@ -290,8 +279,7 @@
"outputs": [],
"source": [
"# select data that occurs within a given time range\n",
"#tsd = tsd.with_timestamp_columns(timestamp='datetime', partition_timestamp=None)\n",
"tsd2 = tsd.time_after(datetime(2019, 1, 2)).time_before(datetime(2019, 1, 10))\n",
"tsd2 = tsd.time_after(datetime(2019, 1, 1)).time_before(datetime(2019, 1, 10))\n",
"tsd2.to_pandas_dataframe().head(5)"
]
},
@@ -327,16 +315,6 @@
"This function takes in a datetime.timedelta and returns a dataset containing the data from datetime.now()-timedelta() to datetime.now()."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"tsd2 = tsd.time_recent(timedelta(weeks=5, days=0))\n",
"tsd2.to_pandas_dataframe().head(5)"
]
},
{
"cell_type": "markdown",
"metadata": {},
@@ -354,6 +332,15 @@
"tsd2.to_pandas_dataframe().tail(5)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Drop and keep columns\n",
"\n",
"You can also choose to drop or keep certain columns."
]
},
{
"cell_type": "markdown",
"metadata": {},
@@ -391,9 +378,11 @@
"metadata": {},
"outputs": [],
"source": [
"from azureml.exceptions import DatasetTimestampMissingError\n",
"\n",
"try:\n",
" tsd2.time_before(datetime(2019, 6, 12)).to_pandas_dataframe().tail(5)\n",
"except Exception as e:\n",
"except DatasetTimestampMissingError as e:\n",
" print('Expected exception : {}'.format(str(e)))"
]
},
@@ -462,7 +451,7 @@
"source": [
"try:\n",
" tsd2.time_before(datetime(2019, 6, 12)).to_pandas_dataframe().tail(5)\n",
"except Exception as e:\n",
"except DatasetTimestampMissingError as e:\n",
" print('Expected exception : {}'.format(str(e)))"
]
},
@@ -515,10 +504,11 @@
"metadata": {},
"outputs": [],
"source": [
"from azureml.exceptions import UserErrorException\n",
"# Illegal clearing, exception is expected.\n",
"try:\n",
" tsd2 = tsd.with_timestamp_columns(timestamp=None, partition_timestamp='partition_time')\n",
"except Exception as e:\n",
"except UserErrorException as e:\n",
" print('Cleaning not allowed because {}'.format(str(e)))\n",
"\n",
"# clear both\n",
@@ -575,7 +565,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.8"
"version": "3.6.10"
},
"notice": "Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License.",
"star_tag": [

3
index.md
View File

@@ -20,7 +20,6 @@ Machine Learning notebook samples and encourage efficient retrieval of topics an
| [Forecasting orange juice sales with deployment](https://github.com/Azure/MachineLearningNotebooks/blob/master//how-to-use-azureml/automated-machine-learning/forecasting-orange-juice-sales/auto-ml-forecasting-orange-juice-sales.ipynb) | Forecasting | Orange Juice Sales | Remote | Azure Container Instance | Azure ML AutoML | None |
| [Register a model and deploy locally](https://github.com/Azure/MachineLearningNotebooks/blob/master//how-to-use-azureml/deployment/deploy-to-local/register-model-deploy-local.ipynb) | Deployment | None | Local | Local | None | None |
| :star:[Data drift on aks](https://github.com/Azure/MachineLearningNotebooks/blob/master//how-to-use-azureml/monitor-models/data-drift/drift-on-aks.ipynb) | Filtering | NOAA | Remote | AKS | Azure ML | Dataset, Timeseries, Drift |
| [Train and deploy a model using Python SDK](https://github.com/Azure/MachineLearningNotebooks/blob/master//how-to-use-azureml/training/train-within-notebook/train-within-notebook.ipynb) | Training and deploying a model from a notebook | Diabetes | Local | Azure Container Instance | None | None |
| :star:[Data drift quickdemo](https://github.com/Azure/MachineLearningNotebooks/blob/master//how-to-use-azureml/work-with-data/datadrift-tutorial/datadrift-tutorial.ipynb) | Filtering | NOAA | Remote | None | Azure ML | Dataset, Timeseries, Drift |
| :star:[Introduction to labeled datasets](https://github.com/Azure/MachineLearningNotebooks/blob/master//how-to-use-azureml/work-with-data/datasets-tutorial/labeled-datasets/labeled-datasets.ipynb) | Train | | Remote | None | Azure ML | Dataset, label, Estimator |
| :star:[Datasets with ML Pipeline](https://github.com/Azure/MachineLearningNotebooks/blob/master//how-to-use-azureml/work-with-data/datasets-tutorial/pipeline-with-datasets/pipeline-for-image-classification.ipynb) | Train | Fashion MNIST | Remote | None | Azure ML | Dataset, Pipeline, Estimator, ScriptRun |
@@ -47,6 +46,8 @@ Machine Learning notebook samples and encourage efficient retrieval of topics an
| :star:[How to use AutoMLStep with AML Pipelines](https://github.com/Azure/MachineLearningNotebooks/blob/master//how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-with-automated-machine-learning-step.ipynb) | Demonstrates the use of AutoMLStep | Custom | AML Compute | None | Automated Machine Learning | None |
| :star:[Azure Machine Learning Pipelines with Data Dependency](https://github.com/Azure/MachineLearningNotebooks/blob/master//how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-with-data-dependency-steps.ipynb) | Demonstrates how to construct a Pipeline with data dependency between steps | Custom | AML Compute | None | Azure ML | None |
| [How to use run a notebook as a step in AML Pipelines](https://github.com/Azure/MachineLearningNotebooks/blob/master//how-to-use-azureml/machine-learning-pipelines/intro-to-pipelines/aml-pipelines-with-notebook-runner-step.ipynb) | Demonstrates the use of NotebookRunnerStep | Custom | AML Compute | None | Azure ML | None |
| [Use MLflow with Azure Machine Learning to Train and Deploy Keras Image Classifier](https://github.com/Azure/MachineLearningNotebooks/blob/master//how-to-use-azureml/track-and-monitor-experiments/using-mlflow/train-and-deploy-keras-auto-logging/train-and-deploy-keras-auto-logging.ipynb) | Use MLflow with Azure Machine Learning to Train and Deploy Keras Image Classifier, leveraging MLflow auto logging | MNIST | Local, AML Compute | Azure Container Instance | Keras | mlflow, keras |
| [Use MLflow with Azure Machine Learning to Train and Deploy PyTorch Image Classifier](https://github.com/Azure/MachineLearningNotebooks/blob/master//how-to-use-azureml/track-and-monitor-experiments/using-mlflow/train-and-deploy-pytorch/train-and-deploy-pytorch.ipynb) | Use MLflow with Azure Machine Learning to train and deploy PyTorch image classifier model | MNIST | Local, AML Compute | Azure Container Instance | PyTorch | mlflow, pytorch |
## Training

2
setup-environment/configuration.ipynb
View File

@@ -102,7 +102,7 @@
"source": [
"import azureml.core\n",
"\n",
"print(\"This notebook was created using version 1.11.0 of the Azure ML SDK\")\n",
"print(\"This notebook was created using version 1.12.0 of the Azure ML SDK\")\n",
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
]
},

54
tutorials/machine-learning-pipelines-advanced/tutorial-pipeline-batch-scoring-classification.ipynb
View File

@@ -64,10 +64,27 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Check core SDK version number\n",
"import azureml.core\n",
"\n",
"print(\"SDK version:\", azureml.core.VERSION)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"from azureml.core import Workspace\n",
"\n",
"ws = Workspace.from_config()"
]
},
@@ -397,7 +414,26 @@
"from azureml.pipeline.core import Pipeline\n",
"\n",
"pipeline = Pipeline(workspace=ws, steps=[batch_score_step])\n",
"pipeline_run = Experiment(ws, \"batch_scoring\").submit(pipeline)\n",
"pipeline_run = Experiment(ws, \"batch_scoring\").submit(pipeline)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# This will output information of the pipeline run, including the link to the details page of portal.\n",
"pipeline_run"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Wait the run for completion and show output log to console\n",
"pipeline_run.wait_for_completion(show_output=True)"
]
},
@@ -544,10 +580,18 @@
"outputs": [],
"source": [
"from azureml.pipeline.core.run import PipelineRun\n",
"from azureml.widgets import RunDetails\n",
"\n",
"published_pipeline_run = PipelineRun(ws.experiments[\"batch_scoring\"], run_id)\n",
"RunDetails(published_pipeline_run).show()"
"published_pipeline_run = PipelineRun(ws.experiments[\"batch_scoring\"], run_id)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Show detail information of the run\n",
"published_pipeline_run"
]
},
{

2
tutorials/regression-automl-nyc-taxi-data/regression-automated-ml.ipynb
View File

@@ -98,7 +98,7 @@
"\n",
"for sample_month in range(12):\n",
" temp_df_green = NycTlcGreen(start + relativedelta(months=sample_month), end + relativedelta(months=sample_month)) \\\n",
" .to_pandas_dataframe()\n",
" .get_tabular_dataset().to_pandas_dataframe()\n",
" green_taxi_df = green_taxi_df.append(temp_df_green.sample(2000))\n",
" \n",
"green_taxi_df.head(10)"