Update notebooks
Update notebooks
This commit is contained in:
rastala
@@ -447,8 +447,9 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"for m in ws.models(name='best_model'):\n",
|
||||
" print(m.name, m.version)"
|
||||
"models = ws.models(name='best_model')\n",
|
||||
"for name, m in models.items():\n",
|
||||
" print(name, m.version)"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,52 +0,0 @@
|
||||
from sklearn.datasets import load_diabetes
|
||||
from sklearn.linear_model import Ridge
|
||||
from sklearn.metrics import mean_squared_error
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.externals import joblib
|
||||
|
||||
import os
|
||||
import argparse
|
||||
|
||||
# Import Run from azureml.core,
|
||||
from azureml.core.run import Run
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--alpha', type=float, dest='alpha',
|
||||
default=0.5, help='regularization strength')
|
||||
args = parser.parse_args()
|
||||
|
||||
# Get handle of current run for logging and history purposes
|
||||
run = Run.get_submitted_run()
|
||||
|
||||
X, y = load_diabetes(return_X_y=True)
|
||||
|
||||
columns = ['age', 'gender', 'bmi', 'bp', 's1', 's2', 's3', 's4', 's5', 's6']
|
||||
|
||||
x_train, x_test, y_train, y_test = train_test_split(
|
||||
X, y, test_size=0.2, random_state=0)
|
||||
data = {"train": {"x": x_train, "y": y_train},
|
||||
"test": {"x": x_test, "y": y_test}}
|
||||
|
||||
alpha = args.alpha
|
||||
print('alpha value is:', alpha)
|
||||
|
||||
reg = Ridge(alpha=alpha)
|
||||
reg.fit(data["train"]["x"], data["train"]["y"])
|
||||
|
||||
print('Ridget model fitted.')
|
||||
|
||||
preds = reg.predict(data["test"]["x"])
|
||||
mse = mean_squared_error(preds, data["test"]["y"])
|
||||
|
||||
# Log metrics
|
||||
run.log("alpha", alpha)
|
||||
run.log("mse", mse)
|
||||
|
||||
os.makedirs('./outputs', exist_ok=True)
|
||||
model_file_name = "model.pkl"
|
||||
|
||||
# Save model as part of the run history
|
||||
with open(model_file_name, "wb") as file:
|
||||
joblib.dump(reg, 'outputs/' + model_file_name)
|
||||
|
||||
print('Mean Squared Error is:', mse)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Binary file not shown.
|
Before Width: | Height: | Size: 572 B |
Binary file not shown.
|
Before Width: | Height: | Size: 555 B |
Binary file not shown.
|
Before Width: | Height: | Size: 615 B |
Binary file not shown.
|
Before Width: | Height: | Size: 561 B |
Binary file not shown.
|
Before Width: | Height: | Size: 504 B |
Binary file not shown.
|
Before Width: | Height: | Size: 546 B |
Binary file not shown.
|
Before Width: | Height: | Size: 578 B |
Binary file not shown.
|
Before Width: | Height: | Size: 586 B |
Binary file not shown.
|
Before Width: | Height: | Size: 562 B |
Binary file not shown.
|
Before Width: | Height: | Size: 592 B |
@@ -1,103 +0,0 @@
|
||||
from __future__ import print_function
|
||||
import tensorflow as tf
|
||||
import numpy as np
|
||||
import os
|
||||
import json
|
||||
import base64
|
||||
from io import BytesIO
|
||||
from PIL import Image
|
||||
|
||||
##############################################
|
||||
# helper functions
|
||||
##############################################
|
||||
|
||||
|
||||
def build_model(x, y_, keep_prob):
|
||||
def weight_variable(shape):
|
||||
initial = tf.truncated_normal(shape, stddev=0.1)
|
||||
return tf.Variable(initial)
|
||||
|
||||
def bias_variable(shape):
|
||||
initial = tf.constant(0.1, shape=shape)
|
||||
return tf.Variable(initial)
|
||||
|
||||
def conv2d(x, W):
|
||||
return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')
|
||||
|
||||
def max_pool_2x2(x):
|
||||
return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
|
||||
|
||||
W_conv1 = weight_variable([5, 5, 1, 32])
|
||||
b_conv1 = bias_variable([32])
|
||||
|
||||
x_image = tf.reshape(x, [-1, 28, 28, 1])
|
||||
|
||||
h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)
|
||||
h_pool1 = max_pool_2x2(h_conv1)
|
||||
|
||||
W_conv2 = weight_variable([5, 5, 32, 64])
|
||||
b_conv2 = bias_variable([64])
|
||||
|
||||
h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
|
||||
h_pool2 = max_pool_2x2(h_conv2)
|
||||
|
||||
W_fc1 = weight_variable([7 * 7 * 64, 1024])
|
||||
b_fc1 = bias_variable([1024])
|
||||
|
||||
h_pool2_flat = tf.reshape(h_pool2, [-1, 7 * 7 * 64])
|
||||
h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)
|
||||
|
||||
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
|
||||
|
||||
W_fc2 = weight_variable([1024, 10])
|
||||
b_fc2 = bias_variable([10])
|
||||
|
||||
y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2
|
||||
|
||||
return y_conv
|
||||
|
||||
|
||||
def base64ToImg(base64ImgString):
|
||||
if base64ImgString.startswith('b\''):
|
||||
base64ImgString = base64ImgString[2:-1]
|
||||
base64Img = base64ImgString.encode('utf-8')
|
||||
decoded_img = base64.b64decode(base64Img)
|
||||
img_buffer = BytesIO(decoded_img)
|
||||
img = Image.open(img_buffer)
|
||||
return img
|
||||
|
||||
##############################################
|
||||
# API init() and run() methods
|
||||
##############################################
|
||||
|
||||
|
||||
def init():
|
||||
global x, keep_prob, y_conv, sess
|
||||
g = tf.Graph()
|
||||
with g.as_default():
|
||||
x = tf.placeholder(tf.float32, shape=[None, 784])
|
||||
y_ = tf.placeholder(tf.float32, shape=[None, 10])
|
||||
keep_prob = tf.placeholder(tf.float32)
|
||||
y_conv = build_model(x, y_, keep_prob)
|
||||
|
||||
saver = tf.train.Saver()
|
||||
init_op = tf.global_variables_initializer()
|
||||
|
||||
model_dir = os.path.join('sample_projects', 'outputs')
|
||||
saved_model_path = os.path.join(model_dir, 'model.ckpt')
|
||||
|
||||
sess = tf.Session(graph=g)
|
||||
sess.run(init_op)
|
||||
saver.restore(sess, saved_model_path)
|
||||
|
||||
|
||||
def run(input_data):
|
||||
img = base64ToImg(json.loads(input_data)['data'])
|
||||
img_data = np.array(img, dtype=np.float32).flatten()
|
||||
img_data.resize((1, 784))
|
||||
|
||||
y_pred = sess.run(y_conv, feed_dict={x: img_data, keep_prob: 1.0})
|
||||
predicted_label = np.argmax(y_pred[0])
|
||||
|
||||
outJsonString = json.dumps({"label": str(predicted_label)})
|
||||
return str(outJsonString)
|
||||
@@ -1,151 +0,0 @@
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import tensorflow as tf
|
||||
# Load MNIST Data
|
||||
from tensorflow.examples.tutorials.mnist import input_data
|
||||
import os
|
||||
import argparse
|
||||
|
||||
from azureml.core.run import Run
|
||||
|
||||
# the following 10 lines can be removed once BUG# 241943 is fixed
|
||||
|
||||
|
||||
def get_logger():
|
||||
try:
|
||||
return Run.get_submitted_run()
|
||||
except Exception:
|
||||
return LocalLogger()
|
||||
|
||||
|
||||
class LocalLogger:
|
||||
def log(self, key, value):
|
||||
print("AML-Log:", key, value)
|
||||
|
||||
|
||||
def build_model(x, y_, keep_prob):
|
||||
def weight_variable(shape):
|
||||
initial = tf.truncated_normal(shape, stddev=0.1)
|
||||
return tf.Variable(initial)
|
||||
|
||||
def bias_variable(shape):
|
||||
initial = tf.constant(0.1, shape=shape)
|
||||
return tf.Variable(initial)
|
||||
|
||||
def conv2d(x, W):
|
||||
return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')
|
||||
|
||||
def max_pool_2x2(x):
|
||||
return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
|
||||
|
||||
W_conv1 = weight_variable([5, 5, 1, 32])
|
||||
b_conv1 = bias_variable([32])
|
||||
|
||||
x_image = tf.reshape(x, [-1, 28, 28, 1])
|
||||
|
||||
h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)
|
||||
h_pool1 = max_pool_2x2(h_conv1)
|
||||
|
||||
W_conv2 = weight_variable([5, 5, 32, 64])
|
||||
b_conv2 = bias_variable([64])
|
||||
|
||||
h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
|
||||
h_pool2 = max_pool_2x2(h_conv2)
|
||||
|
||||
W_fc1 = weight_variable([7 * 7 * 64, 1024])
|
||||
b_fc1 = bias_variable([1024])
|
||||
|
||||
h_pool2_flat = tf.reshape(h_pool2, [-1, 7 * 7 * 64])
|
||||
h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)
|
||||
|
||||
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
|
||||
|
||||
W_fc2 = weight_variable([1024, 10])
|
||||
b_fc2 = bias_variable([10])
|
||||
|
||||
y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2
|
||||
|
||||
return y_conv
|
||||
|
||||
|
||||
def main():
|
||||
# Get command-line arguments
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--learning_rate', type=float,
|
||||
default=0.0001, help='learning rate')
|
||||
parser.add_argument('--minibatch_size', type=int,
|
||||
default=50, help='minibatch size')
|
||||
parser.add_argument('--keep_probability', type=float,
|
||||
default=0.5, help='keep probability for dropout layer')
|
||||
parser.add_argument('--num_iterations', type=int,
|
||||
default=1000, help='number of iterations')
|
||||
parser.add_argument('--output_dir', type=str, default='./outputs',
|
||||
help='output directory to write checkpoints to')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# log parameters
|
||||
run_logger = get_logger()
|
||||
run_logger.log("learning_rate", args.learning_rate)
|
||||
run_logger.log("minibatch_size", args.minibatch_size)
|
||||
run_logger.log("keep_probability", args.keep_probability)
|
||||
run_logger.log("num_iterations", args.num_iterations)
|
||||
|
||||
# Load MNIST data
|
||||
mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
|
||||
|
||||
sess = tf.InteractiveSession()
|
||||
|
||||
x = tf.placeholder(tf.float32, shape=[None, 784])
|
||||
y_ = tf.placeholder(tf.float32, shape=[None, 10])
|
||||
keep_prob = tf.placeholder(tf.float32)
|
||||
|
||||
y_conv = build_model(x, y_, keep_prob)
|
||||
|
||||
cross_entropy = tf.reduce_mean(
|
||||
tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))
|
||||
|
||||
train_step = tf.train.AdamOptimizer(
|
||||
args.learning_rate).minimize(cross_entropy)
|
||||
correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))
|
||||
|
||||
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
|
||||
sess.run(tf.global_variables_initializer())
|
||||
|
||||
for i in range(args.num_iterations):
|
||||
batch = mnist.train.next_batch(args.minibatch_size)
|
||||
if i % 100 == 0:
|
||||
test_acc = accuracy.eval(
|
||||
feed_dict={x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0})
|
||||
train_accuracy = accuracy.eval(
|
||||
feed_dict={x: batch[0], y_: batch[1], keep_prob: 1.0})
|
||||
print("step %d, training accuracy %g, test accuracy, %g" %
|
||||
(i, train_accuracy, test_acc))
|
||||
|
||||
# log test accuracy to AML
|
||||
run_logger.log("Accuracy", float(test_acc))
|
||||
run_logger.log("Iterations", i)
|
||||
|
||||
sess.run(train_step, feed_dict={
|
||||
x: batch[0], y_: batch[1], keep_prob: args.keep_probability})
|
||||
|
||||
# Save the trained model
|
||||
model_dir = args.output_dir
|
||||
model_file = 'model.ckpt'
|
||||
|
||||
os.makedirs(model_dir, exist_ok=True)
|
||||
|
||||
saver = tf.train.Saver()
|
||||
saver.save(sess, os.path.join(model_dir, model_file))
|
||||
|
||||
final_test_acc = sess.run(accuracy, feed_dict={
|
||||
x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0})
|
||||
run_logger.log("Accuracy", float(final_test_acc))
|
||||
run_logger.log("Iterations", args.num_iterations)
|
||||
print("test accuracy %g" % final_test_acc)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -130,8 +130,8 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"regression_models = ws.models(tags=['area'])\n",
|
||||
"for m in regression_models:\n",
|
||||
" print(\"Name:\", m.name,\"\\tVersion:\", m.version, \"\\tDescription:\", m.description, m.tags)"
|
||||
"for name, m in regression_models.items():\n",
|
||||
" print(\"Name:\", name,\"\\tVersion:\", m.version, \"\\tDescription:\", m.description, m.tags)"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -1,14 +1,5 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Copyright (c) Microsoft Corporation. All rights reserved.\n",
|
||||
"\n",
|
||||
"Licensed under the MIT License."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
@@ -90,8 +81,8 @@
|
||||
"source": [
|
||||
"#Register the model\n",
|
||||
"from azureml.core.model import Model\n",
|
||||
"model = Model.register(model_path = 'sklearn_regression_model.pkl', # this points to a local file\n",
|
||||
" model_name = \"best_model\", # this is the name the model is registered as\n",
|
||||
"model = Model.register(model_path = \"sklearn_regression_model.pkl\", # this points to a local file\n",
|
||||
" model_name = \"sklearn_regression_model.pkl\", # this is the name the model is registered as\n",
|
||||
" tags = {'area': \"diabetes\", 'type': \"regression\"},\n",
|
||||
" description = \"Ridge regression model to predict diabetes\",\n",
|
||||
" workspace = ws)\n",
|
||||
@@ -103,22 +94,25 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 4. Update your scoring file with Data Collection\n",
|
||||
"## 4. *Update your scoring file with Data Collection*\n",
|
||||
"The file below, compared to the file used in notebook 11, has the following changes:\n",
|
||||
"### a. Import the module\n",
|
||||
"<code> from azureml.monitoring import ModelDataCollector </code>\n",
|
||||
"```python \n",
|
||||
"from azureml.monitoring import ModelDataCollector```\n",
|
||||
"### b. In your init function add:\n",
|
||||
"<code> global inputs_dc, prediction_d\n",
|
||||
" inputs_dc = ModelDataCollector(\"best_model\", identifier=\"inputs\", feature_names=[\"feat1\", \"feat2\", \"feat3\". \"feat4\", \"feat5\", \"Feat6\"])\n",
|
||||
" prediction_dc = ModelDataCollector(\"best_model\", identifier=\"predictions\", feature_names=[\"prediction1\", \"prediction2\"])</code>\n",
|
||||
"```python \n",
|
||||
"global inputs_dc, prediction_d\n",
|
||||
"inputs_dc = ModelDataCollector(\"best_model\", identifier=\"inputs\", feature_names=[\"feat1\", \"feat2\", \"feat3\". \"feat4\", \"feat5\", \"Feat6\"])\n",
|
||||
"prediction_dc = ModelDataCollector(\"best_model\", identifier=\"predictions\", feature_names=[\"prediction1\", \"prediction2\"])```\n",
|
||||
" \n",
|
||||
"* Identifier: Identifier is later used for building the folder structure in your Blob, it can be used to divide “raw” data versus “processed”.\n",
|
||||
"* CorrelationId: is an optional parameter, you do not need to set it up if your model doesn’t require it. Having a correlationId in place does help you for easier mapping with other data. (Examples include: LoanNumber, CustomerId, etc.)\n",
|
||||
"* Feature Names: These need to be set up in the order of your features in order for them to have column names when the .csv is created.\n",
|
||||
"\n",
|
||||
"### c. In your run function add:\n",
|
||||
"<code> inputs_dc.collect(data)\n",
|
||||
" prediction_dc.collect(result) </code>"
|
||||
"```python\n",
|
||||
"inputs_dc.collect(data)\n",
|
||||
"prediction_dc.collect(result)```"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -130,7 +124,7 @@
|
||||
"%%writefile score.py\n",
|
||||
"import pickle\n",
|
||||
"import json\n",
|
||||
"import numpy as np\n",
|
||||
"import numpy \n",
|
||||
"from sklearn.externals import joblib\n",
|
||||
"from sklearn.linear_model import Ridge\n",
|
||||
"from azureml.core.model import Model\n",
|
||||
@@ -139,31 +133,33 @@
|
||||
"\n",
|
||||
"def init():\n",
|
||||
" global model\n",
|
||||
" #print (\"model initialized\" + time.strftime(\"%H:%M:%S\"))\n",
|
||||
" # note here \"best_model\" is the name of the model registered under the workspace\n",
|
||||
" print (\"model initialized\" + time.strftime(\"%H:%M:%S\"))\n",
|
||||
" # note here \"sklearn_regression_model.pkl\" is the name of the model registered under the workspace\n",
|
||||
" # this call should return the path to the model.pkl file on the local disk.\n",
|
||||
" model_path = Model.get_model_path(model_name = 'best_model')\n",
|
||||
" model_path = Model.get_model_path(model_name = 'sklearn_regression_model.pkl')\n",
|
||||
" # deserialize the model file back into a sklearn model\n",
|
||||
" model = joblib.load(model_path)\n",
|
||||
" global inputs_dc, prediction_dc\n",
|
||||
" # this setup will help us save our inputs under the \"inputs\" path in our Azure Blob\n",
|
||||
" inputs_dc = ModelDataCollector(model_name=\"best_model\", identifier=\"inputs\", feature_names=[\"feat1\", \"feat2\", \"feat3\",\"feat4\", \"feat5\",\"feat6\"]) \n",
|
||||
" inputs_dc = ModelDataCollector(model_name=\"sklearn_regression_model\", identifier=\"inputs\", feature_names=[\"feat1\", \"feat2\"]) \n",
|
||||
" # this setup will help us save our ipredictions under the \"predictions\" path in our Azure Blob\n",
|
||||
" prediction_dc = ModelDataCollector(\"best_model\", identifier=\"predictions\", feature_names=[\"prediction1\", \"prediction2\"]) \n",
|
||||
" prediction_dc = ModelDataCollector(\"sklearn_regression_model\", identifier=\"predictions\", feature_names=[\"prediction1\", \"prediction2\"]) \n",
|
||||
" \n",
|
||||
"# note you can pass in multiple rows for scoring\n",
|
||||
"def run(raw_data):\n",
|
||||
" global inputs_dc, prediction_dc\n",
|
||||
" try:\n",
|
||||
" data = json.loads(raw_data)['data']\n",
|
||||
" data = np.array(data)\n",
|
||||
" data = numpy.array(data)\n",
|
||||
" result = model.predict(data)\n",
|
||||
" print (\"saving input data\" + time.strftime(\"%H:%M:%S\"))\n",
|
||||
" inputs_dc.collect(data) #this call is saving our input data into our blob\n",
|
||||
" prediction_dc.collect(result)#this call is saving our prediction data into our blob\n",
|
||||
" print (\"saving prediction data\" + time.strftime(\"%H:%M:%S\"))\n",
|
||||
" return json.dumps({\"result\": result.tolist()})\n",
|
||||
" except Exception as e:\n",
|
||||
" result = str(e)\n",
|
||||
" #print (result + time.strftime(\"%H:%M:%S\"))\n",
|
||||
" print (result + time.strftime(\"%H:%M:%S\"))\n",
|
||||
" return json.dumps({\"error\": result})"
|
||||
]
|
||||
},
|
||||
@@ -171,7 +167,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 5. Update your myenv.yml file with the required module"
|
||||
"## 5. *Update your myenv.yml file with the required module*"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -180,18 +176,13 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%writefile myenv.yml\n",
|
||||
"name: myenv\n",
|
||||
"channels:\n",
|
||||
" - defaults\n",
|
||||
"dependencies:\n",
|
||||
" - pip:\n",
|
||||
" - numpy\n",
|
||||
" - scikit-learn\n",
|
||||
" # Required packages for AzureML execution, history, and data preparation.\n",
|
||||
" - --extra-index-url https://azuremlsdktestpypi.azureedge.net/sdk-release/Preview/E7501C02541B433786111FE8E140CAA1\n",
|
||||
" - azureml-core\n",
|
||||
" - azureml-monitoring"
|
||||
"from azureml.core.conda_dependencies import CondaDependencies \n",
|
||||
"\n",
|
||||
"myenv = CondaDependencies.create(conda_packages=['numpy','scikit-learn'])\n",
|
||||
"myenv.add_pip_package(\"azureml-monitoring\")\n",
|
||||
"\n",
|
||||
"with open(\"myenv.yml\",\"w\") as f:\n",
|
||||
" f.write(myenv.serialize_to_string())"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -238,9 +229,30 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 7. Deploy to AKS service\n",
|
||||
"For this step you would need to have an AKS cluster setup (Notebook 11).\n",
|
||||
"In this case we are attaching to a previously created service"
|
||||
"## 7. Deploy to AKS service"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Create AKS compute if you haven't done so (Notebook 11)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Use the default configuration (can also provide parameters to customize)\n",
|
||||
"prov_config = AksCompute.provisioning_configuration()\n",
|
||||
"\n",
|
||||
"aks_name = 'my-aks-test1' \n",
|
||||
"# Create the cluster\n",
|
||||
"aks_target = ComputeTarget.create(workspace = ws, \n",
|
||||
" name = aks_name, \n",
|
||||
" provisioning_configuration = prov_config)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -250,20 +262,40 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"resource_id = '/subscriptions/92c76a2f-0e1c-4216-b65e-abf7a3f34c1e/resourcegroups/marthateresource_groupjw/providers/Microsoft.ContainerService/managedClusters/my-aks-colfb348092fd3a760'\n",
|
||||
"create_name= 'my-existing-aks'\n",
|
||||
"aks_target = AksCompute.attach(workspace = ws, \n",
|
||||
" name = create_name, \n",
|
||||
" resource_id=resource_id)\n",
|
||||
"# Wait for the operation to complete\n",
|
||||
"aks_target.wait_for_completion(True)"
|
||||
"aks_target.wait_for_completion(show_output = True)\n",
|
||||
"print(aks_target.provisioning_state)\n",
|
||||
"print(aks_target.provisioning_errors)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### a. Activate Data Collection and App Insights\n",
|
||||
"If you already have a cluster you can attach the service to it:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"scrolled": true
|
||||
},
|
||||
"source": [
|
||||
"```python \n",
|
||||
" %%time\n",
|
||||
" resource_id = '/subscriptions/<subscriptionid>/resourcegroups/<resourcegroupname>/providers/Microsoft.ContainerService/managedClusters/<aksservername>'\n",
|
||||
" create_name= 'myaks4'\n",
|
||||
" aks_target = AksCompute.attach(workspace = ws, \n",
|
||||
" name = create_name, \n",
|
||||
" #esource_id=resource_id)\n",
|
||||
" ## Wait for the operation to complete\n",
|
||||
" aks_target.wait_for_provisioning(True)```"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### a. *Activate Data Collection and App Insights through updating AKS Webservice configuration*\n",
|
||||
"In order to enable Data Collection and App Insights in your service you will need to update your AKS configuration file:"
|
||||
]
|
||||
},
|
||||
@@ -273,7 +305,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"#Set the web service configuration (using default here)\n",
|
||||
"#Set the web service configuration\n",
|
||||
"aks_config = AksWebservice.deploy_configuration(collect_model_data=True, enable_app_insights=True)"
|
||||
]
|
||||
},
|
||||
@@ -291,7 +323,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"aks_service_name ='aks-w-collv5'\n",
|
||||
"aks_service_name ='aks-w-dc2'\n",
|
||||
"\n",
|
||||
"aks_service = Webservice.deploy_from_image(workspace = ws, \n",
|
||||
" name = aks_service_name,\n",
|
||||
@@ -300,7 +332,7 @@
|
||||
" deployment_target = aks_target\n",
|
||||
" )\n",
|
||||
"aks_service.wait_for_deployment(show_output = True)\n",
|
||||
"print(aks_service.state)\n"
|
||||
"print(aks_service.state)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -333,28 +365,12 @@
|
||||
"print(prediction)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"test_sample = json.dumps({'data': [\n",
|
||||
" [1,22,3,4,5,68,7,98,95,310], \n",
|
||||
" [10,92,8,7,6,53,84,23,323,1]\n",
|
||||
"]})\n",
|
||||
"test_sample = bytes(test_sample,encoding = 'utf8')\n",
|
||||
"\n",
|
||||
"prediction = aks_service.run(input_data = test_sample)\n",
|
||||
"print(prediction)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 9. Validate you data and analyze it\n",
|
||||
"You can look into your data following this path format in your Azure Blob:\n",
|
||||
"You can look into your data following this path format in your Azure Blob (it takes up to 15 minutes for the data to appear):\n",
|
||||
"\n",
|
||||
"/modeldata/**subscriptionid>**/**resourcegroupname>**/**workspacename>**/**webservicename>**/**modelname>**/**modelversion>>**/**identifier>**/*year/month/day*/data.csv \n",
|
||||
"\n",
|
||||
@@ -365,13 +381,13 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### a. Create DataBricks cluster and connect it to your blob\n",
|
||||
"### a. Create DataBricks cluter and connect it to your blob\n",
|
||||
"https://docs.microsoft.com/en-us/azure/azure-databricks/quickstart-create-databricks-workspace-portal or in your databricks workspace you can look for the template \"Azure Blob Storage Import Example Notebook\".\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Here is an example for setting up the file location to extract the relevant data:\n",
|
||||
"\n",
|
||||
"<code> file_location = \"wasbs://mycontainer@testmartstoragendbblgwy.blob.core.windows.net/unknown/unknown/unknown-bigdataset-unknown/my_iterate_parking_inputs/2018/°/°/data.csv\" \n",
|
||||
"<code> file_location = \"wasbs://mycontainer@storageaccountname.blob.core.windows.net/unknown/unknown/unknown-bigdataset-unknown/my_iterate_parking_inputs/2018/°/°/data.csv\" \n",
|
||||
"file_type = \"csv\"</code>\n"
|
||||
]
|
||||
},
|
||||
@@ -405,20 +421,13 @@
|
||||
"source": [
|
||||
"aks_service.update(collect_model_data=False)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"display_name": "Python [conda env:myenv3]",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
"name": "conda-env-myenv3-py"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
@@ -430,7 +439,7 @@
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.6.4"
|
||||
"version": "3.6.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
|
||||
Reference in New Issue
Block a user