mirror of
https://github.com/Azure/MachineLearningNotebooks.git
synced 2025-12-20 09:37:04 -05:00
Compare commits
31 Commits
azureml-sd
...
azureml-sd
| Author | SHA1 | Date | |
|---|---|---|---|
|
8dad09a42f | ||
|
|
db2bf8ae93 | ||
|
|
820c09734f | ||
|
|
a2a33c70a6 | ||
|
|
2ff791968a | ||
|
|
7186127804 | ||
|
|
b01c52bfd6 | ||
|
|
28be7bcf58 | ||
|
|
37a9350fde | ||
|
|
5080053a35 | ||
|
|
3c02102691 | ||
|
|
07e1676762 | ||
|
|
919a3c078f | ||
|
|
9b53c924ed | ||
|
|
04ad58056f | ||
|
|
576bf386b5 | ||
|
|
7e62d1cfd6 | ||
|
|
ec67a569af | ||
|
|
6d1e80bcef | ||
|
|
db00d9ad3c | ||
|
|
d33c75abc3 | ||
|
|
d0dc4836ae | ||
|
|
982f8fcc1d | ||
|
|
79739b5e1b | ||
|
|
aac4fa1fb9 | ||
|
|
5b684070e1 | ||
|
|
0ab8b141ee | ||
|
|
9c32ca9db5 | ||
|
|
745b4f0624 | ||
|
|
119fd0a8f6 | ||
|
|
d4a486827d |
@@ -65,7 +65,7 @@ Visit following repos to see projects contributed by Azure ML users:
|
|||||||
- [UMass Amherst Student Samples](https://github.com/katiehouse3/microsoft-azure-ml-notebooks) - A number of end-to-end machine learning notebooks, including machine translation, image classification, and customer churn, created by students in the 696DS course at UMass Amherst.
|
- [UMass Amherst Student Samples](https://github.com/katiehouse3/microsoft-azure-ml-notebooks) - A number of end-to-end machine learning notebooks, including machine translation, image classification, and customer churn, created by students in the 696DS course at UMass Amherst.
|
||||||
|
|
||||||
## Data/Telemetry
|
## Data/Telemetry
|
||||||
This repository collects usage data and sends it to Mircosoft to help improve our products and services. Read Microsoft's [privacy statement to learn more](https://privacy.microsoft.com/en-US/privacystatement)
|
This repository collects usage data and sends it to Microsoft to help improve our products and services. Read Microsoft's [privacy statement to learn more](https://privacy.microsoft.com/en-US/privacystatement)
|
||||||
|
|
||||||
To opt out of tracking, please go to the raw markdown or .ipynb files and remove the following line of code:
|
To opt out of tracking, please go to the raw markdown or .ipynb files and remove the following line of code:
|
||||||
|
|
||||||
|
|||||||
@@ -103,7 +103,7 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"import azureml.core\n",
|
"import azureml.core\n",
|
||||||
"\n",
|
"\n",
|
||||||
"print(\"This notebook was created using version 1.10.0 of the Azure ML SDK\")\n",
|
"print(\"This notebook was created using version 1.13.0 of the Azure ML SDK\")\n",
|
||||||
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
|
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -1,500 +0,0 @@
|
|||||||
1 0.644 0.247 -0.447 0.862 0.374 0.854 -1.126 -0.790 2.173 1.015 -0.201 1.400 0.000 1.575 1.807 1.607 0.000 1.585 -0.190 -0.744 3.102 0.958 1.061 0.980 0.875 0.581 0.905 0.796
|
|
||||||
0 0.385 1.800 1.037 1.044 0.349 1.502 -0.966 1.734 0.000 0.966 -1.960 -0.249 0.000 1.501 0.465 -0.354 2.548 0.834 -0.440 0.638 3.102 0.695 0.909 0.981 0.803 0.813 1.149 1.116
|
|
||||||
0 1.214 -0.166 0.004 0.505 1.434 0.628 -1.174 -1.230 1.087 0.579 -1.047 -0.118 0.000 0.835 0.340 1.234 2.548 0.711 -1.383 1.355 0.000 0.848 0.911 1.043 0.931 1.058 0.744 0.696
|
|
||||||
1 0.420 1.111 0.137 1.516 -1.657 0.854 0.623 1.605 1.087 1.511 -1.297 0.251 0.000 0.872 -0.368 -0.721 0.000 0.543 0.731 1.424 3.102 1.597 1.282 1.105 0.730 0.148 1.231 1.234
|
|
||||||
0 0.897 -1.703 -1.306 1.022 -0.729 0.836 0.859 -0.333 2.173 1.336 -0.965 0.972 2.215 0.671 1.021 -1.439 0.000 0.493 -2.019 -0.289 0.000 0.805 0.930 0.984 1.430 2.198 1.934 1.684
|
|
||||||
0 0.756 1.126 -0.945 2.355 -0.555 0.889 0.800 1.440 0.000 0.585 0.271 0.631 2.215 0.722 1.744 1.051 0.000 0.618 0.924 0.698 1.551 0.976 0.864 0.988 0.803 0.234 0.822 0.911
|
|
||||||
0 1.141 -0.741 0.953 1.478 -0.524 1.197 -0.871 1.689 2.173 0.875 1.321 -0.518 1.107 0.540 0.037 -0.987 0.000 0.879 1.187 0.245 0.000 0.888 0.701 1.747 1.358 2.479 1.491 1.223
|
|
||||||
1 0.606 -0.936 -0.384 1.257 -1.162 2.719 -0.600 0.100 2.173 3.303 -0.284 1.561 1.107 0.689 1.786 -0.326 0.000 0.780 -0.532 1.216 0.000 0.936 2.022 0.985 1.574 4.323 2.263 1.742
|
|
||||||
1 0.603 0.429 -0.279 1.448 1.301 1.008 2.423 -1.295 0.000 0.452 1.305 0.533 0.000 1.076 1.011 1.256 2.548 2.021 1.260 -0.343 0.000 0.890 0.969 1.281 0.763 0.652 0.827 0.785
|
|
||||||
0 1.171 -0.962 0.521 0.841 -0.315 1.196 -0.744 -0.882 2.173 0.726 -1.305 1.377 1.107 0.643 -1.790 -1.264 0.000 1.257 0.222 0.817 0.000 0.862 0.911 0.987 0.846 1.293 0.899 0.756
|
|
||||||
1 1.392 -0.358 0.235 1.494 -0.461 0.895 -0.848 1.549 2.173 0.841 -0.384 0.666 1.107 1.199 2.509 -0.891 0.000 1.109 -0.364 -0.945 0.000 0.693 2.135 1.170 1.362 0.959 2.056 1.842
|
|
||||||
1 1.024 1.076 -0.886 0.851 1.530 0.673 -0.449 0.187 1.087 0.628 -0.895 1.176 2.215 0.696 -0.232 -0.875 0.000 0.411 1.501 0.048 0.000 0.842 0.919 1.063 1.193 0.777 0.964 0.807
|
|
||||||
1 0.890 -0.760 1.182 1.369 0.751 0.696 -0.959 -0.710 1.087 0.775 -0.130 -1.409 2.215 0.701 -0.110 -0.739 0.000 0.508 -0.451 0.390 0.000 0.762 0.738 0.998 1.126 0.788 0.940 0.790
|
|
||||||
1 0.460 0.537 0.636 1.442 -0.269 0.585 0.323 -1.731 2.173 0.503 1.034 -0.927 0.000 0.928 -1.024 1.006 2.548 0.513 -0.618 -1.336 0.000 0.802 0.831 0.992 1.019 0.925 1.056 0.833
|
|
||||||
1 0.364 1.648 0.560 1.720 0.829 1.110 0.811 -0.588 0.000 0.408 1.045 1.054 2.215 0.319 -1.138 1.545 0.000 0.423 1.025 -1.265 3.102 1.656 0.928 1.003 0.544 0.327 0.670 0.746
|
|
||||||
1 0.525 -0.096 1.206 0.948 -1.103 1.519 -0.582 0.606 2.173 1.274 -0.572 -0.934 0.000 0.855 -1.028 -1.222 0.000 0.578 -1.000 -1.725 3.102 0.896 0.878 0.981 0.498 0.909 0.772 0.668
|
|
||||||
0 0.536 -0.821 -1.029 0.703 1.113 0.363 -0.711 0.022 1.087 0.325 1.503 1.249 2.215 0.673 1.041 -0.401 0.000 0.480 2.127 1.681 0.000 0.767 1.034 0.990 0.671 0.836 0.669 0.663
|
|
||||||
1 1.789 -0.583 1.641 0.897 0.799 0.515 -0.100 -1.483 0.000 1.101 0.031 -0.326 2.215 1.195 0.001 0.126 2.548 0.768 -0.148 0.601 0.000 0.916 0.921 1.207 1.069 0.483 0.934 0.795
|
|
||||||
1 1.332 -0.571 0.986 0.580 1.508 0.582 0.634 -0.746 1.087 1.084 -0.964 -0.489 0.000 0.785 0.274 0.343 2.548 0.779 0.721 1.489 0.000 1.733 1.145 0.990 1.270 0.715 0.897 0.915
|
|
||||||
0 1.123 0.629 -1.708 0.597 -0.882 0.752 0.195 1.522 2.173 1.671 1.515 -0.003 0.000 0.778 0.514 0.139 1.274 0.801 1.260 1.600 0.000 1.495 0.976 0.988 0.676 0.921 1.010 0.943
|
|
||||||
0 1.816 -0.515 0.171 0.980 -0.454 0.870 0.202 -1.399 2.173 1.130 1.066 -1.593 0.000 0.844 0.735 1.275 2.548 1.125 -1.133 0.348 0.000 0.837 0.693 0.988 1.112 0.784 1.009 0.974
|
|
||||||
1 0.364 0.694 0.445 1.862 0.159 0.963 -1.356 1.260 1.087 0.887 -0.540 -1.533 2.215 0.658 -2.544 -1.236 0.000 0.516 -0.807 0.039 0.000 0.891 1.004 0.991 1.092 0.976 1.000 0.953
|
|
||||||
1 0.790 -1.175 0.475 1.846 0.094 0.999 -1.090 0.257 0.000 1.422 0.854 1.112 2.215 1.302 1.004 -1.702 1.274 2.557 -0.787 -1.048 0.000 0.890 1.429 0.993 2.807 0.840 2.248 1.821
|
|
||||||
1 0.765 -0.500 -0.603 1.843 -0.560 1.068 0.007 0.746 2.173 1.154 -0.017 1.329 0.000 1.165 1.791 -1.585 0.000 1.116 0.441 -0.886 0.000 0.774 0.982 0.989 1.102 0.633 1.178 1.021
|
|
||||||
1 1.407 1.293 -1.418 0.502 -1.527 2.005 -2.122 0.622 0.000 1.699 1.508 -0.649 2.215 1.665 0.748 -0.755 0.000 2.555 0.811 1.423 1.551 7.531 5.520 0.985 1.115 1.881 4.487 3.379
|
|
||||||
1 0.772 -0.186 -1.372 0.823 -0.140 0.781 0.763 0.046 2.173 1.128 0.516 1.380 0.000 0.797 -0.640 -0.134 2.548 2.019 -0.972 -1.670 0.000 2.022 1.466 0.989 0.856 0.808 1.230 0.991
|
|
||||||
1 0.546 -0.954 0.715 1.335 -1.689 0.783 -0.443 -1.735 2.173 1.081 0.185 -0.435 0.000 1.433 -0.662 -0.389 0.000 0.969 0.924 1.099 0.000 0.910 0.879 0.988 0.683 0.753 0.878 0.865
|
|
||||||
1 0.596 0.276 -1.054 1.358 1.355 1.444 1.813 -0.208 0.000 1.175 -0.949 -1.573 0.000 0.855 -1.228 -0.925 2.548 1.837 -0.400 0.913 0.000 0.637 0.901 1.028 0.553 0.790 0.679 0.677
|
|
||||||
0 0.458 2.292 1.530 0.291 1.283 0.749 -0.930 -0.198 0.000 0.300 -1.560 0.990 0.000 0.811 -0.176 0.995 2.548 1.085 -0.178 -1.213 3.102 0.891 0.648 0.999 0.732 0.655 0.619 0.620
|
|
||||||
0 0.638 -0.575 -1.048 0.125 0.178 0.846 -0.753 -0.339 1.087 0.799 -0.727 1.182 0.000 0.888 0.283 0.717 0.000 1.051 -1.046 -1.557 3.102 0.889 0.871 0.989 0.884 0.923 0.836 0.779
|
|
||||||
1 0.434 -1.119 -0.313 2.427 0.461 0.497 0.261 -1.177 2.173 0.618 -0.737 -0.688 0.000 1.150 -1.237 -1.652 2.548 0.757 -0.054 1.700 0.000 0.809 0.741 0.982 1.450 0.936 1.086 0.910
|
|
||||||
1 0.431 -1.144 -1.030 0.778 -0.655 0.490 0.047 -1.546 0.000 1.583 -0.014 0.891 2.215 0.516 0.956 0.567 2.548 0.935 -1.123 -0.082 0.000 0.707 0.995 0.995 0.700 0.602 0.770 0.685
|
|
||||||
1 1.894 0.222 1.224 1.578 1.715 0.966 2.890 -0.013 0.000 0.922 -0.703 -0.844 0.000 0.691 2.056 1.039 0.000 0.900 -0.733 -1.240 3.102 1.292 1.992 1.026 0.881 0.684 1.759 1.755
|
|
||||||
0 0.985 -0.316 0.141 1.067 -0.946 0.819 -1.177 1.307 2.173 1.080 -0.429 0.557 1.107 1.726 1.435 -1.075 0.000 1.100 1.547 -0.647 0.000 0.873 1.696 1.179 1.146 1.015 1.538 1.270
|
|
||||||
0 0.998 -0.187 -0.236 0.882 0.755 0.468 0.950 -0.439 2.173 0.579 -0.550 -0.624 0.000 1.847 1.196 1.384 1.274 0.846 1.273 -1.072 0.000 1.194 0.797 1.013 1.319 1.174 0.963 0.898
|
|
||||||
0 0.515 0.246 -0.593 1.082 1.591 0.912 -0.623 -0.957 2.173 0.858 0.418 0.844 0.000 0.948 2.519 1.599 0.000 1.158 1.385 -0.095 3.102 0.973 1.033 0.988 0.998 1.716 1.054 0.901
|
|
||||||
0 0.919 -1.001 1.506 1.389 0.653 0.507 -0.616 -0.689 2.173 0.808 0.536 -0.467 2.215 0.496 2.187 -0.859 0.000 0.822 0.807 1.163 0.000 0.876 0.861 1.088 0.947 0.614 0.911 1.087
|
|
||||||
0 0.794 0.051 1.477 1.504 -1.695 0.716 0.315 0.264 1.087 0.879 -0.135 -1.094 2.215 1.433 -0.741 0.201 0.000 1.566 0.534 -0.989 0.000 0.627 0.882 0.974 0.807 1.130 0.929 0.925
|
|
||||||
1 0.455 -0.946 -1.175 1.453 -0.580 0.763 -0.856 0.840 0.000 0.829 1.223 1.174 2.215 0.714 0.638 -0.466 0.000 1.182 0.223 -1.333 0.000 0.977 0.938 0.986 0.713 0.714 0.796 0.843
|
|
||||||
1 0.662 -0.296 -1.287 1.212 -0.707 0.641 1.457 0.222 0.000 0.600 0.525 -1.700 2.215 0.784 -0.835 -0.961 2.548 0.865 1.131 1.162 0.000 0.854 0.877 0.978 0.740 0.734 0.888 0.811
|
|
||||||
0 0.390 0.698 -1.629 1.888 0.298 0.990 1.614 -1.572 0.000 1.666 0.170 0.719 2.215 1.590 1.064 -0.886 1.274 0.952 0.305 -1.216 0.000 1.048 0.897 1.173 0.891 1.936 1.273 1.102
|
|
||||||
0 1.014 0.117 1.384 0.686 -1.047 0.609 -1.245 -0.850 0.000 1.076 -1.158 0.814 1.107 1.598 -0.389 -0.111 0.000 0.907 1.688 -1.673 0.000 1.333 0.866 0.989 0.975 0.442 0.797 0.788
|
|
||||||
0 1.530 -1.408 -0.207 0.440 -1.357 0.902 -0.647 1.325 1.087 1.320 -0.819 0.246 1.107 0.503 1.407 -1.683 0.000 1.189 -0.972 -0.925 0.000 0.386 1.273 0.988 0.829 1.335 1.173 1.149
|
|
||||||
1 1.689 -0.590 0.915 2.076 1.202 0.644 -0.478 -0.238 0.000 0.809 -1.660 -1.184 0.000 1.227 -0.224 -0.808 2.548 1.655 1.047 -0.623 0.000 0.621 1.192 0.988 1.309 0.866 0.924 1.012
|
|
||||||
0 1.102 0.402 -1.622 1.262 1.022 0.576 0.271 -0.269 0.000 0.591 0.495 -1.278 0.000 1.271 0.209 0.575 2.548 0.941 0.964 -0.685 3.102 0.989 0.963 1.124 0.857 0.858 0.716 0.718
|
|
||||||
0 2.491 0.825 0.581 1.593 0.205 0.782 -0.815 1.499 0.000 1.179 -0.999 -1.509 0.000 0.926 0.920 -0.522 2.548 2.068 -1.021 -1.050 3.102 0.874 0.943 0.980 0.945 1.525 1.570 1.652
|
|
||||||
0 0.666 0.254 1.601 1.303 -0.250 1.236 -1.929 0.793 0.000 1.074 0.447 -0.871 0.000 0.991 1.059 -0.342 0.000 1.703 -0.393 -1.419 3.102 0.921 0.945 1.285 0.931 0.462 0.770 0.729
|
|
||||||
0 0.937 -1.126 1.424 1.395 1.743 0.760 0.428 -0.238 2.173 0.846 0.494 1.320 2.215 0.872 -1.826 -0.507 0.000 0.612 1.860 1.403 0.000 3.402 2.109 0.985 1.298 1.165 1.404 1.240
|
|
||||||
1 0.881 -1.086 -0.870 0.513 0.266 2.049 -1.870 1.160 0.000 2.259 -0.428 -0.935 2.215 1.321 -0.655 -0.449 2.548 1.350 -1.766 -0.108 0.000 0.911 1.852 0.987 1.167 0.820 1.903 1.443
|
|
||||||
0 0.410 0.835 -0.819 1.257 1.112 0.871 -1.737 -0.401 0.000 0.927 0.158 1.253 0.000 1.183 0.405 -1.570 0.000 0.807 -0.704 -0.438 3.102 0.932 0.962 0.987 0.653 0.315 0.616 0.648
|
|
||||||
1 0.634 0.196 -1.679 1.379 -0.967 2.260 -0.273 1.114 0.000 1.458 1.070 -0.278 1.107 1.195 0.110 -0.688 2.548 0.907 0.298 -1.359 0.000 0.949 1.129 0.984 0.675 0.877 0.938 0.824
|
|
||||||
1 0.632 -1.254 1.201 0.496 -0.106 0.235 2.731 -0.955 0.000 0.615 -0.805 0.600 0.000 0.633 -0.934 1.641 0.000 1.407 -0.483 -0.962 1.551 0.778 0.797 0.989 0.578 0.722 0.576 0.539
|
|
||||||
0 0.714 1.122 1.566 2.399 -1.431 1.665 0.299 0.323 0.000 1.489 1.087 -0.861 2.215 1.174 0.140 1.083 2.548 0.404 -0.968 1.105 0.000 0.867 0.969 0.981 1.039 1.552 1.157 1.173
|
|
||||||
1 0.477 -0.321 -0.471 1.966 1.034 2.282 1.359 -0.874 0.000 1.672 -0.258 1.109 0.000 1.537 0.604 0.231 2.548 1.534 -0.640 0.827 0.000 0.746 1.337 1.311 0.653 0.721 0.795 0.742
|
|
||||||
1 1.351 0.460 0.031 1.194 -1.185 0.670 -1.157 -1.637 2.173 0.599 -0.823 0.680 0.000 0.478 0.373 1.716 0.000 0.809 -0.919 0.010 1.551 0.859 0.839 1.564 0.994 0.777 0.971 0.826
|
|
||||||
1 0.520 -1.442 -0.348 0.840 1.654 1.273 -0.760 1.317 0.000 0.861 2.579 -0.791 0.000 1.779 0.257 -0.703 0.000 2.154 1.928 0.457 0.000 1.629 3.194 0.992 0.730 1.107 2.447 2.747
|
|
||||||
0 0.700 -0.308 0.920 0.438 -0.879 0.516 1.409 1.101 0.000 0.960 0.701 -0.049 2.215 1.442 -0.416 -1.439 2.548 0.628 1.009 -0.364 0.000 0.848 0.817 0.987 0.759 1.421 0.937 0.920
|
|
||||||
1 0.720 1.061 -0.546 0.798 -1.521 1.066 0.173 0.271 1.087 1.453 0.114 1.336 1.107 0.702 0.616 -0.367 0.000 0.543 -0.386 -1.301 0.000 0.653 0.948 0.989 1.031 1.500 0.965 0.790
|
|
||||||
1 0.735 -0.416 0.588 1.308 -0.382 1.042 0.344 1.609 0.000 0.926 0.163 -0.520 1.107 1.050 -0.427 1.159 2.548 0.834 0.613 0.948 0.000 0.848 1.189 1.042 0.844 1.099 0.829 0.843
|
|
||||||
1 0.777 -0.396 1.540 1.608 0.638 0.955 0.040 0.918 2.173 1.315 1.116 -0.823 0.000 0.781 -0.762 0.564 2.548 0.945 -0.573 1.379 0.000 0.679 0.706 1.124 0.608 0.593 0.515 0.493
|
|
||||||
1 0.934 0.319 -0.257 0.970 -0.980 0.726 0.774 0.731 0.000 0.896 0.038 -1.465 1.107 0.773 -0.055 -0.831 2.548 1.439 -0.229 0.698 0.000 0.964 1.031 0.995 0.845 0.480 0.810 0.762
|
|
||||||
0 0.461 0.771 0.019 2.055 -1.288 1.043 0.147 0.261 2.173 0.833 -0.156 1.425 0.000 0.832 0.805 -0.491 2.548 0.589 1.252 1.414 0.000 0.850 0.906 1.245 1.364 0.850 0.908 0.863
|
|
||||||
1 0.858 -0.116 -0.937 0.966 1.167 0.825 -0.108 1.111 1.087 0.733 1.163 -0.634 0.000 0.894 0.771 0.020 0.000 0.846 -1.124 -1.195 3.102 0.724 1.194 1.195 0.813 0.969 0.985 0.856
|
|
||||||
0 0.720 -0.335 -0.307 1.445 0.540 1.108 -0.034 -1.691 1.087 0.883 -1.356 -0.678 2.215 0.440 1.093 0.253 0.000 0.389 -1.582 -1.097 0.000 1.113 1.034 0.988 1.256 1.572 1.062 0.904
|
|
||||||
1 0.750 -0.811 -0.542 0.985 0.408 0.471 0.477 0.355 0.000 1.347 -0.875 -1.556 2.215 0.564 1.082 -0.724 0.000 0.793 -0.958 -0.020 3.102 0.836 0.825 0.986 1.066 0.924 0.927 0.883
|
|
||||||
0 0.392 -0.468 -0.216 0.680 1.565 1.086 -0.765 -0.581 1.087 1.264 -1.035 1.189 2.215 0.986 -0.338 0.747 0.000 0.884 -1.328 -0.965 0.000 1.228 0.988 0.982 1.135 1.741 1.108 0.956
|
|
||||||
1 0.434 -1.269 0.643 0.713 0.608 0.597 0.832 1.627 0.000 0.708 -0.422 0.079 2.215 1.533 -0.823 -1.127 2.548 0.408 -1.357 -0.828 0.000 1.331 1.087 0.999 1.075 1.015 0.875 0.809
|
|
||||||
0 0.828 -1.803 0.342 0.847 -0.162 1.585 -1.128 -0.272 2.173 1.974 0.039 -1.717 0.000 0.900 0.764 -1.741 0.000 1.349 -0.079 1.035 3.102 0.984 0.815 0.985 0.780 1.661 1.403 1.184
|
|
||||||
1 1.089 -0.350 -0.747 1.472 0.792 1.087 -0.069 -1.192 0.000 0.512 -0.841 -1.284 0.000 2.162 -0.821 0.545 2.548 1.360 2.243 -0.183 0.000 0.977 0.628 1.725 1.168 0.635 0.823 0.822
|
|
||||||
1 0.444 0.451 -1.332 1.176 -0.247 0.898 0.194 0.007 0.000 1.958 0.576 -1.618 2.215 0.584 1.203 0.268 0.000 0.939 1.033 1.264 3.102 0.829 0.886 0.985 1.265 0.751 1.032 0.948
|
|
||||||
0 0.629 0.114 1.177 0.917 -1.204 0.845 0.828 -0.088 0.000 0.962 -1.302 0.823 2.215 0.732 0.358 -1.334 2.548 0.538 0.582 1.561 0.000 1.028 0.834 0.988 0.904 1.205 1.039 0.885
|
|
||||||
1 1.754 -1.259 -0.573 0.959 -1.483 0.358 0.448 -1.452 0.000 0.711 0.313 0.499 2.215 1.482 -0.390 1.474 2.548 1.879 -1.540 0.668 0.000 0.843 0.825 1.313 1.315 0.939 1.048 0.871
|
|
||||||
1 0.549 0.706 -1.437 0.894 0.891 0.680 -0.762 -1.568 0.000 0.981 0.499 -0.425 2.215 1.332 0.678 0.485 1.274 0.803 0.022 -0.893 0.000 0.793 1.043 0.987 0.761 0.899 0.915 0.794
|
|
||||||
0 0.475 0.542 -0.987 1.569 0.069 0.551 1.543 -1.488 0.000 0.608 0.301 1.734 2.215 0.277 0.499 -0.522 0.000 1.375 1.212 0.696 3.102 0.652 0.756 0.987 0.828 0.830 0.715 0.679
|
|
||||||
1 0.723 0.049 -1.153 1.300 0.083 0.723 -0.749 0.630 0.000 1.126 0.412 -0.384 0.000 1.272 1.256 1.358 2.548 3.108 0.777 -1.486 3.102 0.733 1.096 1.206 1.269 0.899 1.015 0.903
|
|
||||||
1 1.062 0.296 0.725 0.285 -0.531 0.819 1.277 -0.667 0.000 0.687 0.829 -0.092 0.000 1.158 0.447 1.047 2.548 1.444 -0.186 -1.491 3.102 0.863 1.171 0.986 0.769 0.828 0.919 0.840
|
|
||||||
0 0.572 -0.349 1.396 2.023 0.795 0.577 0.457 -0.533 0.000 1.351 0.701 -1.091 0.000 0.724 -1.012 -0.182 2.548 0.923 -0.012 0.789 3.102 0.936 1.025 0.985 1.002 0.600 0.828 0.909
|
|
||||||
1 0.563 0.387 0.412 0.553 1.050 0.723 -0.992 -0.447 0.000 0.748 0.948 0.546 2.215 1.761 -0.559 -1.183 0.000 1.114 -0.251 1.192 3.102 0.936 0.912 0.976 0.578 0.722 0.829 0.892
|
|
||||||
1 1.632 1.577 -0.697 0.708 -1.263 0.863 0.012 1.197 2.173 0.498 0.990 -0.806 0.000 0.627 2.387 -1.283 0.000 0.607 1.290 -0.174 3.102 0.916 1.328 0.986 0.557 0.971 0.935 0.836
|
|
||||||
1 0.562 -0.360 0.399 0.803 -1.334 1.443 -0.116 1.628 2.173 0.750 0.987 0.135 1.107 0.795 0.298 -0.556 0.000 1.150 -0.113 -0.093 0.000 0.493 1.332 0.985 1.001 1.750 1.013 0.886
|
|
||||||
1 0.987 0.706 -0.492 0.861 0.607 0.593 0.088 -0.184 0.000 0.802 0.894 1.608 2.215 0.782 -0.471 1.500 2.548 0.521 0.772 -0.960 0.000 0.658 0.893 1.068 0.877 0.664 0.709 0.661
|
|
||||||
1 1.052 0.883 -0.581 1.566 0.860 0.931 1.515 -0.873 0.000 0.493 0.145 -0.672 0.000 1.133 0.935 1.581 2.548 1.630 0.695 0.923 3.102 1.105 1.087 1.713 0.948 0.590 0.872 0.883
|
|
||||||
1 2.130 -0.516 -0.291 0.776 -1.230 0.689 -0.257 0.800 2.173 0.730 -0.274 -1.437 0.000 0.615 0.241 1.083 0.000 0.834 0.757 1.613 3.102 0.836 0.806 1.333 1.061 0.730 0.889 0.783
|
|
||||||
1 0.742 0.797 1.628 0.311 -0.418 0.620 0.685 -1.457 0.000 0.683 1.774 -1.082 0.000 1.700 1.104 0.225 2.548 0.382 -2.184 -1.307 0.000 0.945 1.228 0.984 0.864 0.931 0.988 0.838
|
|
||||||
0 0.311 -1.249 -0.927 1.272 -1.262 0.642 -1.228 -0.136 0.000 1.220 -0.804 -1.558 2.215 0.950 -0.828 0.495 1.274 2.149 -1.672 0.634 0.000 1.346 0.887 0.981 0.856 1.101 1.001 1.106
|
|
||||||
0 0.660 -1.834 -0.667 0.601 1.236 0.932 -0.933 -0.135 2.173 1.373 -0.122 1.429 0.000 0.654 -0.034 -0.847 2.548 0.711 0.911 0.703 0.000 1.144 0.942 0.984 0.822 0.739 0.992 0.895
|
|
||||||
0 3.609 -0.590 0.851 0.615 0.455 1.280 0.003 -0.866 1.087 1.334 0.708 -1.131 0.000 0.669 0.480 0.092 0.000 0.975 0.983 -1.429 3.102 1.301 1.089 0.987 1.476 0.934 1.469 1.352
|
|
||||||
1 0.905 -0.403 1.567 2.651 0.953 1.194 -0.241 -0.567 1.087 0.308 -0.384 -0.007 0.000 0.608 -0.175 -1.163 2.548 0.379 0.941 1.662 0.000 0.580 0.721 1.126 0.895 0.544 1.097 0.836
|
|
||||||
1 0.983 0.255 1.093 0.905 -0.874 0.863 0.060 -0.368 0.000 0.824 -0.747 -0.633 0.000 0.614 0.961 1.052 0.000 0.792 -0.260 1.632 3.102 0.874 0.883 1.280 0.663 0.406 0.592 0.645
|
|
||||||
1 1.160 -1.027 0.274 0.460 0.322 2.085 -1.623 -0.840 0.000 1.634 -1.046 1.182 2.215 0.492 -0.367 1.174 0.000 0.824 -0.998 1.617 0.000 0.943 0.884 1.001 1.209 1.313 1.034 0.866
|
|
||||||
0 0.299 0.028 -1.372 1.930 -0.661 0.840 -0.979 0.664 1.087 0.535 -2.041 1.434 0.000 1.087 -1.797 0.344 0.000 0.485 -0.560 -1.105 3.102 0.951 0.890 0.980 0.483 0.684 0.730 0.706
|
|
||||||
0 0.293 1.737 -1.418 2.074 0.794 0.679 1.024 -1.457 0.000 1.034 1.094 -0.168 1.107 0.506 1.680 -0.661 0.000 0.523 -0.042 -1.274 3.102 0.820 0.944 0.987 0.842 0.694 0.761 0.750
|
|
||||||
0 0.457 -0.393 1.560 0.738 -0.007 0.475 -0.230 0.246 0.000 0.776 -1.264 -0.606 2.215 0.865 -0.731 -1.576 2.548 1.153 0.343 1.436 0.000 1.060 0.883 0.988 0.972 0.703 0.758 0.720
|
|
||||||
0 0.935 -0.582 0.240 2.401 0.818 1.231 -0.618 -1.289 0.000 0.799 0.544 -0.228 2.215 0.525 -1.494 -0.969 0.000 0.609 -1.123 1.168 3.102 0.871 0.767 1.035 1.154 0.919 0.868 1.006
|
|
||||||
1 0.902 -0.745 -1.215 1.174 -0.501 1.215 0.167 1.162 0.000 0.896 1.217 -0.976 0.000 0.585 -0.429 1.036 0.000 1.431 -0.416 0.151 3.102 0.524 0.952 0.990 0.707 0.271 0.592 0.826
|
|
||||||
1 0.653 0.337 -0.320 1.118 -0.934 1.050 0.745 0.529 1.087 1.075 1.742 -1.538 0.000 0.585 1.090 0.973 0.000 1.091 -0.187 1.160 1.551 1.006 1.108 0.978 1.121 0.838 0.947 0.908
|
|
||||||
0 1.157 1.401 0.340 0.395 -1.218 0.945 1.928 -0.876 0.000 1.384 0.320 1.002 1.107 1.900 1.177 -0.462 2.548 1.122 1.316 1.720 0.000 1.167 1.096 0.989 0.937 1.879 1.307 1.041
|
|
||||||
0 0.960 0.355 -0.152 0.872 -0.338 0.391 0.348 0.956 1.087 0.469 2.664 1.409 0.000 0.756 -1.561 1.500 0.000 0.525 1.436 1.728 3.102 1.032 0.946 0.996 0.929 0.470 0.698 0.898
|
|
||||||
1 1.038 0.274 0.825 1.198 0.963 1.078 -0.496 -1.014 2.173 0.739 -0.727 -0.151 2.215 1.035 -0.799 0.398 0.000 1.333 -0.872 -1.498 0.000 0.849 1.033 0.985 0.886 0.936 0.975 0.823
|
|
||||||
0 0.490 0.277 0.318 1.303 0.694 1.333 -1.620 -0.563 0.000 1.459 -1.326 1.140 0.000 0.779 -0.673 -1.324 2.548 0.860 -1.247 0.043 0.000 0.857 0.932 0.992 0.792 0.278 0.841 1.498
|
|
||||||
0 1.648 -0.688 -1.386 2.790 0.995 1.087 1.359 -0.687 0.000 1.050 -0.223 -0.261 2.215 0.613 -0.889 1.335 0.000 1.204 0.827 0.309 3.102 0.464 0.973 2.493 1.737 0.827 1.319 1.062
|
|
||||||
0 1.510 -0.662 1.668 0.860 0.280 0.705 0.974 -1.647 1.087 0.662 -0.393 -0.225 0.000 0.610 -0.996 0.532 2.548 0.464 1.305 0.102 0.000 0.859 1.057 1.498 0.799 1.260 0.946 0.863
|
|
||||||
1 0.850 -1.185 -0.117 0.943 -0.449 1.142 0.875 -0.030 0.000 2.223 -0.461 1.627 2.215 0.767 -1.761 -1.692 0.000 1.012 -0.727 0.639 3.102 3.649 2.062 0.985 1.478 1.087 1.659 1.358
|
|
||||||
0 0.933 1.259 0.130 0.326 -0.890 0.306 1.136 1.142 0.000 0.964 0.705 -1.373 2.215 0.546 -0.196 -0.001 0.000 0.578 -1.169 1.004 3.102 0.830 0.836 0.988 0.837 1.031 0.749 0.655
|
|
||||||
0 0.471 0.697 1.570 1.109 0.201 1.248 0.348 -1.448 0.000 2.103 0.773 0.686 2.215 1.451 -0.087 -0.453 2.548 1.197 -0.045 -1.026 0.000 0.793 1.094 0.987 0.851 1.804 1.378 1.089
|
|
||||||
1 2.446 -0.701 -1.568 0.059 0.822 1.401 -0.600 -0.044 2.173 0.324 -0.001 1.344 2.215 0.913 -0.818 1.049 0.000 0.442 -1.088 -0.005 0.000 0.611 1.062 0.979 0.562 0.988 0.998 0.806
|
|
||||||
0 0.619 2.029 0.933 0.528 -0.903 0.974 0.760 -0.311 2.173 0.825 0.658 -1.466 1.107 0.894 1.594 0.370 0.000 0.882 -0.258 1.661 0.000 1.498 1.088 0.987 0.867 1.139 0.900 0.779
|
|
||||||
1 0.674 -0.131 -0.362 0.518 -1.574 0.876 0.442 0.145 1.087 0.497 -1.526 -1.704 0.000 0.680 2.514 -1.374 0.000 0.792 -0.479 0.773 1.551 0.573 1.198 0.984 0.800 0.667 0.987 0.832
|
|
||||||
1 1.447 1.145 -0.937 0.307 -1.458 0.478 1.264 0.816 1.087 0.558 1.015 -0.101 2.215 0.937 -0.190 1.177 0.000 0.699 0.954 -1.512 0.000 0.877 0.838 0.990 0.873 0.566 0.646 0.713
|
|
||||||
1 0.976 0.308 -0.844 0.436 0.610 1.253 0.149 -1.585 2.173 1.415 0.568 0.096 2.215 0.953 -0.855 0.441 0.000 0.867 -0.650 1.643 0.000 0.890 1.234 0.988 0.796 2.002 1.179 0.977
|
|
||||||
0 0.697 0.401 -0.718 0.920 0.735 0.958 -0.172 0.168 2.173 0.872 -0.097 -1.335 0.000 0.513 -1.192 -1.710 1.274 0.426 -1.637 1.368 0.000 0.997 1.227 1.072 0.800 1.013 0.786 0.749
|
|
||||||
1 1.305 -2.157 1.740 0.661 -0.912 0.705 -0.516 0.759 2.173 0.989 -0.716 -0.300 2.215 0.627 -1.052 -1.736 0.000 0.467 -2.467 0.568 0.000 0.807 0.964 0.988 1.427 1.012 1.165 0.926
|
|
||||||
0 1.847 1.663 -0.618 0.280 1.258 1.462 -0.054 1.371 0.000 0.900 0.309 -0.544 0.000 0.331 -2.149 -0.341 0.000 1.091 -0.833 0.710 3.102 1.496 0.931 0.989 1.549 0.115 1.140 1.150
|
|
||||||
0 0.410 -0.323 1.069 2.160 0.010 0.892 0.942 -1.640 2.173 0.946 0.938 1.314 0.000 1.213 -1.099 -0.794 2.548 0.650 0.053 0.056 0.000 1.041 0.916 1.063 0.985 1.910 1.246 1.107
|
|
||||||
1 0.576 1.092 -0.088 0.777 -1.579 0.757 0.271 0.109 0.000 0.819 0.827 -1.554 2.215 1.313 2.341 -1.568 0.000 2.827 0.239 -0.338 0.000 0.876 0.759 0.986 0.692 0.457 0.796 0.791
|
|
||||||
1 0.537 0.925 -1.406 0.306 -0.050 0.906 1.051 0.037 0.000 1.469 -0.177 -1.320 2.215 1.872 0.723 1.158 0.000 1.313 0.227 -0.501 3.102 0.953 0.727 0.978 0.755 0.892 0.932 0.781
|
|
||||||
0 0.716 -0.065 -0.484 1.313 -1.563 0.596 -0.242 0.678 2.173 0.426 -1.909 0.616 0.000 0.885 -0.406 -1.343 2.548 0.501 -1.327 -0.340 0.000 0.470 0.728 1.109 0.919 0.881 0.665 0.692
|
|
||||||
1 0.624 -0.389 0.128 1.636 -1.110 1.025 0.573 -0.843 2.173 0.646 -0.697 1.064 0.000 0.632 -1.442 0.961 0.000 0.863 -0.106 1.717 0.000 0.825 0.917 1.257 0.983 0.713 0.890 0.824
|
|
||||||
0 0.484 2.101 1.714 1.131 -0.823 0.750 0.583 -1.304 1.087 0.894 0.421 0.559 2.215 0.921 -0.063 0.282 0.000 0.463 -0.474 -1.387 0.000 0.742 0.886 0.995 0.993 1.201 0.806 0.754
|
|
||||||
0 0.570 0.339 -1.478 0.528 0.439 0.978 1.479 -1.411 2.173 0.763 1.541 -0.734 0.000 1.375 0.840 0.903 0.000 0.965 1.599 0.364 0.000 0.887 1.061 0.992 1.322 1.453 1.013 0.969
|
|
||||||
0 0.940 1.303 1.636 0.851 -1.732 0.803 -0.030 -0.177 0.000 0.480 -0.125 -0.954 0.000 0.944 0.709 0.296 2.548 1.342 -0.418 1.197 3.102 0.853 0.989 0.979 0.873 0.858 0.719 0.786
|
|
||||||
1 0.599 0.544 -0.238 0.816 1.043 0.857 0.660 1.128 2.173 0.864 -0.624 -0.843 0.000 1.159 0.367 0.174 0.000 1.520 -0.543 -1.508 0.000 0.842 0.828 0.984 0.759 0.895 0.918 0.791
|
|
||||||
1 1.651 1.897 -0.914 0.423 0.315 0.453 0.619 -1.607 2.173 0.532 -0.424 0.209 1.107 0.369 2.479 0.034 0.000 0.701 0.217 0.984 0.000 0.976 0.951 1.035 0.879 0.825 0.915 0.798
|
|
||||||
1 0.926 -0.574 -0.763 0.285 1.094 0.672 2.314 1.545 0.000 1.124 0.415 0.809 0.000 1.387 0.270 -0.949 2.548 1.547 -0.631 -0.200 3.102 0.719 0.920 0.986 0.889 0.933 0.797 0.777
|
|
||||||
0 0.677 1.698 -0.890 0.641 -0.449 0.607 1.754 1.720 0.000 0.776 0.372 0.782 2.215 0.511 1.491 -0.480 0.000 0.547 -0.341 0.853 3.102 0.919 1.026 0.997 0.696 0.242 0.694 0.687
|
|
||||||
0 1.266 0.602 0.958 0.487 1.256 0.709 0.843 -1.196 0.000 0.893 1.303 -0.594 1.107 1.090 1.320 0.354 0.000 0.797 1.846 1.139 0.000 0.780 0.896 0.986 0.661 0.709 0.790 0.806
|
|
||||||
1 0.628 -0.616 -0.329 0.764 -1.150 0.477 -0.715 1.187 2.173 1.250 0.607 1.026 2.215 0.983 -0.023 -0.583 0.000 0.377 1.344 -1.015 0.000 0.744 0.954 0.987 0.837 0.841 0.795 0.694
|
|
||||||
1 1.035 -0.828 -1.358 1.870 -1.060 1.075 0.130 0.448 2.173 0.660 0.697 0.641 0.000 0.425 1.006 -1.035 0.000 0.751 1.055 1.364 3.102 0.826 0.822 0.988 0.967 0.901 1.077 0.906
|
|
||||||
1 0.830 0.265 -0.150 0.660 1.105 0.592 -0.557 0.908 2.173 0.670 -1.419 -0.671 0.000 1.323 -0.409 1.644 2.548 0.850 -0.033 -0.615 0.000 0.760 0.967 0.984 0.895 0.681 0.747 0.770
|
|
||||||
1 1.395 1.100 1.167 1.088 0.218 0.400 -0.132 0.024 2.173 0.743 0.530 -1.361 2.215 0.341 -0.691 -0.238 0.000 0.396 -1.426 -0.933 0.000 0.363 0.472 1.287 0.922 0.810 0.792 0.656
|
|
||||||
1 1.070 1.875 -1.298 1.215 -0.106 0.767 0.795 0.514 1.087 0.401 2.780 1.276 0.000 0.686 1.127 1.721 2.548 0.391 -0.259 -1.167 0.000 1.278 1.113 1.389 0.852 0.824 0.838 0.785
|
|
||||||
0 1.114 -0.071 1.719 0.399 -1.383 0.849 0.254 0.481 0.000 0.958 -0.579 0.742 0.000 1.190 -0.140 -0.862 2.548 0.479 1.390 0.856 0.000 0.952 0.988 0.985 0.764 0.419 0.835 0.827
|
|
||||||
0 0.714 0.376 -0.568 1.578 -1.165 0.648 0.141 0.639 2.173 0.472 0.569 1.449 1.107 0.783 1.483 0.361 0.000 0.540 -0.790 0.032 0.000 0.883 0.811 0.982 0.775 0.572 0.760 0.745
|
|
||||||
0 0.401 -1.731 0.765 0.974 1.648 0.652 -1.024 0.191 0.000 0.544 -0.366 -1.246 2.215 0.627 0.140 1.008 2.548 0.810 0.409 0.429 0.000 0.950 0.934 0.977 0.621 0.580 0.677 0.650
|
|
||||||
1 0.391 1.679 -1.298 0.605 -0.832 0.549 1.338 0.522 2.173 1.244 0.884 1.070 0.000 1.002 0.846 -1.345 2.548 0.783 -2.464 -0.237 0.000 4.515 2.854 0.981 0.877 0.939 1.942 1.489
|
|
||||||
1 0.513 -0.220 -0.444 1.699 0.479 1.109 0.181 -0.999 2.173 0.883 -0.335 -1.716 2.215 1.075 -0.380 1.352 0.000 0.857 0.048 0.147 0.000 0.937 0.758 0.986 1.206 0.958 0.949 0.876
|
|
||||||
0 1.367 -0.388 0.798 1.158 1.078 0.811 -1.024 -1.628 0.000 1.504 0.097 -0.999 2.215 1.652 -0.860 0.054 2.548 0.573 -0.142 -1.401 0.000 0.869 0.833 1.006 1.412 1.641 1.214 1.041
|
|
||||||
1 1.545 -0.533 -1.517 1.177 1.289 2.331 -0.370 -0.073 0.000 1.295 -0.358 -0.891 2.215 0.476 0.756 0.985 0.000 1.945 -0.016 -1.651 3.102 1.962 1.692 1.073 0.656 0.941 1.312 1.242
|
|
||||||
0 0.858 0.978 -1.258 0.286 0.161 0.729 1.230 1.087 2.173 0.561 2.670 -0.109 0.000 0.407 2.346 0.938 0.000 1.078 0.729 -0.658 3.102 0.597 0.921 0.982 0.579 0.954 0.733 0.769
|
|
||||||
1 1.454 -1.384 0.870 0.067 0.394 1.033 -0.673 0.318 0.000 1.166 -0.763 -1.533 2.215 2.848 -0.045 -0.856 2.548 0.697 -0.140 1.134 0.000 0.931 1.293 0.977 1.541 1.326 1.201 1.078
|
|
||||||
1 0.559 -0.913 0.486 1.104 -0.321 1.073 -0.348 1.345 0.000 0.901 -0.827 -0.842 0.000 0.739 0.047 -0.415 2.548 0.433 -1.132 1.268 0.000 0.797 0.695 0.985 0.868 0.346 0.674 0.623
|
|
||||||
1 1.333 0.780 -0.964 0.916 1.202 1.822 -0.071 0.742 2.173 1.486 -0.399 -0.824 0.000 0.740 0.568 -0.134 0.000 0.971 -0.070 -1.589 3.102 1.278 0.929 1.421 1.608 1.214 1.215 1.137
|
|
||||||
1 2.417 0.631 -0.317 0.323 0.581 0.841 1.524 -1.738 0.000 0.543 1.176 -0.325 0.000 0.827 0.700 0.866 0.000 0.834 -0.262 -1.702 3.102 0.932 0.820 0.988 0.646 0.287 0.595 0.589
|
|
||||||
0 0.955 -1.242 0.938 1.104 0.474 0.798 -0.743 1.535 0.000 1.356 -1.357 -1.080 2.215 1.320 -1.396 -0.132 2.548 0.728 -0.529 -0.633 0.000 0.832 0.841 0.988 0.923 1.077 0.988 0.816
|
|
||||||
1 1.305 -1.918 0.391 1.161 0.063 0.724 2.593 1.481 0.000 0.592 -1.207 -0.329 0.000 0.886 -0.836 -1.168 2.548 1.067 -1.481 -1.440 0.000 0.916 0.688 0.991 0.969 0.550 0.665 0.638
|
|
||||||
0 1.201 0.071 -1.123 2.242 -1.533 0.702 -0.256 0.688 0.000 0.967 0.491 1.040 2.215 1.271 -0.558 0.095 0.000 1.504 0.676 -0.383 3.102 0.917 1.006 0.985 1.017 1.057 0.928 1.057
|
|
||||||
0 0.994 -1.607 1.596 0.774 -1.391 0.625 -0.134 -0.862 2.173 0.746 -0.765 -0.316 2.215 1.131 -0.320 0.869 0.000 0.607 0.826 0.301 0.000 0.798 0.967 0.999 0.880 0.581 0.712 0.774
|
|
||||||
1 0.482 -0.467 0.729 1.419 1.458 0.824 0.376 -0.242 0.000 1.368 0.023 1.459 2.215 0.826 0.669 -1.079 2.548 0.936 2.215 -0.309 0.000 1.883 1.216 0.997 1.065 0.946 1.224 1.526
|
|
||||||
1 0.383 1.588 1.611 0.748 1.194 0.866 -0.279 -0.636 0.000 0.707 0.536 0.801 2.215 1.647 -1.155 0.367 0.000 1.292 0.303 -1.681 3.102 2.016 1.581 0.986 0.584 0.684 1.107 0.958
|
|
||||||
0 0.629 0.203 0.736 0.671 -0.271 1.350 -0.486 0.761 2.173 0.496 -0.805 -1.718 0.000 2.393 0.044 -1.046 1.274 0.651 -0.116 -0.541 0.000 0.697 1.006 0.987 1.069 2.317 1.152 0.902
|
|
||||||
0 0.905 -0.564 -0.570 0.263 1.096 1.219 -1.397 -1.414 1.087 1.164 -0.533 -0.208 0.000 1.459 1.965 0.784 0.000 2.220 -1.421 0.452 0.000 0.918 1.360 0.993 0.904 0.389 2.118 1.707
|
|
||||||
1 1.676 1.804 1.171 0.529 1.175 1.664 0.354 -0.530 0.000 1.004 0.691 -1.280 2.215 0.838 0.373 0.626 2.548 1.094 1.774 0.501 0.000 0.806 1.100 0.991 0.769 0.976 0.807 0.740
|
|
||||||
1 1.364 -1.936 0.020 1.327 0.428 1.021 -1.665 -0.907 2.173 0.818 -2.701 1.303 0.000 0.716 -0.590 -1.629 2.548 0.895 -2.280 -1.602 0.000 1.211 0.849 0.989 1.320 0.864 1.065 0.949
|
|
||||||
0 0.629 -0.626 0.609 1.828 1.280 0.644 -0.856 -0.873 2.173 0.555 1.066 -0.640 0.000 0.477 -1.364 -1.021 2.548 1.017 0.036 0.380 0.000 0.947 0.941 0.994 1.128 0.241 0.793 0.815
|
|
||||||
1 1.152 -0.843 0.926 1.802 0.800 2.493 -1.449 -1.127 0.000 1.737 0.833 0.488 0.000 1.026 0.929 -0.990 2.548 1.408 0.689 1.142 3.102 1.171 0.956 0.993 2.009 0.867 1.499 1.474
|
|
||||||
0 2.204 0.081 0.008 1.021 -0.679 2.676 0.090 1.163 0.000 2.210 -1.686 -1.195 0.000 1.805 0.891 -0.148 2.548 0.450 -0.502 -1.295 3.102 6.959 3.492 1.205 0.908 0.845 2.690 2.183
|
|
||||||
1 0.957 0.954 1.702 0.043 -0.503 1.113 0.033 -0.308 0.000 0.757 -0.363 -1.129 2.215 1.635 0.068 1.048 1.274 0.415 -2.098 0.061 0.000 1.010 0.979 0.992 0.704 1.125 0.761 0.715
|
|
||||||
0 1.222 0.418 1.059 1.303 1.442 0.282 -1.499 -1.286 0.000 1.567 0.016 -0.164 2.215 0.451 2.229 -1.229 0.000 0.660 -0.513 -0.296 3.102 2.284 1.340 0.985 1.531 0.314 1.032 1.094
|
|
||||||
1 0.603 1.675 -0.973 0.703 -1.709 1.023 0.652 1.296 2.173 1.078 0.363 -0.263 0.000 0.734 -0.457 -0.745 1.274 0.561 1.434 -0.042 0.000 0.888 0.771 0.984 0.847 1.234 0.874 0.777
|
|
||||||
0 0.897 0.949 -0.848 1.115 -0.085 0.522 -1.267 -1.418 0.000 0.684 -0.599 1.474 0.000 1.176 0.922 0.641 2.548 0.470 0.103 0.148 3.102 0.775 0.697 0.984 0.839 0.358 0.847 1.008
|
|
||||||
1 0.987 1.013 -1.504 0.468 -0.259 1.160 0.476 -0.971 2.173 1.266 0.919 0.780 0.000 0.634 1.695 0.233 0.000 0.487 -0.082 0.719 3.102 0.921 0.641 0.991 0.730 0.828 0.952 0.807
|
|
||||||
1 0.847 1.581 -1.397 1.629 1.529 1.053 0.816 -0.344 2.173 0.895 0.779 0.332 0.000 0.750 1.311 0.419 2.548 1.604 0.844 1.367 0.000 1.265 0.798 0.989 1.328 0.783 0.930 0.879
|
|
||||||
1 0.805 1.416 -1.327 0.397 0.589 0.488 0.982 0.843 0.000 0.664 -0.999 0.129 0.000 0.624 0.613 -0.558 0.000 1.431 -0.667 -1.561 3.102 0.959 1.103 0.989 0.590 0.632 0.926 0.798
|
|
||||||
0 1.220 -0.313 -0.489 1.759 0.201 1.698 -0.220 0.241 2.173 1.294 1.390 -1.682 0.000 1.447 -1.623 -1.296 0.000 1.710 0.872 -1.356 3.102 1.198 0.981 1.184 0.859 2.165 1.807 1.661
|
|
||||||
0 0.772 -0.611 -0.549 0.465 -1.528 1.103 -0.140 0.001 2.173 0.854 -0.406 1.655 0.000 0.733 -1.250 1.072 0.000 0.883 0.627 -1.132 3.102 0.856 0.927 0.987 1.094 1.013 0.938 0.870
|
|
||||||
1 1.910 0.771 0.828 0.231 1.267 1.398 1.455 -0.295 2.173 0.837 -2.564 0.770 0.000 0.540 2.189 1.287 0.000 1.345 1.311 -1.151 0.000 0.861 0.869 0.984 1.359 1.562 1.105 0.963
|
|
||||||
1 0.295 0.832 1.399 1.222 -0.517 2.480 0.013 1.591 0.000 2.289 0.436 0.287 2.215 1.995 -0.367 -0.409 1.274 0.375 1.367 -1.716 0.000 1.356 2.171 0.990 1.467 1.664 1.855 1.705
|
|
||||||
1 1.228 0.339 -0.575 0.417 1.474 0.480 -1.416 -1.498 2.173 0.614 -0.933 -0.961 0.000 1.189 1.690 1.003 0.000 1.690 -1.065 0.106 3.102 0.963 1.147 0.987 1.086 0.948 0.930 0.866
|
|
||||||
0 2.877 -1.014 1.440 0.782 0.483 1.134 -0.735 -0.196 2.173 1.123 0.084 -0.596 0.000 1.796 -0.356 1.044 2.548 1.406 1.582 -0.991 0.000 0.939 1.178 1.576 0.996 1.629 1.216 1.280
|
|
||||||
1 2.178 0.259 1.107 0.256 1.222 0.979 -0.440 -0.538 1.087 0.496 -0.760 -0.049 0.000 1.471 1.683 -1.486 0.000 0.646 0.695 -1.577 3.102 1.093 1.070 0.984 0.608 0.889 0.962 0.866
|
|
||||||
1 0.604 0.592 1.295 0.964 0.348 1.178 -0.016 0.832 2.173 1.626 -0.420 -0.760 0.000 0.748 0.461 -0.906 0.000 0.728 0.309 -1.269 1.551 0.852 0.604 0.989 0.678 0.949 1.021 0.878
|
|
||||||
0 0.428 -1.352 -0.912 1.713 0.797 1.894 -1.452 0.191 2.173 2.378 2.113 -1.190 0.000 0.860 2.174 0.949 0.000 1.693 0.759 1.426 3.102 0.885 1.527 1.186 1.090 3.294 4.492 3.676
|
|
||||||
0 0.473 0.485 0.154 1.433 -1.504 0.766 1.257 -1.302 2.173 0.414 0.119 0.238 0.000 0.805 0.242 -0.691 2.548 0.734 0.749 0.753 0.000 0.430 0.893 1.137 0.686 0.724 0.618 0.608
|
|
||||||
1 0.763 -0.601 0.876 0.182 -1.678 0.818 0.599 0.481 2.173 0.658 -0.737 -0.553 0.000 0.857 -1.138 -1.435 0.000 1.540 -1.466 -0.447 0.000 0.870 0.566 0.989 0.728 0.658 0.821 0.726
|
|
||||||
0 0.619 -0.273 -0.143 0.992 -1.267 0.566 0.876 -1.396 2.173 0.515 0.892 0.618 0.000 0.434 -0.902 0.862 2.548 0.490 -0.539 0.549 0.000 0.568 0.794 0.984 0.667 0.867 0.597 0.578
|
|
||||||
0 0.793 0.970 0.324 0.570 0.816 0.761 -0.550 1.519 2.173 1.150 0.496 -0.447 0.000 0.925 0.724 1.008 1.274 1.135 -0.275 -0.843 0.000 0.829 1.068 0.978 1.603 0.892 1.041 1.059
|
|
||||||
1 0.480 0.364 -0.067 1.906 -1.582 1.397 1.159 0.140 0.000 0.639 0.398 -1.102 0.000 1.597 -0.668 1.607 2.548 1.306 -0.797 0.288 3.102 0.856 1.259 1.297 1.022 1.032 1.049 0.939
|
|
||||||
0 0.514 1.304 1.490 1.741 -0.220 0.648 0.155 0.535 0.000 0.562 -1.016 0.837 0.000 0.863 -0.780 -0.815 2.548 1.688 -0.130 -1.545 3.102 0.887 0.980 1.309 1.269 0.654 1.044 1.035
|
|
||||||
0 1.225 0.333 0.656 0.893 0.859 1.037 -0.876 1.603 1.087 1.769 0.272 -0.227 2.215 1.000 0.579 -1.690 0.000 1.385 0.471 -0.860 0.000 0.884 1.207 0.995 1.097 2.336 1.282 1.145
|
|
||||||
0 2.044 -1.472 -0.294 0.392 0.369 0.927 0.718 1.492 1.087 1.619 -0.736 0.047 2.215 1.884 -0.101 -1.540 0.000 0.548 -0.441 1.117 0.000 0.798 0.877 0.981 0.750 2.272 1.469 1.276
|
|
||||||
0 1.037 -0.276 0.735 3.526 1.156 2.498 0.401 -0.590 1.087 0.714 -1.203 1.393 2.215 0.681 0.629 1.534 0.000 0.719 -0.355 -0.706 0.000 0.831 0.857 0.988 2.864 2.633 1.988 1.466
|
|
||||||
1 0.651 -1.218 -0.791 0.770 -1.449 0.610 -0.535 0.960 2.173 0.380 -1.072 -0.031 2.215 0.415 2.123 -1.100 0.000 0.776 0.217 0.420 0.000 0.986 1.008 1.001 0.853 0.588 0.799 0.776
|
|
||||||
0 1.586 -0.409 0.085 3.258 0.405 1.647 -0.674 -1.519 0.000 0.640 -1.027 -1.681 0.000 1.452 -0.444 -0.957 2.548 0.927 -0.017 1.215 3.102 0.519 0.866 0.992 0.881 0.847 1.018 1.278
|
|
||||||
0 0.712 0.092 -0.466 0.688 1.236 0.921 -1.217 -1.022 2.173 2.236 -1.167 0.868 2.215 0.851 -1.892 -0.753 0.000 0.475 -1.216 -0.383 0.000 0.668 0.758 0.988 1.180 2.093 1.157 0.934
|
|
||||||
0 0.419 0.471 0.974 2.805 0.235 1.473 -0.198 1.255 1.087 0.931 1.083 -0.712 0.000 1.569 1.358 -1.179 2.548 2.506 0.199 -0.842 0.000 0.929 0.991 0.992 1.732 2.367 1.549 1.430
|
|
||||||
1 0.667 1.003 1.504 0.368 1.061 0.885 -0.318 -0.353 0.000 1.438 -1.939 0.710 0.000 1.851 0.277 -1.460 2.548 1.403 0.517 -0.157 0.000 0.883 1.019 1.000 0.790 0.859 0.938 0.841
|
|
||||||
1 1.877 -0.492 0.372 0.441 0.955 1.034 -1.220 -0.846 1.087 0.952 -0.320 1.125 0.000 0.542 0.308 -1.261 2.548 1.018 -1.415 -1.547 0.000 1.280 0.932 0.991 1.273 0.878 0.921 0.906
|
|
||||||
0 1.052 0.901 1.176 1.280 1.517 0.562 -1.150 -0.079 2.173 1.228 -0.308 -0.354 0.000 0.790 -1.492 -0.963 0.000 0.942 -0.672 -1.588 3.102 1.116 0.902 0.988 1.993 0.765 1.375 1.325
|
|
||||||
1 0.518 -0.254 1.642 0.865 0.725 0.980 0.734 0.023 0.000 1.448 0.780 -1.736 2.215 0.955 0.513 -0.519 0.000 0.365 -0.444 -0.243 3.102 0.833 0.555 0.984 0.827 0.795 0.890 0.786
|
|
||||||
0 0.870 0.815 -0.506 0.663 -0.518 0.935 0.289 -1.675 2.173 1.188 0.005 0.635 0.000 0.580 0.066 -1.455 2.548 0.580 -0.634 -0.199 0.000 0.852 0.788 0.979 1.283 0.208 0.856 0.950
|
|
||||||
0 0.628 1.382 0.135 0.683 0.571 1.097 0.564 -0.950 2.173 0.617 -0.326 0.371 0.000 1.093 0.918 1.667 2.548 0.460 1.221 0.708 0.000 0.743 0.861 0.975 1.067 1.007 0.843 0.762
|
|
||||||
0 4.357 0.816 -1.609 1.845 -1.288 3.292 0.726 0.324 2.173 1.528 0.583 -0.801 2.215 0.605 0.572 1.406 0.000 0.794 -0.791 0.122 0.000 0.967 1.132 1.124 3.602 2.811 2.460 1.861
|
|
||||||
0 0.677 -1.265 1.559 0.866 -0.618 0.823 0.260 0.185 0.000 1.133 0.337 1.589 2.215 0.563 -0.830 0.510 0.000 0.777 0.117 -0.941 3.102 0.839 0.763 0.986 1.182 0.649 0.796 0.851
|
|
||||||
0 2.466 -1.838 -1.648 1.717 1.533 1.676 -1.553 -0.109 2.173 0.670 -0.666 0.284 0.000 0.334 -2.480 0.316 0.000 0.366 -0.804 -1.298 3.102 0.875 0.894 0.997 0.548 0.770 1.302 1.079
|
|
||||||
1 1.403 0.129 -1.307 0.688 0.306 0.579 0.753 0.814 1.087 0.474 0.694 -1.400 0.000 0.520 1.995 0.185 0.000 0.929 -0.504 1.270 3.102 0.972 0.998 1.353 0.948 0.650 0.688 0.724
|
|
||||||
1 0.351 1.188 -0.360 0.254 -0.346 1.129 0.545 1.691 0.000 0.652 -0.039 -0.258 2.215 1.089 0.655 0.472 2.548 0.554 -0.493 1.366 0.000 0.808 1.045 0.992 0.570 0.649 0.809 0.744
|
|
||||||
0 1.875 -0.013 -0.128 0.236 1.163 0.902 0.426 0.590 2.173 1.251 -1.210 -0.616 0.000 1.035 1.534 0.912 0.000 1.944 1.789 -1.691 0.000 0.974 1.113 0.990 0.925 1.120 0.956 0.912
|
|
||||||
0 0.298 0.750 -0.507 1.555 1.463 0.804 1.200 -0.665 0.000 0.439 -0.829 -0.252 1.107 0.770 -1.090 0.947 2.548 1.165 -0.166 -0.763 0.000 1.140 0.997 0.988 1.330 0.555 1.005 1.012
|
|
||||||
0 0.647 0.342 0.245 4.340 -0.157 2.229 0.068 1.170 2.173 2.133 -0.201 -1.441 0.000 1.467 0.697 -0.532 1.274 1.457 0.583 -1.640 0.000 0.875 1.417 0.976 2.512 2.390 1.794 1.665
|
|
||||||
1 1.731 -0.803 -1.013 1.492 -0.020 1.646 -0.541 1.121 2.173 0.459 -1.251 -1.495 2.215 0.605 -1.711 -0.232 0.000 0.658 0.634 -0.068 0.000 1.214 0.886 1.738 1.833 1.024 1.192 1.034
|
|
||||||
0 0.515 1.416 -1.089 1.697 1.426 1.414 0.941 0.027 0.000 1.480 0.133 -1.595 2.215 1.110 0.752 0.760 2.548 1.062 0.697 -0.492 0.000 0.851 0.955 0.994 1.105 1.255 1.175 1.095
|
|
||||||
0 1.261 0.858 1.465 0.757 0.305 2.310 0.679 1.080 2.173 1.544 2.518 -0.464 0.000 2.326 0.270 -0.841 0.000 2.163 0.839 -0.500 3.102 0.715 0.825 1.170 0.980 2.371 1.527 1.221
|
|
||||||
1 1.445 1.509 1.471 0.414 -1.285 0.767 0.864 -0.677 2.173 0.524 1.388 0.171 0.000 0.826 0.190 0.121 2.548 0.572 1.691 -1.603 0.000 0.870 0.935 0.994 0.968 0.735 0.783 0.777
|
|
||||||
1 0.919 -0.264 -1.245 0.681 -1.722 1.022 1.010 0.097 2.173 0.685 0.403 -1.351 0.000 1.357 -0.429 1.262 1.274 0.687 1.021 -0.563 0.000 0.953 0.796 0.991 0.873 1.749 1.056 0.917
|
|
||||||
1 0.293 -2.258 -1.427 1.191 1.202 0.394 -2.030 1.438 0.000 0.723 0.596 -0.024 2.215 0.525 -1.678 -0.290 0.000 0.788 -0.824 -1.029 3.102 0.821 0.626 0.976 1.080 0.810 0.842 0.771
|
|
||||||
0 3.286 0.386 1.688 1.619 -1.620 1.392 -0.009 0.280 0.000 1.179 -0.776 -0.110 2.215 1.256 0.248 -1.114 2.548 0.777 0.825 -0.156 0.000 1.026 1.065 0.964 0.909 1.249 1.384 1.395
|
|
||||||
1 1.075 0.603 0.561 0.656 -0.685 0.985 0.175 0.979 2.173 1.154 0.584 -0.886 0.000 1.084 -0.354 -1.004 2.548 0.865 1.224 1.269 0.000 1.346 1.073 1.048 0.873 1.310 1.003 0.865
|
|
||||||
1 1.098 -0.091 1.466 1.558 0.915 0.649 1.314 -1.182 2.173 0.791 0.073 0.351 0.000 0.517 0.940 1.195 0.000 1.150 1.187 -0.692 3.102 0.866 0.822 0.980 1.311 0.394 1.119 0.890
|
|
||||||
1 0.481 -1.042 0.148 1.135 -1.249 1.202 -0.344 0.308 1.087 0.779 -1.431 1.581 0.000 0.860 -0.860 -1.125 0.000 0.785 0.303 1.199 3.102 0.878 0.853 0.988 1.072 0.827 0.936 0.815
|
|
||||||
0 1.348 0.497 0.318 0.806 0.976 1.393 -0.152 0.632 2.173 2.130 0.515 -1.054 0.000 0.908 0.062 -0.780 0.000 1.185 0.687 1.668 1.551 0.720 0.898 0.985 0.683 1.292 1.320 1.131
|
|
||||||
0 2.677 -0.420 -1.685 1.828 1.433 2.040 -0.718 -0.039 0.000 0.400 -0.873 0.472 0.000 0.444 0.340 -0.830 2.548 0.431 0.768 -1.417 3.102 0.869 0.917 0.996 0.707 0.193 0.728 1.154
|
|
||||||
1 1.300 0.586 -0.122 1.306 0.609 0.727 -0.556 -1.652 2.173 0.636 0.720 1.393 2.215 0.328 1.280 -0.390 0.000 0.386 0.752 -0.905 0.000 0.202 0.751 1.106 0.864 0.799 0.928 0.717
|
|
||||||
0 0.637 -0.176 1.737 1.322 -0.414 0.702 -0.964 -0.680 0.000 1.054 -0.461 0.889 2.215 0.861 -0.267 0.225 0.000 1.910 -1.888 1.027 0.000 0.919 0.899 1.186 0.993 1.109 0.862 0.775
|
|
||||||
1 0.723 -0.104 1.572 0.428 -0.840 0.655 0.544 1.401 2.173 1.522 -0.154 -0.452 2.215 0.996 0.190 0.273 0.000 1.906 -0.176 0.966 0.000 0.945 0.894 0.990 0.981 1.555 0.988 0.893
|
|
||||||
0 2.016 -0.570 1.612 0.798 0.441 0.334 0.191 -0.909 0.000 0.939 0.146 0.021 2.215 0.553 -0.444 1.156 2.548 0.781 -1.545 -0.520 0.000 0.922 0.956 1.528 0.722 0.699 0.778 0.901
|
|
||||||
0 1.352 -0.707 1.284 0.665 0.580 0.694 -1.040 -0.899 2.173 0.692 -2.048 0.029 0.000 0.545 -2.042 1.259 0.000 0.661 -0.808 -1.251 3.102 0.845 0.991 0.979 0.662 0.225 0.685 0.769
|
|
||||||
1 1.057 -1.561 -0.411 0.952 -0.681 1.236 -1.107 1.045 2.173 1.288 -2.521 -0.521 0.000 1.361 -1.239 1.546 0.000 0.373 -1.540 0.028 0.000 0.794 0.782 0.987 0.889 0.832 0.972 0.828
|
|
||||||
0 1.118 -0.017 -1.227 1.077 1.256 0.714 0.624 -0.811 0.000 0.800 0.704 0.387 1.107 0.604 0.234 0.986 0.000 1.306 -0.456 0.094 3.102 0.828 0.984 1.195 0.987 0.672 0.774 0.748
|
|
||||||
1 0.602 2.201 0.212 0.119 0.182 0.474 2.130 1.270 0.000 0.370 2.088 -0.573 0.000 0.780 -0.725 -1.033 0.000 1.642 0.598 0.303 3.102 0.886 0.988 0.985 0.644 0.756 0.651 0.599
|
|
||||||
0 1.677 -0.844 1.581 0.585 0.887 1.012 -2.315 0.752 0.000 1.077 0.748 -0.195 0.000 0.718 0.832 -1.337 1.274 1.181 -0.557 -1.006 3.102 1.018 1.247 0.988 0.908 0.651 1.311 1.120
|
|
||||||
1 1.695 0.259 1.224 1.344 1.067 0.718 -1.752 -0.215 0.000 0.473 0.991 -0.993 0.000 0.891 1.285 -1.500 2.548 0.908 -0.131 0.288 0.000 0.945 0.824 0.979 1.009 0.951 0.934 0.833
|
|
||||||
0 0.793 0.628 0.432 1.707 0.302 0.919 1.045 -0.784 0.000 1.472 0.175 -1.284 2.215 1.569 0.155 0.971 2.548 0.435 0.735 1.625 0.000 0.801 0.907 0.992 0.831 1.446 1.082 1.051
|
|
||||||
1 0.537 -0.664 -0.244 1.104 1.272 1.154 0.394 1.633 0.000 1.527 0.963 0.559 2.215 1.744 0.650 -0.912 0.000 1.097 0.730 -0.368 3.102 1.953 1.319 1.045 1.309 0.869 1.196 1.126
|
|
||||||
1 0.585 -1.469 1.005 0.749 -1.060 1.224 -0.717 -0.323 2.173 1.012 -0.201 1.268 0.000 0.359 -0.567 0.476 0.000 1.117 -1.124 1.557 3.102 0.636 1.281 0.986 0.616 1.289 0.890 0.881
|
|
||||||
1 0.354 -1.517 0.667 2.534 -1.298 1.020 -0.375 1.254 0.000 1.119 -0.060 -1.538 2.215 1.059 -0.395 -0.140 0.000 2.609 0.199 -0.778 1.551 0.957 0.975 1.286 1.666 1.003 1.224 1.135
|
|
||||||
1 0.691 -1.619 -1.380 0.361 1.727 1.493 -1.093 -0.289 0.000 1.447 -0.640 1.341 0.000 1.453 -0.617 -1.456 1.274 1.061 -1.481 -0.091 0.000 0.744 0.649 0.987 0.596 0.727 0.856 0.797
|
|
||||||
0 1.336 1.293 -1.359 0.357 0.067 1.110 -0.058 -0.515 0.000 0.976 1.498 1.207 0.000 1.133 0.437 1.053 2.548 0.543 1.374 0.171 0.000 0.764 0.761 0.984 0.827 0.553 0.607 0.612
|
|
||||||
0 0.417 -1.111 1.661 2.209 -0.683 1.931 -0.642 0.959 1.087 1.514 -2.032 -0.686 0.000 1.521 -0.539 1.344 0.000 0.978 -0.866 0.363 1.551 2.813 1.850 1.140 1.854 0.799 1.600 1.556
|
|
||||||
0 1.058 0.390 -0.591 0.134 1.149 0.346 -1.550 0.186 0.000 1.108 -0.999 0.843 1.107 1.124 0.415 -1.514 0.000 1.067 -0.426 -1.000 3.102 1.744 1.050 0.985 1.006 1.010 0.883 0.789
|
|
||||||
1 1.655 0.253 1.216 0.270 1.703 0.500 -0.006 -1.418 2.173 0.690 -0.350 0.170 2.215 1.045 -0.924 -0.774 0.000 0.996 -0.745 -0.123 0.000 0.839 0.820 0.993 0.921 0.869 0.725 0.708
|
|
||||||
0 1.603 -0.850 0.564 0.829 0.093 1.270 -1.113 -1.155 2.173 0.853 -1.021 1.248 2.215 0.617 -1.270 1.733 0.000 0.935 -0.092 0.136 0.000 1.011 1.074 0.977 0.823 1.269 1.054 0.878
|
|
||||||
0 1.568 -0.792 1.005 0.545 0.896 0.895 -1.698 -0.988 0.000 0.608 -1.634 1.705 0.000 0.826 0.208 0.618 1.274 2.063 -1.743 -0.520 0.000 0.939 0.986 0.990 0.600 0.435 1.033 1.087
|
|
||||||
0 0.489 -1.335 -1.102 1.738 1.028 0.628 -0.992 -0.627 0.000 0.652 -0.064 -0.215 0.000 1.072 0.173 -1.251 2.548 1.042 0.057 0.841 3.102 0.823 0.895 1.200 1.164 0.770 0.837 0.846
|
|
||||||
1 1.876 0.870 1.234 0.556 -1.262 1.764 0.855 -0.467 2.173 1.079 1.351 0.852 0.000 0.773 0.383 0.874 0.000 1.292 0.829 -1.228 3.102 0.707 0.969 1.102 1.601 1.017 1.112 1.028
|
|
||||||
0 1.033 0.407 -0.374 0.705 -1.254 0.690 -0.231 1.502 2.173 0.433 -2.009 -0.057 0.000 0.861 1.151 0.334 0.000 0.960 -0.839 1.299 3.102 2.411 1.480 0.982 0.995 0.377 1.012 0.994
|
|
||||||
0 1.092 0.653 -0.801 0.463 0.426 0.529 -1.055 0.040 0.000 0.663 0.999 1.255 1.107 0.749 -1.106 1.185 2.548 0.841 -0.745 -1.029 0.000 0.841 0.743 0.988 0.750 1.028 0.831 0.868
|
|
||||||
1 0.799 -0.285 -0.011 0.531 1.392 1.063 0.854 0.494 2.173 1.187 -1.065 -0.851 0.000 0.429 -0.296 1.072 0.000 0.942 -1.985 1.172 0.000 0.873 0.693 0.992 0.819 0.689 1.131 0.913
|
|
||||||
0 0.503 1.973 -0.377 1.515 -1.514 0.708 1.081 -0.313 2.173 1.110 -0.417 0.839 0.000 0.712 -1.153 1.165 0.000 0.675 -0.303 -0.930 1.551 0.709 0.761 1.032 0.986 0.698 0.963 1.291
|
|
||||||
0 0.690 -0.574 -1.608 1.182 1.118 0.557 -2.243 0.144 0.000 0.969 0.216 -1.383 1.107 1.054 0.888 -0.709 2.548 0.566 1.663 -0.550 0.000 0.752 1.528 0.987 1.408 0.740 1.290 1.123
|
|
||||||
1 0.890 1.501 0.786 0.779 -0.615 1.126 0.716 1.541 2.173 0.887 0.728 -0.673 2.215 1.216 0.332 -0.020 0.000 0.965 1.828 0.101 0.000 0.827 0.715 1.099 1.088 1.339 0.924 0.878
|
|
||||||
0 0.566 0.883 0.655 1.600 0.034 1.155 2.028 -1.499 0.000 0.723 -0.871 0.763 0.000 1.286 -0.696 -0.676 2.548 1.134 -0.113 1.207 3.102 4.366 2.493 0.984 0.960 0.962 1.843 1.511
|
|
||||||
0 1.146 1.086 -0.911 0.838 1.298 0.821 0.127 -0.145 0.000 1.352 0.474 -1.580 2.215 1.619 -0.081 0.675 2.548 1.382 -0.748 0.127 0.000 0.958 0.976 1.239 0.876 1.481 1.116 1.076
|
|
||||||
0 1.739 -0.326 -1.661 0.420 -1.705 1.193 -0.031 -1.212 2.173 1.783 -0.442 0.522 0.000 1.064 -0.692 0.027 0.000 1.314 0.359 -0.037 3.102 0.968 0.897 0.986 0.907 1.196 1.175 1.112
|
|
||||||
1 0.669 0.194 -0.703 0.657 -0.260 0.899 -2.511 0.311 0.000 1.482 0.773 0.974 2.215 3.459 0.037 -1.299 1.274 2.113 0.067 1.516 0.000 0.740 0.871 0.979 1.361 2.330 1.322 1.046
|
|
||||||
1 1.355 -1.033 -1.173 0.552 -0.048 0.899 -0.482 -1.287 2.173 1.422 -1.227 0.390 1.107 1.937 -0.028 0.914 0.000 0.849 -0.230 -1.734 0.000 0.986 1.224 1.017 1.051 1.788 1.150 1.009
|
|
||||||
1 0.511 -0.202 1.029 0.780 1.154 0.816 0.532 -0.731 0.000 0.757 0.517 0.749 2.215 1.302 0.289 -1.188 0.000 0.584 1.211 -0.350 0.000 0.876 0.943 0.995 0.963 0.256 0.808 0.891
|
|
||||||
1 1.109 0.572 1.484 0.753 1.543 1.711 -0.145 -0.746 1.087 1.759 0.631 0.845 2.215 0.945 0.542 0.003 0.000 0.378 -1.150 -0.044 0.000 0.764 1.042 0.992 1.045 2.736 1.441 1.140
|
|
||||||
0 0.712 -0.025 0.553 0.928 -0.711 1.304 0.045 -0.300 0.000 0.477 0.720 0.969 0.000 1.727 -0.474 1.328 1.274 1.282 2.222 1.684 0.000 0.819 0.765 1.023 0.961 0.657 0.799 0.744
|
|
||||||
1 1.131 -0.302 1.079 0.901 0.236 0.904 -0.249 1.694 2.173 1.507 -0.702 -1.128 0.000 0.774 0.565 0.284 2.548 1.802 1.446 -0.192 0.000 3.720 2.108 0.986 0.930 1.101 1.484 1.238
|
|
||||||
0 1.392 1.253 0.118 0.864 -1.358 0.922 -0.447 -1.243 1.087 1.969 1.031 0.774 2.215 1.333 -0.359 -0.681 0.000 1.099 -0.257 1.473 0.000 1.246 0.909 1.475 1.234 2.531 1.449 1.306
|
|
||||||
0 1.374 2.291 -0.479 1.339 -0.243 0.687 2.345 1.310 0.000 0.467 1.081 0.772 0.000 0.656 1.155 -1.636 2.548 0.592 0.536 -1.269 3.102 0.981 0.821 1.010 0.877 0.217 0.638 0.758
|
|
||||||
1 0.401 -1.516 0.909 2.738 0.519 0.887 0.566 -1.202 0.000 0.909 -0.176 1.682 0.000 2.149 -0.878 -0.514 2.548 0.929 -0.563 -1.555 3.102 1.228 0.803 0.980 1.382 0.884 1.025 1.172
|
|
||||||
1 0.430 -1.589 1.417 2.158 1.226 1.180 -0.829 -0.781 2.173 0.798 1.400 -0.111 0.000 0.939 -0.878 1.076 2.548 0.576 1.335 -0.826 0.000 0.861 0.970 0.982 1.489 1.308 1.015 0.992
|
|
||||||
1 1.943 -0.391 -0.840 0.621 -1.613 2.026 1.734 1.025 0.000 0.930 0.573 -0.912 0.000 1.326 0.847 -0.220 1.274 1.181 0.079 0.709 3.102 1.164 1.007 0.987 1.094 0.821 0.857 0.786
|
|
||||||
1 0.499 0.436 0.887 0.859 1.509 0.733 -0.559 1.111 1.087 1.011 -0.796 0.279 2.215 1.472 -0.510 -0.982 0.000 1.952 0.379 -0.733 0.000 1.076 1.358 0.991 0.589 0.879 1.068 0.922
|
|
||||||
0 0.998 -0.407 -1.711 0.139 0.652 0.810 -0.331 -0.721 0.000 0.471 -0.533 0.442 0.000 0.531 -1.405 0.120 2.548 0.707 0.098 -1.176 1.551 1.145 0.809 0.988 0.529 0.612 0.562 0.609
|
|
||||||
1 1.482 0.872 0.638 1.288 0.362 0.856 0.900 -0.511 1.087 1.072 1.061 -1.432 2.215 1.770 -2.292 -1.547 0.000 1.131 1.374 0.783 0.000 6.316 4.381 1.002 1.317 1.048 2.903 2.351
|
|
||||||
1 2.084 -0.422 1.289 1.125 0.735 1.104 -0.518 -0.326 2.173 0.413 -0.719 -0.699 0.000 0.857 0.108 -1.631 0.000 0.527 0.641 -1.362 3.102 0.791 0.952 1.016 0.776 0.856 0.987 0.836
|
|
||||||
0 0.464 0.674 0.025 0.430 -1.703 0.982 -1.311 -0.808 2.173 1.875 1.060 0.821 2.215 0.954 -0.480 -1.677 0.000 0.567 0.702 -0.939 0.000 0.781 1.076 0.989 1.256 3.632 1.652 1.252
|
|
||||||
1 0.457 -1.944 -1.010 1.409 0.931 1.098 -0.742 -0.415 0.000 1.537 -0.834 0.945 2.215 1.752 -0.287 -1.269 2.548 0.692 -1.537 -0.223 0.000 0.801 1.192 1.094 1.006 1.659 1.175 1.122
|
|
||||||
0 3.260 -0.943 1.737 0.920 1.309 0.946 -0.139 -0.271 2.173 0.994 -0.952 -0.311 0.000 0.563 -0.136 -0.881 0.000 1.236 -0.507 0.906 1.551 0.747 0.869 0.985 1.769 1.034 1.179 1.042
|
|
||||||
0 0.615 -0.778 0.246 1.861 1.619 0.560 -0.943 -0.204 2.173 0.550 -0.759 -1.342 2.215 0.578 0.076 -0.973 0.000 0.939 0.035 0.680 0.000 0.810 0.747 1.401 0.772 0.702 0.719 0.662
|
|
||||||
1 2.370 -0.064 -0.237 1.737 0.154 2.319 -1.838 -1.673 0.000 1.053 -1.305 -0.075 0.000 0.925 0.149 0.318 1.274 0.851 -0.922 0.981 3.102 0.919 0.940 0.989 0.612 0.598 1.219 1.626
|
|
||||||
1 1.486 0.311 -1.262 1.354 -0.847 0.886 -0.158 1.213 2.173 1.160 -0.218 0.239 0.000 1.166 0.494 0.278 2.548 0.575 1.454 -1.701 0.000 0.429 1.129 0.983 1.111 1.049 1.006 0.920
|
|
||||||
1 1.294 1.587 -0.864 0.487 -0.312 0.828 1.051 -0.031 1.087 2.443 1.216 1.609 2.215 1.167 0.813 0.921 0.000 1.751 -0.415 0.119 0.000 1.015 1.091 0.974 1.357 2.093 1.178 1.059
|
|
||||||
1 0.984 0.465 -1.661 0.379 -0.554 0.977 0.237 0.365 0.000 0.510 0.143 1.101 0.000 1.099 -0.662 -1.593 2.548 1.104 -0.197 -0.648 3.102 0.925 0.922 0.986 0.642 0.667 0.806 0.722
|
|
||||||
1 0.930 -0.009 0.047 0.667 1.367 1.065 -0.231 0.815 0.000 1.199 -1.114 -0.877 2.215 0.940 0.824 -1.583 0.000 1.052 -0.407 -0.076 1.551 1.843 1.257 1.013 1.047 0.751 1.158 0.941
|
|
||||||
0 0.767 -0.011 -0.637 0.341 -1.437 1.438 -0.425 -0.450 2.173 1.073 -0.718 1.341 2.215 0.633 -1.394 0.486 0.000 0.603 -1.945 -1.626 0.000 0.703 0.790 0.984 1.111 1.848 1.129 1.072
|
|
||||||
1 1.779 0.017 0.432 0.402 1.022 0.959 1.480 1.595 2.173 1.252 1.365 0.006 0.000 1.188 -0.174 -1.107 0.000 1.181 0.518 -0.258 0.000 1.057 0.910 0.991 1.616 0.779 1.158 1.053
|
|
||||||
0 0.881 0.630 1.029 1.990 0.508 1.102 0.742 -1.298 2.173 1.565 1.085 0.686 2.215 2.691 1.391 -0.904 0.000 0.499 1.388 -1.199 0.000 0.347 0.861 0.997 0.881 1.920 1.233 1.310
|
|
||||||
0 1.754 -0.266 0.389 0.347 -0.030 0.462 -1.408 -0.957 2.173 0.515 -2.341 -1.700 0.000 0.588 -0.797 1.355 2.548 0.608 0.329 -1.389 0.000 1.406 0.909 0.988 0.760 0.593 0.768 0.847
|
|
||||||
0 1.087 0.311 -1.447 0.173 0.567 0.854 0.362 0.584 0.000 1.416 -0.716 -1.211 2.215 0.648 -0.358 -0.692 1.274 0.867 -0.513 0.206 0.000 0.803 0.813 0.984 1.110 0.491 0.921 0.873
|
|
||||||
0 0.279 1.114 -1.190 3.004 -0.738 1.233 0.896 1.092 2.173 0.454 -0.374 0.117 2.215 0.357 0.119 1.270 0.000 0.458 1.343 0.316 0.000 0.495 0.540 0.988 1.715 1.139 1.618 1.183
|
|
||||||
1 1.773 -0.694 -1.518 2.306 -1.200 3.104 0.749 0.362 0.000 1.871 0.230 -1.686 2.215 0.805 -0.179 -0.871 1.274 0.910 0.607 -0.246 0.000 1.338 1.598 0.984 1.050 0.919 1.678 1.807
|
|
||||||
0 0.553 0.683 0.827 0.973 -0.706 1.488 0.149 1.140 2.173 1.788 0.447 -0.478 0.000 0.596 1.043 1.607 0.000 0.373 -0.868 -1.308 1.551 1.607 1.026 0.998 1.134 0.808 1.142 0.936
|
|
||||||
1 0.397 1.101 -1.139 1.688 0.146 0.972 0.541 1.518 0.000 1.549 -0.873 -1.012 0.000 2.282 -0.151 0.314 2.548 1.174 0.033 -1.368 0.000 0.937 0.776 1.039 1.143 0.959 0.986 1.013
|
|
||||||
1 0.840 1.906 -0.959 0.869 0.576 0.642 0.554 -1.351 0.000 0.756 0.923 -0.823 2.215 1.251 1.130 0.545 2.548 1.513 0.410 1.073 0.000 1.231 0.985 1.163 0.812 0.987 0.816 0.822
|
|
||||||
1 0.477 1.665 0.814 0.763 -0.382 0.828 -0.008 0.280 2.173 1.213 -0.001 1.560 0.000 1.136 0.311 -1.289 0.000 0.797 1.091 -0.616 3.102 1.026 0.964 0.992 0.772 0.869 0.916 0.803
|
|
||||||
0 2.655 0.020 0.273 1.464 0.482 1.709 -0.107 -1.456 2.173 0.825 0.141 -0.386 0.000 1.342 -0.592 1.635 1.274 0.859 -0.175 -0.874 0.000 0.829 0.946 1.003 2.179 0.836 1.505 1.176
|
|
||||||
0 0.771 -1.992 -0.720 0.732 -1.464 0.869 -1.290 0.388 2.173 0.926 -1.072 -1.489 2.215 0.640 -1.232 0.840 0.000 0.528 -2.440 -0.446 0.000 0.811 0.868 0.993 0.995 1.317 0.809 0.714
|
|
||||||
0 1.357 1.302 0.076 0.283 -1.060 0.783 1.559 -0.994 0.000 0.947 1.212 1.617 0.000 1.127 0.311 0.442 2.548 0.582 -0.052 1.186 1.551 1.330 0.995 0.985 0.846 0.404 0.858 0.815
|
|
||||||
0 0.442 -0.381 -0.424 1.244 0.591 0.731 0.605 -0.713 2.173 0.629 2.762 1.040 0.000 0.476 2.693 -0.617 0.000 0.399 0.442 1.486 3.102 0.839 0.755 0.988 0.869 0.524 0.877 0.918
|
|
||||||
0 0.884 0.422 0.055 0.818 0.624 0.950 -0.763 1.624 0.000 0.818 -0.609 -1.166 0.000 1.057 -0.528 1.070 2.548 1.691 -0.124 -0.335 3.102 1.104 0.933 0.985 0.913 1.000 0.863 1.056
|
|
||||||
0 1.276 0.156 1.714 1.053 -1.189 0.672 -0.464 -0.030 2.173 0.469 -2.483 0.442 0.000 0.564 2.580 -0.253 0.000 0.444 -0.628 1.080 1.551 5.832 2.983 0.985 1.162 0.494 1.809 1.513
|
|
||||||
0 1.106 -0.556 0.406 0.573 -1.400 0.769 -0.518 1.457 2.173 0.743 -0.352 -0.010 0.000 1.469 -0.550 -0.930 2.548 0.540 1.236 -0.571 0.000 0.962 0.970 1.101 0.805 1.107 0.873 0.773
|
|
||||||
0 0.539 -0.964 -0.464 1.371 -1.606 0.667 -0.160 0.655 0.000 0.952 0.352 -0.740 2.215 0.952 0.007 1.123 0.000 1.061 -0.505 1.389 3.102 1.063 0.991 1.019 0.633 0.967 0.732 0.799
|
|
||||||
1 0.533 -0.989 -1.608 0.462 -1.723 1.204 -0.598 -0.098 2.173 1.343 -0.460 1.632 2.215 0.577 0.221 -0.492 0.000 0.628 -0.073 0.472 0.000 0.518 0.880 0.988 1.179 1.874 1.041 0.813
|
|
||||||
1 1.024 1.075 -0.795 0.286 -1.436 1.365 0.857 -0.309 2.173 0.804 1.532 1.435 0.000 1.511 0.722 1.494 0.000 1.778 0.903 0.753 1.551 0.686 0.810 0.999 0.900 1.360 1.133 0.978
|
|
||||||
1 2.085 -0.269 -1.423 0.789 1.298 0.281 1.652 0.187 0.000 0.658 -0.760 -0.042 2.215 0.663 0.024 0.120 0.000 0.552 -0.299 -0.428 3.102 0.713 0.811 1.130 0.705 0.218 0.675 0.743
|
|
||||||
1 0.980 -0.443 0.813 0.785 -1.253 0.719 0.448 -1.458 0.000 1.087 0.595 0.635 1.107 1.428 0.029 -0.995 0.000 1.083 1.562 -0.092 0.000 0.834 0.891 1.165 0.967 0.661 0.880 0.817
|
|
||||||
1 0.903 -0.733 -0.980 0.634 -0.639 0.780 0.266 -0.287 2.173 1.264 -0.936 1.004 0.000 1.002 -0.056 -1.344 2.548 1.183 -0.098 1.169 0.000 0.733 1.002 0.985 0.711 0.916 0.966 0.875
|
|
||||||
0 0.734 -0.304 -1.175 2.851 1.674 0.904 -0.634 0.412 2.173 1.363 -1.050 -0.282 0.000 1.476 -1.603 0.103 0.000 2.231 -0.718 1.708 3.102 0.813 0.896 1.088 0.686 1.392 1.033 1.078
|
|
||||||
1 1.680 0.591 -0.243 0.111 -0.478 0.326 -0.079 -1.555 2.173 0.711 0.714 0.922 2.215 0.355 0.858 1.682 0.000 0.727 1.620 1.360 0.000 0.334 0.526 1.001 0.862 0.633 0.660 0.619
|
|
||||||
1 1.163 0.225 -0.202 0.501 -0.979 1.609 -0.938 1.424 0.000 1.224 -0.118 -1.274 0.000 2.034 1.241 -0.254 0.000 1.765 0.536 0.237 3.102 0.894 0.838 0.988 0.693 0.579 0.762 0.726
|
|
||||||
0 1.223 1.232 1.471 0.489 1.728 0.703 -0.111 0.411 0.000 1.367 1.014 -1.294 1.107 1.524 -0.414 -0.164 2.548 1.292 0.833 0.316 0.000 0.861 0.752 0.994 0.836 1.814 1.089 0.950
|
|
||||||
0 0.816 1.637 -1.557 1.036 -0.342 0.913 1.333 0.949 2.173 0.812 0.756 -0.628 2.215 1.333 0.470 1.495 0.000 1.204 -2.222 -1.675 0.000 1.013 0.924 1.133 0.758 1.304 0.855 0.860
|
|
||||||
0 0.851 -0.564 -0.691 0.692 1.345 1.219 1.014 0.318 0.000 1.422 -0.262 -1.635 2.215 0.531 1.802 0.008 0.000 0.508 0.515 -1.267 3.102 0.821 0.787 1.026 0.783 0.432 1.149 1.034
|
|
||||||
0 0.800 -0.599 0.204 0.552 -0.484 0.974 0.413 0.961 2.173 1.269 -0.984 -1.039 2.215 0.380 -1.213 1.371 0.000 0.551 0.332 -0.659 0.000 0.694 0.852 0.984 1.057 2.037 1.096 0.846
|
|
||||||
0 0.744 -0.071 -0.255 0.638 0.512 1.125 0.407 0.844 2.173 0.860 -0.481 -0.677 0.000 1.102 0.181 -1.194 0.000 1.011 -1.081 -1.713 3.102 0.854 0.862 0.982 1.111 1.372 1.042 0.920
|
|
||||||
1 0.400 1.049 -0.625 0.880 -0.407 1.040 2.150 -1.359 0.000 0.747 -0.144 0.847 2.215 0.560 -1.829 0.698 0.000 1.663 -0.668 0.267 0.000 0.845 0.964 0.996 0.820 0.789 0.668 0.668
|
|
||||||
0 1.659 -0.705 -1.057 1.803 -1.436 1.008 0.693 0.005 0.000 0.895 -0.007 0.681 1.107 1.085 0.125 1.476 2.548 1.214 1.068 0.486 0.000 0.867 0.919 0.986 1.069 0.692 1.026 1.313
|
|
||||||
0 0.829 -0.153 0.861 0.615 -0.548 0.589 1.077 -0.041 2.173 1.056 0.763 -1.737 0.000 0.639 0.970 0.725 0.000 0.955 1.227 -0.799 3.102 1.020 1.024 0.985 0.750 0.525 0.685 0.671
|
|
||||||
1 0.920 -0.806 -0.840 1.048 0.278 0.973 -0.077 -1.364 2.173 1.029 0.309 0.133 0.000 1.444 1.484 1.618 1.274 1.419 -0.482 0.417 0.000 0.831 1.430 1.151 1.829 1.560 1.343 1.224
|
|
||||||
1 0.686 0.249 -0.905 0.343 -1.731 0.724 -2.823 -0.901 0.000 0.982 0.303 1.312 1.107 1.016 0.245 0.610 0.000 1.303 -0.557 -0.360 3.102 1.384 1.030 0.984 0.862 1.144 0.866 0.779
|
|
||||||
0 1.603 0.444 0.508 0.586 0.401 0.610 0.467 -1.735 2.173 0.914 0.626 -1.019 0.000 0.812 0.422 -0.408 2.548 0.902 1.679 1.490 0.000 1.265 0.929 0.990 1.004 0.816 0.753 0.851
|
|
||||||
1 0.623 0.780 -0.203 0.056 0.015 0.899 0.793 1.326 1.087 0.803 1.478 -1.499 2.215 1.561 1.492 -0.120 0.000 0.904 0.795 0.137 0.000 0.548 1.009 0.850 0.924 0.838 0.914 0.860
|
|
||||||
0 1.654 -2.032 -1.160 0.859 -1.583 0.689 -1.965 0.891 0.000 0.646 -1.014 -0.288 2.215 0.630 -0.815 0.402 0.000 0.638 0.316 0.655 3.102 0.845 0.879 0.993 1.067 0.625 1.041 0.958
|
|
||||||
1 0.828 -1.269 -1.203 0.744 -0.213 0.626 -1.017 -0.404 0.000 1.281 -0.931 1.733 2.215 0.699 -0.351 1.287 0.000 1.251 -1.171 0.197 0.000 0.976 1.186 0.987 0.646 0.655 0.733 0.671
|
|
||||||
1 0.677 0.111 1.090 1.580 1.591 1.560 0.654 -0.341 2.173 0.794 -0.266 0.702 0.000 0.823 0.651 -1.239 2.548 0.730 1.467 -1.530 0.000 1.492 1.023 0.983 1.909 1.022 1.265 1.127
|
|
||||||
1 0.736 0.882 -1.060 0.589 0.168 1.663 0.781 1.022 2.173 2.025 1.648 -1.292 0.000 1.240 0.924 -0.421 1.274 1.354 0.065 0.501 0.000 0.316 0.925 0.988 0.664 1.736 0.992 0.807
|
|
||||||
1 1.040 -0.822 1.638 0.974 -0.674 0.393 0.830 0.011 2.173 0.770 -0.140 -0.402 0.000 0.294 -0.133 0.030 0.000 1.220 0.807 0.638 0.000 0.826 1.063 1.216 1.026 0.705 0.934 0.823
|
|
||||||
1 0.711 0.602 0.048 1.145 0.966 0.934 0.263 -1.589 2.173 0.971 -0.496 -0.421 1.107 0.628 -0.865 0.845 0.000 0.661 -0.008 -0.565 0.000 0.893 0.705 0.988 0.998 1.339 0.908 0.872
|
|
||||||
1 0.953 -1.651 -0.167 0.885 1.053 1.013 -1.239 0.133 0.000 1.884 -1.122 1.222 2.215 1.906 -0.860 -1.184 1.274 1.413 -0.668 -1.647 0.000 1.873 1.510 1.133 1.050 1.678 1.246 1.061
|
|
||||||
1 0.986 -0.892 -1.380 0.917 1.134 0.950 -1.162 -0.469 0.000 0.569 -1.393 0.215 0.000 0.320 2.667 1.712 0.000 1.570 -0.375 1.457 3.102 0.925 1.128 1.011 0.598 0.824 0.913 0.833
|
|
||||||
1 1.067 0.099 1.154 0.527 -0.789 1.085 0.623 -1.602 2.173 1.511 -0.230 0.022 2.215 0.269 -0.377 0.883 0.000 0.571 -0.540 -0.512 0.000 0.414 0.803 1.022 0.959 2.053 1.041 0.780
|
|
||||||
0 0.825 -2.118 0.217 1.453 -0.493 0.819 0.313 -0.942 0.000 2.098 -0.725 1.096 2.215 0.484 1.336 1.458 0.000 0.482 0.100 1.163 0.000 0.913 0.536 0.990 1.679 0.957 1.095 1.143
|
|
||||||
1 1.507 0.054 1.120 0.698 -1.340 0.912 0.384 0.015 1.087 0.720 0.247 -0.820 0.000 0.286 0.154 1.578 2.548 0.629 1.582 -0.576 0.000 0.828 0.893 1.136 0.514 0.632 0.699 0.709
|
|
||||||
1 0.610 1.180 -0.993 0.816 0.301 0.932 0.758 1.539 0.000 0.726 -0.830 0.248 2.215 0.883 0.857 -1.305 0.000 1.338 1.009 -0.252 3.102 0.901 1.074 0.987 0.875 1.159 1.035 0.858
|
|
||||||
1 1.247 -1.360 1.502 1.525 -1.332 0.618 1.063 0.755 0.000 0.582 -0.155 0.473 2.215 1.214 -0.422 -0.551 2.548 0.838 -1.171 -1.166 0.000 2.051 1.215 1.062 1.091 0.725 0.896 1.091
|
|
||||||
0 0.373 -0.600 1.291 2.573 0.207 0.765 -0.209 1.667 0.000 0.668 0.724 -1.499 0.000 1.045 -0.338 -0.754 2.548 0.558 -0.469 0.029 3.102 0.868 0.939 1.124 0.519 0.383 0.636 0.838
|
|
||||||
0 0.791 0.336 -0.307 0.494 1.213 1.158 0.336 1.081 2.173 0.918 1.289 -0.449 0.000 0.735 -0.521 -0.969 0.000 1.052 0.499 -1.188 3.102 0.699 1.013 0.987 0.622 1.050 0.712 0.661
|
|
||||||
0 1.321 0.856 0.464 0.202 0.901 1.144 0.120 -1.651 0.000 0.803 0.577 -0.509 2.215 0.695 -0.114 0.423 2.548 0.621 1.852 -0.420 0.000 0.697 0.964 0.983 0.527 0.659 0.719 0.729
|
|
||||||
0 0.563 2.081 0.913 0.982 -0.533 0.549 -0.481 -1.730 0.000 0.962 0.921 0.569 2.215 0.731 1.184 -0.679 1.274 0.918 0.931 -1.432 0.000 1.008 0.919 0.993 0.895 0.819 0.810 0.878
|
|
||||||
1 1.148 0.345 0.953 0.921 0.617 0.991 1.103 -0.484 0.000 0.970 1.978 1.525 0.000 1.150 0.689 -0.757 2.548 0.517 0.995 1.245 0.000 1.093 1.140 0.998 1.006 0.756 0.864 0.838
|
|
||||||
1 1.400 0.128 -1.695 1.169 1.070 1.094 -0.345 -0.249 0.000 1.224 0.364 -0.036 2.215 1.178 0.530 -1.544 0.000 1.334 0.933 1.604 0.000 0.560 1.267 1.073 0.716 0.780 0.832 0.792
|
|
||||||
0 0.330 -2.133 1.403 0.628 0.379 1.686 -0.995 0.030 1.087 2.071 0.127 -0.457 0.000 4.662 -0.855 1.477 0.000 2.072 -0.917 -1.416 3.102 5.403 3.074 0.977 0.936 1.910 2.325 1.702
|
|
||||||
0 0.989 0.473 0.968 1.970 1.368 0.844 0.574 -0.290 2.173 0.866 -0.345 -1.019 0.000 1.130 0.605 -0.752 0.000 0.956 -0.888 0.870 3.102 0.885 0.886 0.982 1.157 1.201 1.100 1.068
|
|
||||||
1 0.773 0.418 0.753 1.388 1.070 1.104 -0.378 -0.758 0.000 1.027 0.397 -0.496 2.215 1.234 0.027 1.084 2.548 0.936 0.209 1.677 0.000 1.355 1.020 0.983 0.550 1.206 0.916 0.931
|
|
||||||
0 0.319 2.015 1.534 0.570 -1.134 0.632 0.124 0.757 0.000 0.477 0.598 -1.109 1.107 0.449 0.438 -0.755 2.548 0.574 -0.659 0.691 0.000 0.440 0.749 0.985 0.517 0.158 0.505 0.522
|
|
||||||
0 1.215 1.453 -1.386 1.276 1.298 0.643 0.570 -0.196 2.173 0.588 2.104 0.498 0.000 0.617 -0.296 -0.801 2.548 0.452 0.110 0.313 0.000 0.815 0.953 1.141 1.166 0.547 0.892 0.807
|
|
||||||
1 1.257 -1.869 -0.060 0.265 0.653 1.527 -0.346 1.163 2.173 0.758 -2.119 -0.604 0.000 1.473 -1.133 -1.290 2.548 0.477 -0.428 -0.066 0.000 0.818 0.841 0.984 1.446 1.729 1.211 1.054
|
|
||||||
1 1.449 0.464 1.585 1.418 -1.488 1.540 0.942 0.087 0.000 0.898 0.402 -0.631 2.215 0.753 0.039 -1.729 0.000 0.859 0.849 -1.054 0.000 0.791 0.677 0.995 0.687 0.527 0.703 0.606
|
|
||||||
1 1.084 -1.997 0.900 1.333 1.024 0.872 -0.864 -1.500 2.173 1.072 -0.813 -0.421 2.215 0.924 0.478 0.304 0.000 0.992 -0.398 -1.022 0.000 0.741 1.085 0.980 1.221 1.176 1.032 0.961
|
|
||||||
0 1.712 1.129 0.125 1.120 -1.402 1.749 0.951 -1.575 2.173 1.711 0.445 0.578 0.000 1.114 0.234 -1.011 0.000 1.577 -0.088 0.086 3.102 2.108 1.312 1.882 1.597 2.009 1.441 1.308
|
|
||||||
0 0.530 0.248 1.622 1.450 -1.012 1.221 -1.154 -0.763 2.173 1.698 -0.586 0.733 0.000 0.889 1.042 1.038 1.274 0.657 0.008 0.701 0.000 0.430 1.005 0.983 0.930 2.264 1.357 1.146
|
|
||||||
1 0.921 1.735 0.883 0.699 -1.614 0.821 1.463 0.319 1.087 1.099 0.814 -1.600 2.215 1.375 0.702 -0.691 0.000 0.869 1.326 -0.790 0.000 0.980 0.900 0.988 0.832 1.452 0.816 0.709
|
|
||||||
0 2.485 -0.823 -0.297 0.886 -1.404 0.989 0.835 1.615 2.173 0.382 0.588 -0.224 0.000 1.029 -0.456 1.546 2.548 0.613 -0.359 -0.789 0.000 0.768 0.977 1.726 2.007 0.913 1.338 1.180
|
|
||||||
1 0.657 -0.069 -0.078 1.107 1.549 0.804 1.335 -1.630 2.173 1.271 0.481 0.153 1.107 1.028 0.144 -0.762 0.000 1.098 0.132 1.570 0.000 0.830 0.979 1.175 1.069 1.624 1.000 0.868
|
|
||||||
1 2.032 0.329 -1.003 0.493 -0.136 1.159 -0.224 0.750 1.087 0.396 0.546 0.587 0.000 0.620 1.805 0.982 0.000 1.236 0.744 -1.621 0.000 0.930 1.200 0.988 0.482 0.771 0.887 0.779
|
|
||||||
0 0.524 -1.319 0.634 0.471 1.221 0.599 -0.588 -0.461 0.000 1.230 -1.504 -1.517 1.107 1.436 -0.035 0.104 2.548 0.629 1.997 -1.282 0.000 2.084 1.450 0.984 1.084 1.827 1.547 1.213
|
|
||||||
1 0.871 0.618 -1.544 0.718 0.186 1.041 -1.180 0.434 2.173 1.133 1.558 -1.301 0.000 0.452 -0.595 0.522 0.000 0.665 0.567 0.130 3.102 1.872 1.114 1.095 1.398 0.979 1.472 1.168
|
|
||||||
1 3.308 1.037 -0.634 0.690 -0.619 1.975 0.949 1.280 0.000 0.826 0.546 -0.139 2.215 0.635 -0.045 0.427 0.000 1.224 0.112 1.339 3.102 1.756 1.050 0.992 0.738 0.903 0.968 1.238
|
|
||||||
0 0.588 2.104 -0.872 1.136 1.743 0.842 0.638 0.015 0.000 0.481 0.928 1.000 2.215 0.595 0.125 1.429 0.000 0.951 -1.140 -0.511 3.102 1.031 1.057 0.979 0.673 1.064 1.001 0.891
|
|
||||||
0 0.289 0.823 0.013 0.615 -1.601 0.177 2.403 -0.015 0.000 0.258 1.151 1.036 2.215 0.694 0.553 -1.326 2.548 0.411 0.366 0.106 0.000 0.482 0.562 0.989 0.670 0.404 0.516 0.561
|
|
||||||
1 0.294 -0.660 -1.162 1.752 0.384 0.860 0.513 1.119 0.000 2.416 0.107 -1.342 0.000 1.398 0.361 -0.350 2.548 1.126 -0.902 0.040 1.551 0.650 1.125 0.988 0.531 0.843 0.912 0.911
|
|
||||||
0 0.599 -0.616 1.526 1.381 0.507 0.955 -0.646 -0.085 2.173 0.775 -0.533 1.116 2.215 0.789 -0.136 -1.176 0.000 2.449 1.435 -1.433 0.000 1.692 1.699 1.000 0.869 1.119 1.508 1.303
|
|
||||||
1 1.100 -1.174 -1.114 1.601 -1.576 1.056 -1.343 0.547 2.173 0.555 0.367 0.592 2.215 0.580 -1.862 -0.914 0.000 0.904 0.508 -0.444 0.000 1.439 1.105 0.986 1.408 1.104 1.190 1.094
|
|
||||||
1 2.237 -0.701 1.470 0.719 -0.199 0.745 -0.132 -0.737 1.087 0.976 -0.227 0.093 2.215 0.699 0.057 1.133 0.000 0.661 0.573 -0.679 0.000 0.785 0.772 1.752 1.235 0.856 0.990 0.825
|
|
||||||
1 0.455 -0.880 -1.482 1.260 -0.178 1.499 0.158 1.022 0.000 1.867 -0.435 -0.675 2.215 1.234 0.783 1.586 0.000 0.641 -0.454 -0.409 3.102 1.002 0.964 0.986 0.761 0.240 1.190 0.995
|
|
||||||
1 1.158 -0.778 -0.159 0.823 1.641 1.341 -0.830 -1.169 2.173 0.840 -1.554 0.934 0.000 0.693 0.488 -1.218 2.548 1.042 1.395 0.276 0.000 0.946 0.785 1.350 1.079 0.893 1.267 1.151
|
|
||||||
1 0.902 -0.078 -0.055 0.872 -0.012 0.843 1.276 1.739 2.173 0.838 1.492 0.918 0.000 0.626 0.904 -0.648 2.548 0.412 -2.027 -0.883 0.000 2.838 1.664 0.988 1.803 0.768 1.244 1.280
|
|
||||||
1 0.649 -1.028 -1.521 1.097 0.774 1.216 -0.383 -0.318 2.173 1.643 -0.285 -1.705 0.000 0.911 -0.091 0.341 0.000 0.592 0.537 0.732 3.102 0.911 0.856 1.027 1.160 0.874 0.986 0.893
|
|
||||||
1 1.192 1.846 -0.781 1.326 -0.747 1.550 1.177 1.366 0.000 1.196 0.151 0.387 2.215 0.527 2.261 -0.190 0.000 0.390 1.474 0.381 0.000 0.986 1.025 1.004 1.392 0.761 0.965 1.043
|
|
||||||
0 0.438 -0.358 -1.549 0.836 0.436 0.818 0.276 -0.708 2.173 0.707 0.826 0.392 0.000 1.050 1.741 -1.066 0.000 1.276 -1.583 0.842 0.000 1.475 1.273 0.986 0.853 1.593 1.255 1.226
|
|
||||||
1 1.083 0.142 1.701 0.605 -0.253 1.237 0.791 1.183 2.173 0.842 2.850 -0.082 0.000 0.724 -0.464 -0.694 0.000 1.499 0.456 -0.226 3.102 0.601 0.799 1.102 0.995 1.389 1.013 0.851
|
|
||||||
0 0.828 1.897 -0.615 0.572 -0.545 0.572 0.461 0.464 2.173 0.393 0.356 1.069 2.215 1.840 0.088 1.500 0.000 0.407 -0.663 -0.787 0.000 0.950 0.965 0.979 0.733 0.363 0.618 0.733
|
|
||||||
0 0.735 1.438 1.197 1.123 -0.214 0.641 0.949 0.858 0.000 1.162 0.524 -0.896 2.215 0.992 0.454 -1.475 2.548 0.902 1.079 0.019 0.000 0.822 0.917 1.203 1.032 0.569 0.780 0.764
|
|
||||||
0 0.437 -2.102 0.044 1.779 -1.042 1.231 -0.181 -0.515 1.087 2.666 0.863 1.466 2.215 1.370 0.345 -1.371 0.000 0.906 0.363 1.611 0.000 1.140 1.362 1.013 3.931 3.004 2.724 2.028
|
|
||||||
1 0.881 1.814 -0.987 0.384 0.800 2.384 1.422 0.640 0.000 1.528 0.292 -0.962 1.107 2.126 -0.371 -1.401 2.548 0.700 0.109 0.203 0.000 0.450 0.813 0.985 0.956 1.013 0.993 0.774
|
|
||||||
1 0.630 0.408 0.152 0.194 0.316 0.710 -0.824 -0.358 2.173 0.741 0.535 -0.851 2.215 0.933 0.406 1.148 0.000 0.523 -0.479 -0.625 0.000 0.873 0.960 0.988 0.830 0.921 0.711 0.661
|
|
||||||
1 0.870 -0.448 -1.134 0.616 0.135 0.600 0.649 -0.622 2.173 0.768 0.709 -0.123 0.000 1.308 0.500 1.468 0.000 1.973 -0.286 1.462 3.102 0.909 0.944 0.990 0.835 1.250 0.798 0.776
|
|
||||||
0 1.290 0.552 1.330 0.615 -1.353 0.661 0.240 -0.393 0.000 0.531 0.053 -1.588 0.000 0.675 0.839 -0.345 1.274 1.597 0.020 0.536 3.102 1.114 0.964 0.987 0.783 0.675 0.662 0.675
|
|
||||||
1 0.943 0.936 1.068 1.373 0.671 2.170 -2.011 -1.032 0.000 0.640 0.361 -0.806 0.000 2.239 -0.083 0.590 2.548 1.224 0.646 -1.723 0.000 0.879 0.834 0.981 1.436 0.568 0.916 0.931
|
|
||||||
1 0.431 1.686 -1.053 0.388 1.739 0.457 -0.471 -0.743 2.173 0.786 1.432 -0.547 2.215 0.537 -0.413 1.256 0.000 0.413 2.311 -0.408 0.000 1.355 1.017 0.982 0.689 1.014 0.821 0.715
|
|
||||||
0 1.620 -0.055 -0.862 1.341 -1.571 0.634 -0.906 0.935 2.173 0.501 -2.198 -0.525 0.000 0.778 -0.708 -0.060 0.000 0.988 -0.621 0.489 3.102 0.870 0.956 1.216 0.992 0.336 0.871 0.889
|
|
||||||
1 0.549 0.304 -1.443 1.309 -0.312 1.116 0.644 1.519 2.173 1.078 -0.303 -0.736 0.000 1.261 0.387 0.628 2.548 0.945 -0.190 0.090 0.000 0.893 1.043 1.000 1.124 1.077 1.026 0.886
|
|
||||||
0 0.412 -0.618 -1.486 1.133 -0.665 0.646 0.436 1.520 0.000 0.993 0.976 0.106 2.215 0.832 0.091 0.164 2.548 0.672 -0.650 1.256 0.000 0.695 1.131 0.991 1.017 0.455 1.226 1.087
|
|
||||||
0 1.183 -0.084 1.644 1.389 0.967 0.843 0.938 -0.670 0.000 0.480 0.256 0.123 2.215 0.437 1.644 0.491 0.000 0.501 -0.416 0.101 3.102 1.060 0.804 1.017 0.775 0.173 0.535 0.760
|
|
||||||
0 1.629 -1.486 -0.683 2.786 -0.492 1.347 -2.638 1.453 0.000 1.857 0.208 0.873 0.000 0.519 -1.265 -1.602 1.274 0.903 -1.102 -0.329 1.551 6.892 3.522 0.998 0.570 0.477 2.039 2.006
|
|
||||||
1 2.045 -0.671 -1.235 0.490 -0.952 0.525 -1.252 1.289 0.000 1.088 -0.993 0.648 2.215 0.975 -0.109 -0.254 2.548 0.556 -1.095 -0.194 0.000 0.803 0.861 0.980 1.282 0.945 0.925 0.811
|
|
||||||
0 0.448 -0.058 -0.974 0.945 -1.633 1.181 -1.139 0.266 2.173 1.118 -0.761 1.502 1.107 1.706 0.585 -0.680 0.000 0.487 -1.951 0.945 0.000 2.347 1.754 0.993 1.161 1.549 1.414 1.176
|
|
||||||
0 0.551 0.519 0.448 2.183 1.293 1.220 0.628 -0.627 2.173 1.019 -0.002 -0.652 0.000 1.843 -0.386 1.042 2.548 0.400 -1.102 -1.014 0.000 0.648 0.792 1.049 0.888 2.132 1.262 1.096
|
|
||||||
0 1.624 0.488 1.403 0.760 0.559 0.812 0.777 -1.244 2.173 0.613 0.589 -0.030 2.215 0.692 1.058 0.683 0.000 1.054 1.165 -0.765 0.000 0.915 0.875 1.059 0.821 0.927 0.792 0.721
|
|
||||||
1 0.774 0.444 1.257 0.515 -0.689 0.515 1.448 -1.271 0.000 0.793 0.118 0.811 1.107 0.679 0.326 -0.426 0.000 1.066 -0.865 -0.049 3.102 0.960 1.046 0.986 0.716 0.772 0.855 0.732
|
|
||||||
1 2.093 -1.240 1.615 0.918 -1.202 1.412 -0.541 0.640 1.087 2.019 0.872 -0.639 0.000 0.672 -0.936 0.972 0.000 0.896 0.235 0.212 0.000 0.810 0.700 1.090 0.797 0.862 1.049 0.874
|
|
||||||
1 0.908 1.069 0.283 0.400 1.293 0.609 1.452 -1.136 0.000 0.623 0.417 -0.098 2.215 1.023 0.775 1.054 1.274 0.706 2.346 -1.305 0.000 0.744 1.006 0.991 0.606 0.753 0.796 0.753
|
|
||||||
0 0.403 -1.328 -0.065 0.901 1.052 0.708 -0.354 -0.718 2.173 0.892 0.633 1.684 2.215 0.999 -1.205 0.941 0.000 0.930 1.072 -0.809 0.000 2.105 1.430 0.989 0.838 1.147 1.042 0.883
|
|
||||||
0 1.447 0.453 0.118 1.731 0.650 0.771 0.446 -1.564 0.000 0.973 -2.014 0.354 0.000 1.949 -0.643 -1.531 1.274 1.106 -0.334 -1.163 0.000 0.795 0.821 1.013 1.699 0.918 1.118 1.018
|
|
||||||
1 1.794 0.123 -0.454 0.057 1.489 0.966 -1.190 1.090 1.087 0.539 -0.535 1.035 0.000 1.096 -1.069 -1.236 2.548 0.659 -1.196 -0.283 0.000 0.803 0.756 0.985 1.343 1.109 0.993 0.806
|
|
||||||
0 1.484 -2.047 0.813 0.591 -0.295 0.923 0.312 -1.164 2.173 0.654 -0.316 0.752 2.215 0.599 1.966 -1.128 0.000 0.626 -0.304 -1.431 0.000 1.112 0.910 1.090 0.986 1.189 1.350 1.472
|
|
||||||
0 0.417 -2.016 0.849 1.817 0.040 1.201 -1.676 -1.394 0.000 0.792 0.537 0.641 2.215 0.794 -1.222 0.187 0.000 0.825 -0.217 1.334 3.102 1.470 0.931 0.987 1.203 0.525 0.833 0.827
|
|
||||||
1 0.603 1.009 0.033 0.486 1.225 0.884 -0.617 -1.058 0.000 0.500 -1.407 -0.567 0.000 1.476 -0.876 0.605 2.548 0.970 0.560 1.092 3.102 0.853 1.153 0.988 0.846 0.920 0.944 0.835
|
|
||||||
1 1.381 -0.326 0.552 0.417 -0.027 1.030 -0.835 -1.287 2.173 0.941 -0.421 1.519 2.215 0.615 -1.650 0.377 0.000 0.606 0.644 0.650 0.000 1.146 0.970 0.990 1.191 0.884 0.897 0.826
|
|
||||||
1 0.632 1.200 -0.703 0.438 -1.700 0.779 -0.731 0.958 1.087 0.605 0.393 -1.376 0.000 0.670 -0.827 -1.315 2.548 0.626 -0.501 0.417 0.000 0.904 0.903 0.998 0.673 0.803 0.722 0.640
|
|
||||||
1 1.561 -0.569 1.580 0.329 0.237 1.059 0.731 0.415 2.173 0.454 0.016 -0.828 0.000 0.587 0.008 -0.291 1.274 0.597 1.119 1.191 0.000 0.815 0.908 0.988 0.733 0.690 0.892 0.764
|
|
||||||
1 2.102 0.087 0.449 1.164 -0.390 1.085 -0.408 -1.116 2.173 0.578 0.197 -0.137 0.000 1.202 0.917 1.523 0.000 0.959 -0.832 1.404 3.102 1.380 1.109 1.486 1.496 0.886 1.066 1.025
|
|
||||||
1 1.698 -0.489 -0.552 0.976 -1.009 1.620 -0.721 0.648 1.087 1.481 -1.860 -1.354 0.000 1.142 -1.140 1.401 2.548 1.000 -1.274 -0.158 0.000 1.430 1.130 0.987 1.629 1.154 1.303 1.223
|
|
||||||
1 1.111 -0.249 -1.457 0.421 0.939 0.646 -2.076 0.362 0.000 1.315 0.796 -1.436 2.215 0.780 0.130 0.055 0.000 1.662 -0.834 0.461 0.000 0.920 0.948 0.990 1.046 0.905 1.493 1.169
|
|
||||||
1 0.945 0.390 -1.159 1.675 0.437 0.356 0.261 0.543 1.087 0.574 0.838 1.599 2.215 0.496 -1.220 -0.022 0.000 0.558 -2.454 1.440 0.000 0.763 0.983 1.728 1.000 0.578 0.922 1.003
|
|
||||||
1 2.076 0.014 -1.314 0.854 -0.306 3.446 1.341 0.598 0.000 2.086 0.227 -0.747 2.215 1.564 -0.216 1.649 2.548 0.965 -0.857 -1.062 0.000 0.477 0.734 1.456 1.003 1.660 1.001 0.908
|
|
||||||
1 1.992 0.192 -0.103 0.108 -1.599 0.938 0.595 -1.360 2.173 0.869 -1.012 1.432 0.000 1.302 0.850 0.436 2.548 0.487 1.051 -1.027 0.000 0.502 0.829 0.983 1.110 1.394 0.904 0.836
|
|
||||||
0 0.460 1.625 1.485 1.331 1.242 0.675 -0.329 -1.039 1.087 0.671 -1.028 -0.514 0.000 1.265 -0.788 0.415 1.274 0.570 -0.683 -1.738 0.000 0.725 0.758 1.004 1.024 1.156 0.944 0.833
|
|
||||||
0 0.871 0.839 -1.536 0.428 1.198 0.875 -1.256 -0.466 1.087 0.684 -0.768 0.150 0.000 0.556 -1.793 0.389 0.000 0.942 -1.126 1.339 1.551 0.624 0.734 0.986 1.357 0.960 1.474 1.294
|
|
||||||
1 0.951 1.651 0.576 1.273 1.495 0.834 0.048 -0.578 2.173 0.386 -0.056 -1.448 0.000 0.597 -0.196 0.162 2.548 0.524 1.649 1.625 0.000 0.737 0.901 1.124 1.014 0.556 1.039 0.845
|
|
||||||
1 1.049 -0.223 0.685 0.256 -1.191 2.506 0.238 -0.359 0.000 1.510 -0.904 1.158 1.107 2.733 -0.902 1.679 2.548 0.407 -0.474 -1.572 0.000 1.513 2.472 0.982 1.238 0.978 1.985 1.510
|
|
||||||
0 0.455 -0.028 0.265 1.286 1.373 0.459 0.331 -0.922 0.000 0.343 0.634 0.430 0.000 0.279 -0.084 -0.272 0.000 0.475 0.926 -0.123 3.102 0.803 0.495 0.987 0.587 0.211 0.417 0.445
|
|
||||||
1 2.074 0.388 0.878 1.110 1.557 1.077 -0.226 -0.295 2.173 0.865 -0.319 -1.116 2.215 0.707 -0.835 0.722 0.000 0.632 -0.608 -0.728 0.000 0.715 0.802 1.207 1.190 0.960 1.143 0.926
|
|
||||||
1 1.390 0.265 1.196 0.919 -1.371 1.858 0.506 0.786 0.000 1.280 -1.367 -0.720 2.215 1.483 -0.441 -0.675 2.548 1.076 0.294 -0.539 0.000 1.126 0.830 1.155 1.551 0.702 1.103 0.933
|
|
||||||
1 1.014 -0.079 1.597 1.038 -0.281 1.135 -0.722 -0.177 2.173 0.544 -1.475 -1.501 0.000 1.257 -1.315 1.212 0.000 0.496 -0.060 1.180 1.551 0.815 0.611 1.411 1.110 0.792 0.846 0.853
|
|
||||||
0 0.335 1.267 -1.154 2.011 -0.574 0.753 0.618 1.411 0.000 0.474 0.748 0.681 2.215 0.608 -0.446 -0.354 2.548 0.399 1.295 -0.581 0.000 0.911 0.882 0.975 0.832 0.598 0.580 0.678
|
|
||||||
1 0.729 -0.189 1.182 0.293 1.310 0.412 0.459 -0.632 0.000 0.869 -1.128 -0.625 2.215 1.173 -0.893 0.478 2.548 0.584 -2.394 -1.727 0.000 2.016 1.272 0.995 1.034 0.905 0.966 1.038
|
|
||||||
1 1.225 -1.215 -0.088 0.881 -0.237 0.600 -0.976 1.462 2.173 0.876 0.506 1.583 2.215 0.718 1.228 -0.031 0.000 0.653 -1.292 1.216 0.000 0.838 1.108 0.981 1.805 0.890 1.251 1.197
|
|
||||||
1 2.685 -0.444 0.847 0.253 0.183 0.641 -1.541 -0.873 2.173 0.417 2.874 -0.551 0.000 0.706 -1.431 0.764 0.000 1.390 -0.596 -1.397 0.000 0.894 0.829 0.993 0.789 0.654 0.883 0.746
|
|
||||||
0 0.638 -0.481 0.683 1.457 -1.024 0.707 -1.338 1.498 0.000 0.980 0.518 0.289 2.215 0.964 -0.531 -0.423 0.000 0.694 -0.654 -1.314 3.102 0.807 1.283 1.335 0.658 0.907 0.797 0.772
|
|
||||||
1 1.789 -0.765 -0.732 0.421 -0.020 1.142 -1.353 1.439 2.173 0.725 -1.518 -1.261 0.000 0.812 -2.597 -0.463 0.000 1.203 -0.120 1.001 0.000 0.978 0.673 0.985 1.303 1.400 1.078 0.983
|
|
||||||
1 0.784 -1.431 1.724 0.848 0.559 0.615 -1.643 -1.456 0.000 1.339 -0.513 0.040 2.215 0.394 -2.483 1.304 0.000 0.987 0.889 -0.339 0.000 0.732 0.713 0.987 0.973 0.705 0.875 0.759
|
|
||||||
1 0.911 1.098 -1.289 0.421 0.823 1.218 -0.503 0.431 0.000 0.775 0.432 -1.680 0.000 0.855 -0.226 -0.460 2.548 0.646 -0.947 -1.243 1.551 2.201 1.349 0.985 0.730 0.451 0.877 0.825
|
|
||||||
1 0.959 0.372 -0.269 1.255 0.702 1.151 0.097 0.805 2.173 0.993 1.011 0.767 2.215 1.096 0.185 0.381 0.000 1.001 -0.205 0.059 0.000 0.979 0.997 1.168 0.796 0.771 0.839 0.776
|
|
||||||
0 0.283 -1.864 -1.663 0.219 1.624 0.955 -1.213 0.932 2.173 0.889 0.395 -0.268 0.000 0.597 -1.083 -0.921 2.548 0.584 1.325 -1.072 0.000 0.856 0.927 0.996 0.937 0.936 1.095 0.892
|
|
||||||
0 2.017 -0.488 -0.466 1.029 -0.870 3.157 0.059 -0.343 2.173 3.881 0.872 1.502 1.107 3.631 1.720 0.963 0.000 0.633 -1.264 -1.734 0.000 4.572 3.339 1.005 1.407 5.590 3.614 3.110
|
|
||||||
1 1.088 0.414 -0.841 0.485 0.605 0.860 1.110 -0.568 0.000 1.152 -0.325 1.203 2.215 0.324 1.652 -0.104 0.000 0.510 1.095 -1.728 0.000 0.880 0.722 0.989 0.977 0.711 0.888 0.762
|
|
||||||
0 0.409 -1.717 0.712 0.809 -1.301 0.701 -1.529 -1.411 0.000 1.191 -0.582 0.438 2.215 1.147 0.813 -0.571 2.548 1.039 0.543 0.892 0.000 0.636 0.810 0.986 0.861 1.411 0.907 0.756
|
|
||||||
1 1.094 1.577 -0.988 0.497 -0.149 0.891 -2.459 1.034 0.000 0.646 0.792 -1.022 0.000 1.573 0.254 -0.053 2.548 1.428 0.190 -1.641 3.102 4.322 2.687 0.985 0.881 1.135 1.907 1.831
|
|
||||||
1 0.613 1.993 -0.280 0.544 0.931 0.909 1.526 1.559 0.000 0.840 1.473 -0.483 2.215 0.856 0.352 0.408 2.548 1.058 1.733 -1.396 0.000 0.801 1.066 0.984 0.639 0.841 0.871 0.748
|
|
||||||
0 0.958 -1.202 0.600 0.434 0.170 0.783 -0.214 1.319 0.000 0.835 -0.454 -0.615 2.215 0.658 -1.858 -0.891 0.000 0.640 0.172 -1.204 3.102 1.790 1.086 0.997 0.804 0.403 0.793 0.756
|
|
||||||
1 1.998 -0.238 0.972 0.058 0.266 0.759 1.576 -0.357 2.173 1.004 -0.349 -0.747 2.215 0.962 0.490 -0.453 0.000 1.592 0.661 -1.405 0.000 0.874 1.086 0.990 1.436 1.527 1.177 0.993
|
|
||||||
1 0.796 -0.171 -0.818 0.574 -1.625 1.201 -0.737 1.451 2.173 0.651 0.404 -0.452 0.000 1.150 -0.652 -0.120 0.000 1.008 -0.093 0.531 3.102 0.884 0.706 0.979 1.193 0.937 0.943 0.881
|
|
||||||
1 0.773 1.023 0.527 1.537 -0.201 2.967 -0.574 -1.534 2.173 2.346 -0.307 0.394 2.215 1.393 0.135 -0.027 0.000 3.015 0.187 0.516 0.000 0.819 1.260 0.982 2.552 3.862 2.179 1.786
|
|
||||||
0 1.823 1.008 -1.489 0.234 -0.962 0.591 0.461 0.996 2.173 0.568 -1.297 -0.410 0.000 0.887 2.157 1.194 0.000 2.079 0.369 -0.085 3.102 0.770 0.945 0.995 1.179 0.971 0.925 0.983
|
|
||||||
0 0.780 0.640 0.490 0.680 -1.301 0.715 -0.137 0.152 2.173 0.616 -0.831 1.668 0.000 1.958 0.528 -0.982 2.548 0.966 -1.551 0.462 0.000 1.034 1.079 1.008 0.827 1.369 1.152 0.983
|
|
||||||
1 0.543 0.801 1.543 1.134 -0.772 0.954 -0.849 0.410 1.087 0.851 -1.988 1.686 0.000 0.799 -0.912 -1.156 0.000 0.479 0.097 1.334 0.000 0.923 0.597 0.989 1.231 0.759 0.975 0.867
|
|
||||||
0 1.241 -0.014 0.129 1.158 0.670 0.445 -0.732 1.739 2.173 0.918 0.659 -1.340 2.215 0.557 2.410 -1.404 0.000 0.966 -1.545 -1.120 0.000 0.874 0.918 0.987 1.001 0.798 0.904 0.937
|
|
||||||
0 1.751 -0.266 -1.575 0.489 1.292 1.112 1.533 0.137 2.173 1.204 -0.414 -0.928 0.000 0.879 1.237 -0.415 2.548 1.479 1.469 0.913 0.000 2.884 1.747 0.989 1.742 0.600 1.363 1.293
|
|
||||||
1 1.505 1.208 -1.476 0.995 -0.836 2.800 -1.600 0.111 0.000 2.157 1.241 1.110 2.215 1.076 2.619 -0.913 0.000 1.678 2.204 -1.575 0.000 0.849 1.224 0.990 1.412 0.976 1.271 1.105
|
|
||||||
0 0.816 0.611 0.779 1.694 0.278 0.575 -0.787 1.592 2.173 1.148 1.076 -0.831 2.215 0.421 1.316 0.632 0.000 0.589 0.452 -1.466 0.000 0.779 0.909 0.990 1.146 1.639 1.236 0.949
|
|
||||||
1 0.551 -0.808 0.330 1.188 -0.294 0.447 -0.035 -0.993 0.000 0.432 -0.276 -0.481 2.215 1.959 -0.288 1.195 2.548 0.638 0.583 1.107 0.000 0.832 0.924 0.993 0.723 0.976 0.968 0.895
|
|
||||||
0 1.316 -0.093 0.995 0.860 -0.621 0.593 -0.560 -1.599 2.173 0.524 -0.318 -0.240 2.215 0.566 0.759 -0.368 0.000 0.483 -2.030 -1.104 0.000 1.468 1.041 1.464 0.811 0.778 0.690 0.722
|
|
||||||
1 1.528 0.067 -0.855 0.959 -1.464 1.143 -0.082 1.023 0.000 0.702 -0.763 -0.244 0.000 0.935 -0.881 0.206 2.548 0.614 -0.831 1.657 3.102 1.680 1.105 0.983 1.078 0.559 0.801 0.809
|
|
||||||
0 0.558 -0.833 -0.598 1.436 -1.724 1.316 -0.661 1.593 2.173 1.148 -0.503 -0.132 1.107 1.584 -0.125 0.380 0.000 1.110 -1.216 -0.181 0.000 1.258 0.860 1.053 0.790 1.814 1.159 1.007
|
|
||||||
1 0.819 0.879 1.221 0.598 -1.450 0.754 0.417 -0.369 2.173 0.477 1.199 0.274 0.000 1.073 0.368 0.273 2.548 1.599 2.047 1.690 0.000 0.933 0.984 0.983 0.788 0.613 0.728 0.717
|
|
||||||
0 0.981 -1.007 0.489 0.923 1.261 0.436 -0.698 -0.506 2.173 0.764 -1.105 -1.241 2.215 0.577 -2.573 -0.036 0.000 0.565 -1.628 1.610 0.000 0.688 0.801 0.991 0.871 0.554 0.691 0.656
|
|
||||||
0 2.888 0.568 -1.416 1.461 -1.157 1.756 -0.900 0.522 0.000 0.657 0.409 1.076 2.215 1.419 0.672 -0.019 0.000 1.436 -0.184 -0.980 3.102 0.946 0.919 0.995 1.069 0.890 0.834 0.856
|
|
||||||
1 0.522 1.805 -0.963 1.136 0.418 0.727 -0.195 -1.695 2.173 0.309 2.559 -0.178 0.000 0.521 1.794 0.919 0.000 0.788 0.174 -0.406 3.102 0.555 0.729 1.011 1.385 0.753 0.927 0.832
|
|
||||||
1 0.793 -0.162 -1.643 0.634 0.337 0.898 -0.633 1.689 0.000 0.806 -0.826 -0.356 2.215 0.890 -0.142 -1.268 0.000 1.293 0.574 0.725 0.000 0.833 1.077 0.988 0.721 0.679 0.867 0.753
|
|
||||||
0 1.298 1.098 0.280 0.371 -0.373 0.855 -0.306 -1.186 0.000 0.977 -0.421 1.003 0.000 0.978 0.956 -1.249 2.548 0.735 0.577 -0.037 3.102 0.974 1.002 0.992 0.549 0.587 0.725 0.954
|
|
||||||
1 0.751 -0.520 -1.653 0.168 -0.419 0.878 -1.023 -1.364 2.173 1.310 -0.667 0.863 0.000 1.196 -0.827 0.358 0.000 1.154 -0.165 -0.360 1.551 0.871 0.950 0.983 0.907 0.955 0.959 0.874
|
|
||||||
0 1.730 0.666 -1.432 0.446 1.302 0.921 -0.203 0.621 0.000 1.171 -0.365 -0.611 1.107 0.585 0.807 1.150 0.000 0.415 -0.843 1.311 0.000 0.968 0.786 0.986 1.059 0.371 0.790 0.848
|
|
||||||
1 0.596 -1.486 0.690 1.045 -1.344 0.928 0.867 0.820 2.173 0.610 0.999 -1.329 2.215 0.883 -0.001 -0.106 0.000 1.145 2.184 -0.808 0.000 2.019 1.256 1.056 1.751 1.037 1.298 1.518
|
|
||||||
1 0.656 -1.993 -0.519 1.643 -0.143 0.815 0.256 1.220 1.087 0.399 -1.184 -1.458 0.000 0.738 1.361 -1.443 0.000 0.842 0.033 0.293 0.000 0.910 0.891 0.993 0.668 0.562 0.958 0.787
|
|
||||||
1 1.127 -0.542 0.645 0.318 -1.496 0.661 -0.640 0.369 2.173 0.992 0.358 1.702 0.000 1.004 0.316 -1.109 0.000 1.616 -0.936 -0.707 1.551 0.875 1.191 0.985 0.651 0.940 0.969 0.834
|
|
||||||
0 0.916 -1.423 -1.490 1.248 -0.538 0.625 -0.535 -0.174 0.000 0.769 -0.389 1.608 2.215 0.667 -1.138 -1.738 1.274 0.877 -0.019 0.482 0.000 0.696 0.917 1.121 0.678 0.347 0.647 0.722
|
|
||||||
1 2.756 -0.637 -1.715 1.331 1.124 0.913 -0.296 -0.491 0.000 0.983 -0.831 0.000 2.215 1.180 -0.428 0.742 0.000 1.113 0.005 -1.157 1.551 1.681 1.096 1.462 0.976 0.917 1.009 1.040
|
|
||||||
0 0.755 1.754 0.701 2.111 0.256 1.243 0.057 -1.502 2.173 0.565 -0.034 -1.078 1.107 0.529 1.696 -1.090 0.000 0.665 0.292 0.107 0.000 0.870 0.780 0.990 2.775 0.465 1.876 1.758
|
|
||||||
1 0.593 -0.762 1.743 0.908 0.442 0.773 -1.357 -0.768 2.173 0.432 1.421 1.236 0.000 0.579 0.291 -0.403 0.000 0.966 -0.309 1.016 3.102 0.893 0.743 0.989 0.857 1.030 0.943 0.854
|
|
||||||
1 0.891 -1.151 -1.269 0.504 -0.622 0.893 -0.549 0.700 0.000 0.828 -0.825 0.154 2.215 1.083 0.632 -1.141 0.000 1.059 -0.557 1.526 3.102 2.117 1.281 0.987 0.819 0.802 0.917 0.828
|
|
||||||
1 2.358 -0.248 0.080 0.747 -0.975 1.019 1.374 1.363 0.000 0.935 0.127 -1.707 2.215 0.312 -0.827 0.017 0.000 0.737 1.059 -0.327 0.000 0.716 0.828 1.495 0.953 0.704 0.880 0.745
|
|
||||||
0 0.660 -0.017 -1.138 0.453 1.002 0.645 0.518 0.703 2.173 0.751 0.705 -0.592 2.215 0.744 -0.909 -1.596 0.000 0.410 -1.135 0.481 0.000 0.592 0.922 0.989 0.897 0.948 0.777 0.701
|
|
||||||
1 0.718 0.518 0.225 1.710 -0.022 1.888 -0.424 1.092 0.000 4.134 0.185 -1.366 0.000 1.415 1.293 0.242 2.548 2.351 0.264 -0.057 3.102 0.830 1.630 0.976 1.215 0.890 1.422 1.215
|
|
||||||
1 1.160 0.203 0.941 0.594 0.212 0.636 -0.556 0.679 2.173 1.089 -0.481 -1.008 1.107 1.245 -0.056 -1.357 0.000 0.587 1.007 0.056 0.000 1.106 0.901 0.987 0.786 1.224 0.914 0.837
|
|
||||||
1 0.697 0.542 0.619 0.985 1.481 0.745 0.415 1.644 2.173 0.903 0.495 -0.958 2.215 1.165 1.195 0.346 0.000 1.067 -0.881 -0.264 0.000 0.830 1.025 0.987 0.690 0.863 0.894 0.867
|
|
||||||
0 1.430 0.190 -0.700 0.246 0.518 1.302 0.660 -0.247 2.173 1.185 -0.539 1.504 0.000 1.976 -0.401 1.079 0.000 0.855 -0.958 -1.110 3.102 0.886 0.953 0.993 0.889 1.400 1.376 1.119
|
|
||||||
1 1.122 -0.795 0.202 0.397 -1.553 0.597 -1.459 -0.734 2.173 0.522 1.044 1.027 2.215 0.783 -1.243 1.701 0.000 0.371 1.737 0.199 0.000 1.719 1.176 0.988 0.723 1.583 1.063 0.914
|
|
||||||
0 1.153 0.526 1.236 0.266 0.001 1.139 -1.236 -0.585 2.173 1.337 -0.215 -1.356 2.215 1.780 1.129 0.902 0.000 1.608 -0.391 -0.161 0.000 1.441 1.633 0.990 1.838 1.516 1.635 1.373
|
|
||||||
1 0.760 1.012 0.758 0.937 0.051 0.941 0.687 -1.247 2.173 1.288 -0.743 0.822 0.000 1.552 1.782 -1.533 0.000 0.767 1.349 0.168 0.000 0.716 0.862 0.988 0.595 0.359 0.697 0.623
|
|
||||||
1 1.756 -1.469 1.395 1.345 -1.595 0.817 0.017 -0.741 2.173 0.483 -0.008 0.293 0.000 1.768 -0.663 0.438 1.274 1.202 -1.387 -0.222 0.000 1.022 1.058 0.992 1.407 1.427 1.356 1.133
|
|
||||||
0 0.397 0.582 -0.758 1.260 -1.735 0.889 -0.515 1.139 2.173 0.973 1.616 0.460 0.000 1.308 1.001 -0.709 2.548 0.858 0.995 -0.231 0.000 0.749 0.888 0.979 1.487 1.804 1.208 1.079
|
|
||||||
0 0.515 -0.984 0.425 1.114 -0.439 1.999 0.818 1.561 0.000 1.407 0.009 -0.380 0.000 1.332 0.230 0.397 0.000 1.356 -0.616 -1.057 3.102 0.978 1.017 0.990 1.118 0.862 0.835 0.919
|
|
||||||
1 1.368 -0.921 -0.866 0.842 -0.598 0.456 -1.176 1.219 1.087 0.419 -1.974 -0.819 0.000 0.791 -1.640 0.881 0.000 1.295 -0.782 0.442 3.102 0.945 0.761 0.974 0.915 0.535 0.733 0.651
|
|
||||||
0 2.276 0.134 0.399 2.525 0.376 1.111 -1.078 -1.571 0.000 0.657 2.215 -0.900 0.000 1.183 -0.662 -0.508 2.548 1.436 -0.517 0.960 3.102 0.569 0.931 0.993 1.170 0.967 0.879 1.207
|
|
||||||
0 0.849 0.907 0.124 0.652 1.585 0.715 0.355 -1.200 0.000 0.599 -0.892 1.301 0.000 1.106 1.151 0.582 0.000 1.895 -0.279 -0.568 3.102 0.881 0.945 0.998 0.559 0.649 0.638 0.660
|
|
||||||
1 2.105 0.248 -0.797 0.530 0.206 1.957 -2.175 0.797 0.000 1.193 0.637 -1.646 2.215 0.881 1.111 -1.046 0.000 0.872 -0.185 1.085 1.551 0.986 1.343 1.151 1.069 0.714 2.063 1.951
|
|
||||||
1 1.838 1.060 1.637 1.017 1.370 0.913 0.461 -0.609 1.087 0.766 -0.461 0.303 2.215 0.724 -0.061 0.886 0.000 0.941 1.123 -0.745 0.000 0.858 0.847 0.979 1.313 1.083 1.094 0.910
|
|
||||||
0 0.364 1.274 1.066 1.570 -0.394 0.485 0.012 -1.716 0.000 0.317 -1.233 0.534 2.215 0.548 -2.165 0.762 0.000 0.729 0.169 -0.318 3.102 0.892 0.944 1.013 0.594 0.461 0.688 0.715
|
|
||||||
1 0.503 1.343 -0.031 1.134 -1.204 0.590 -0.309 0.174 2.173 0.408 2.372 -0.628 0.000 1.850 0.400 1.147 2.548 0.664 -0.458 -0.885 0.000 1.445 1.283 0.989 1.280 1.118 1.127 1.026
|
|
||||||
0 1.873 0.258 0.103 2.491 0.530 1.678 0.644 -1.738 2.173 1.432 0.848 -1.340 0.000 0.621 1.323 -1.316 0.000 0.628 0.789 -0.206 1.551 0.426 0.802 1.125 0.688 1.079 1.338 1.239
|
|
||||||
1 0.826 -0.732 1.587 0.582 -1.236 0.495 0.757 -0.741 2.173 0.940 1.474 0.354 2.215 0.474 1.055 -1.657 0.000 0.415 1.758 0.841 0.000 0.451 0.578 0.984 0.757 0.922 0.860 0.696
|
|
||||||
0 0.935 -1.614 -0.597 0.299 1.223 0.707 -0.853 -1.026 0.000 0.751 0.007 -1.691 0.000 1.062 -0.125 0.976 2.548 0.877 1.275 0.646 0.000 0.962 1.074 0.980 0.608 0.726 0.741 0.662
|
|
||||||
1 0.643 0.542 -1.285 0.474 -0.366 0.667 -0.446 1.195 2.173 1.076 0.145 -0.126 0.000 0.970 -0.661 0.394 1.274 1.218 -0.184 -1.722 0.000 1.331 1.019 0.985 1.192 0.677 0.973 0.910
|
|
||||||
0 0.713 0.164 1.080 1.427 -0.460 0.960 -0.152 -0.940 2.173 1.427 -0.901 1.036 1.107 0.440 -1.269 -0.194 0.000 0.452 1.932 -0.532 0.000 1.542 1.210 1.374 1.319 1.818 1.220 1.050
|
|
||||||
0 0.876 -0.463 -1.224 2.458 -1.689 1.007 -0.752 0.398 0.000 2.456 -1.285 -0.152 1.107 1.641 1.838 1.717 0.000 0.458 0.194 0.488 3.102 4.848 2.463 0.986 1.981 0.974 2.642 2.258
|
|
||||||
1 0.384 -0.275 0.387 1.403 -0.994 0.620 -1.529 1.685 0.000 1.091 -1.644 1.078 0.000 0.781 -1.311 0.326 2.548 1.228 -0.728 -0.633 1.551 0.920 0.854 0.987 0.646 0.609 0.740 0.884
|
|
||||||
0 0.318 -1.818 -1.008 0.977 1.268 0.457 2.451 -1.522 0.000 0.881 1.351 0.461 2.215 0.929 0.239 -0.380 2.548 0.382 -0.613 1.330 0.000 1.563 1.193 0.994 0.829 0.874 0.901 1.026
|
|
||||||
1 0.612 -1.120 1.098 0.402 -0.480 0.818 0.188 1.511 0.000 0.800 -0.253 0.977 0.000 1.175 0.271 -1.289 1.274 2.531 0.226 -0.409 3.102 0.889 0.947 0.979 1.486 0.940 1.152 1.119
|
|
||||||
1 0.587 -0.737 -0.228 0.970 1.119 0.823 0.184 1.594 0.000 1.104 0.301 -0.818 2.215 0.819 0.712 -0.560 0.000 2.240 -0.419 0.340 3.102 1.445 1.103 0.988 0.715 1.363 1.019 0.926
|
|
||||||
0 1.030 -0.694 -1.638 0.893 -1.074 1.160 -0.766 0.485 0.000 1.632 -0.698 -1.142 2.215 1.050 -1.092 0.952 0.000 1.475 0.286 0.125 3.102 0.914 1.075 0.982 0.732 1.493 1.219 1.079
|
|
||||||
1 2.142 0.617 1.517 0.387 -0.862 0.345 1.203 -1.014 2.173 0.609 1.092 0.275 0.000 1.331 0.582 -0.183 2.548 0.557 1.540 -1.642 0.000 0.801 0.737 1.060 0.715 0.626 0.749 0.674
|
|
||||||
0 1.076 0.240 -0.246 0.871 -1.241 0.496 0.282 0.746 2.173 1.095 -0.648 1.100 2.215 0.446 -1.756 0.764 0.000 0.434 0.788 -0.991 0.000 1.079 0.868 1.047 0.818 0.634 0.795 0.733
|
|
||||||
0 1.400 0.901 -1.617 0.625 -0.163 0.661 -0.411 -1.616 2.173 0.685 0.524 0.425 0.000 0.881 -0.766 0.312 0.000 0.979 0.255 -0.667 3.102 0.898 1.105 1.253 0.730 0.716 0.738 0.795
|
|
||||||
0 3.302 1.132 1.051 0.658 0.768 1.308 0.251 -0.374 1.087 1.673 0.015 -0.898 0.000 0.688 -0.535 1.363 1.274 0.871 1.325 -1.583 0.000 1.646 1.249 0.995 1.919 1.288 1.330 1.329
|
|
||||||
0 1.757 0.202 0.750 0.767 -0.362 0.932 -1.033 -1.366 0.000 1.529 -1.012 -0.771 0.000 1.161 -0.287 0.059 0.000 2.185 1.147 1.099 3.102 0.795 0.529 1.354 1.144 1.491 1.319 1.161
|
|
||||||
0 1.290 0.905 -1.711 1.017 -0.695 1.008 -1.038 0.693 2.173 1.202 -0.595 0.187 0.000 1.011 0.139 -1.607 0.000 0.789 -0.613 -1.041 3.102 1.304 0.895 1.259 1.866 0.955 1.211 1.200
|
|
||||||
1 1.125 -0.004 1.694 0.373 0.329 0.978 0.640 -0.391 0.000 1.122 -0.376 1.521 2.215 0.432 2.413 -1.259 0.000 0.969 0.730 0.512 3.102 0.716 0.773 0.991 0.624 0.977 0.981 0.875
|
|
||||||
0 1.081 0.861 1.252 1.621 1.474 1.293 0.600 0.630 0.000 1.991 -0.090 -0.675 2.215 0.861 1.105 -0.201 0.000 1.135 2.489 -1.659 0.000 1.089 0.657 0.991 2.179 0.412 1.334 1.071
|
|
||||||
1 0.652 -0.294 1.241 1.034 0.490 1.033 0.551 -0.963 2.173 0.661 1.031 -1.654 2.215 1.376 -0.018 0.843 0.000 0.943 -0.329 -0.269 0.000 1.085 1.067 0.991 1.504 0.773 1.135 0.993
|
|
||||||
1 1.408 -1.028 -1.018 0.252 -0.242 0.465 -0.364 -0.200 0.000 1.466 0.669 0.739 1.107 1.031 0.415 -1.468 2.548 0.457 -1.091 -1.722 0.000 0.771 0.811 0.979 1.459 1.204 1.041 0.866
|
|
||||||
1 0.781 -1.143 -0.659 0.961 1.266 1.183 -0.686 0.119 2.173 1.126 -0.064 1.447 0.000 0.730 1.430 -1.535 0.000 1.601 0.513 1.658 0.000 0.871 1.345 1.184 1.058 0.620 1.107 0.978
|
|
||||||
1 1.300 -0.616 1.032 0.751 -0.731 0.961 -0.716 1.592 0.000 2.079 -1.063 -0.271 2.215 0.475 0.518 1.695 1.274 0.395 -2.204 0.349 0.000 1.350 0.983 1.369 1.265 1.428 1.135 0.982
|
|
||||||
1 0.833 0.809 1.657 1.637 1.019 0.705 1.077 -0.968 2.173 1.261 0.114 -0.298 1.107 1.032 0.017 0.236 0.000 0.640 -0.026 -1.598 0.000 0.894 0.982 0.981 1.250 1.054 1.018 0.853
|
|
||||||
1 1.686 -1.090 -0.301 0.890 0.557 1.304 -0.284 -1.393 2.173 0.388 2.118 0.513 0.000 0.514 -0.015 0.891 0.000 0.460 0.547 0.627 3.102 0.942 0.524 1.186 1.528 0.889 1.015 1.122
|
|
||||||
1 0.551 0.911 0.879 0.379 -0.796 1.154 -0.808 -0.966 0.000 1.168 -0.513 0.355 2.215 0.646 -1.309 0.773 0.000 0.544 -0.283 1.301 3.102 0.847 0.705 0.990 0.772 0.546 0.790 0.719
|
|
||||||
1 1.597 0.793 -1.119 0.691 -1.455 0.370 0.337 1.354 0.000 0.646 -1.005 0.732 2.215 1.019 0.040 0.209 0.000 0.545 0.958 0.239 3.102 0.962 0.793 0.994 0.719 0.745 0.812 0.739
|
|
||||||
0 1.033 -1.193 -0.452 0.247 0.970 0.503 -1.424 1.362 0.000 1.062 -0.416 -1.156 2.215 0.935 -0.023 0.555 2.548 0.410 -1.766 0.379 0.000 0.590 0.953 0.991 0.717 1.081 0.763 0.690
|
|
||||||
1 0.859 -1.004 1.521 0.781 -0.993 0.677 0.643 -0.338 2.173 0.486 0.409 1.283 0.000 0.679 0.110 0.285 0.000 0.715 -0.735 -0.157 1.551 0.702 0.773 0.984 0.627 0.633 0.694 0.643
|
|
||||||
0 0.612 -1.127 1.074 1.225 -0.426 0.927 -2.141 -0.473 0.000 1.290 -0.927 -1.085 2.215 1.183 1.981 -1.687 0.000 2.176 0.406 -1.581 0.000 0.945 0.651 1.170 0.895 1.604 1.179 1.142
|
|
||||||
1 0.535 0.321 -1.095 0.281 -0.960 0.876 -0.709 -0.076 0.000 1.563 -0.666 1.536 2.215 0.773 -0.321 0.435 0.000 0.682 -0.801 -0.952 3.102 0.711 0.667 0.985 0.888 0.741 0.872 0.758
|
|
||||||
1 0.745 1.586 1.578 0.863 -1.423 0.530 1.714 1.085 0.000 1.174 0.679 1.015 0.000 1.158 0.609 -1.186 2.548 1.851 0.832 -0.248 3.102 0.910 1.164 0.983 0.947 0.858 0.928 0.823
|
|
||||||
0 0.677 -1.014 -1.648 1.455 1.461 0.596 -2.358 0.517 0.000 0.800 0.849 -0.743 2.215 1.024 -0.282 -1.004 0.000 1.846 -0.977 0.378 3.102 2.210 1.423 0.982 1.074 1.623 1.417 1.258
|
|
||||||
1 0.815 -1.263 0.057 1.018 -0.208 0.339 -0.347 -1.646 2.173 1.223 0.600 -1.658 2.215 1.435 0.042 0.926 0.000 0.777 1.698 -0.698 0.000 1.022 1.058 1.000 0.784 0.477 0.886 0.836
|
|
||||||
0 3.512 -1.094 -0.220 0.338 -0.328 1.962 -1.099 1.544 1.087 1.461 -1.305 -0.922 2.215 1.219 -1.289 0.400 0.000 0.731 0.155 1.249 0.000 1.173 1.366 0.993 2.259 2.000 1.626 1.349
|
|
||||||
0 0.904 1.248 0.325 0.317 -1.624 0.685 -0.538 1.665 2.173 0.685 -2.145 -1.106 0.000 0.632 -1.460 1.017 0.000 1.085 -0.182 0.162 3.102 0.885 0.801 0.989 0.930 0.904 1.012 0.961
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,500 +0,0 @@
|
|||||||
1 0.644 0.247 -0.447 0.862 0.374 0.854 -1.126 -0.790 2.173 1.015 -0.201 1.400 0.000 1.575 1.807 1.607 0.000 1.585 -0.190 -0.744 3.102 0.958 1.061 0.980 0.875 0.581 0.905 0.796
|
|
||||||
0 0.385 1.800 1.037 1.044 0.349 1.502 -0.966 1.734 0.000 0.966 -1.960 -0.249 0.000 1.501 0.465 -0.354 2.548 0.834 -0.440 0.638 3.102 0.695 0.909 0.981 0.803 0.813 1.149 1.116
|
|
||||||
0 1.214 -0.166 0.004 0.505 1.434 0.628 -1.174 -1.230 1.087 0.579 -1.047 -0.118 0.000 0.835 0.340 1.234 2.548 0.711 -1.383 1.355 0.000 0.848 0.911 1.043 0.931 1.058 0.744 0.696
|
|
||||||
1 0.420 1.111 0.137 1.516 -1.657 0.854 0.623 1.605 1.087 1.511 -1.297 0.251 0.000 0.872 -0.368 -0.721 0.000 0.543 0.731 1.424 3.102 1.597 1.282 1.105 0.730 0.148 1.231 1.234
|
|
||||||
0 0.897 -1.703 -1.306 1.022 -0.729 0.836 0.859 -0.333 2.173 1.336 -0.965 0.972 2.215 0.671 1.021 -1.439 0.000 0.493 -2.019 -0.289 0.000 0.805 0.930 0.984 1.430 2.198 1.934 1.684
|
|
||||||
0 0.756 1.126 -0.945 2.355 -0.555 0.889 0.800 1.440 0.000 0.585 0.271 0.631 2.215 0.722 1.744 1.051 0.000 0.618 0.924 0.698 1.551 0.976 0.864 0.988 0.803 0.234 0.822 0.911
|
|
||||||
0 1.141 -0.741 0.953 1.478 -0.524 1.197 -0.871 1.689 2.173 0.875 1.321 -0.518 1.107 0.540 0.037 -0.987 0.000 0.879 1.187 0.245 0.000 0.888 0.701 1.747 1.358 2.479 1.491 1.223
|
|
||||||
1 0.606 -0.936 -0.384 1.257 -1.162 2.719 -0.600 0.100 2.173 3.303 -0.284 1.561 1.107 0.689 1.786 -0.326 0.000 0.780 -0.532 1.216 0.000 0.936 2.022 0.985 1.574 4.323 2.263 1.742
|
|
||||||
1 0.603 0.429 -0.279 1.448 1.301 1.008 2.423 -1.295 0.000 0.452 1.305 0.533 0.000 1.076 1.011 1.256 2.548 2.021 1.260 -0.343 0.000 0.890 0.969 1.281 0.763 0.652 0.827 0.785
|
|
||||||
0 1.171 -0.962 0.521 0.841 -0.315 1.196 -0.744 -0.882 2.173 0.726 -1.305 1.377 1.107 0.643 -1.790 -1.264 0.000 1.257 0.222 0.817 0.000 0.862 0.911 0.987 0.846 1.293 0.899 0.756
|
|
||||||
1 1.392 -0.358 0.235 1.494 -0.461 0.895 -0.848 1.549 2.173 0.841 -0.384 0.666 1.107 1.199 2.509 -0.891 0.000 1.109 -0.364 -0.945 0.000 0.693 2.135 1.170 1.362 0.959 2.056 1.842
|
|
||||||
1 1.024 1.076 -0.886 0.851 1.530 0.673 -0.449 0.187 1.087 0.628 -0.895 1.176 2.215 0.696 -0.232 -0.875 0.000 0.411 1.501 0.048 0.000 0.842 0.919 1.063 1.193 0.777 0.964 0.807
|
|
||||||
1 0.890 -0.760 1.182 1.369 0.751 0.696 -0.959 -0.710 1.087 0.775 -0.130 -1.409 2.215 0.701 -0.110 -0.739 0.000 0.508 -0.451 0.390 0.000 0.762 0.738 0.998 1.126 0.788 0.940 0.790
|
|
||||||
1 0.460 0.537 0.636 1.442 -0.269 0.585 0.323 -1.731 2.173 0.503 1.034 -0.927 0.000 0.928 -1.024 1.006 2.548 0.513 -0.618 -1.336 0.000 0.802 0.831 0.992 1.019 0.925 1.056 0.833
|
|
||||||
1 0.364 1.648 0.560 1.720 0.829 1.110 0.811 -0.588 0.000 0.408 1.045 1.054 2.215 0.319 -1.138 1.545 0.000 0.423 1.025 -1.265 3.102 1.656 0.928 1.003 0.544 0.327 0.670 0.746
|
|
||||||
1 0.525 -0.096 1.206 0.948 -1.103 1.519 -0.582 0.606 2.173 1.274 -0.572 -0.934 0.000 0.855 -1.028 -1.222 0.000 0.578 -1.000 -1.725 3.102 0.896 0.878 0.981 0.498 0.909 0.772 0.668
|
|
||||||
0 0.536 -0.821 -1.029 0.703 1.113 0.363 -0.711 0.022 1.087 0.325 1.503 1.249 2.215 0.673 1.041 -0.401 0.000 0.480 2.127 1.681 0.000 0.767 1.034 0.990 0.671 0.836 0.669 0.663
|
|
||||||
1 1.789 -0.583 1.641 0.897 0.799 0.515 -0.100 -1.483 0.000 1.101 0.031 -0.326 2.215 1.195 0.001 0.126 2.548 0.768 -0.148 0.601 0.000 0.916 0.921 1.207 1.069 0.483 0.934 0.795
|
|
||||||
1 1.332 -0.571 0.986 0.580 1.508 0.582 0.634 -0.746 1.087 1.084 -0.964 -0.489 0.000 0.785 0.274 0.343 2.548 0.779 0.721 1.489 0.000 1.733 1.145 0.990 1.270 0.715 0.897 0.915
|
|
||||||
0 1.123 0.629 -1.708 0.597 -0.882 0.752 0.195 1.522 2.173 1.671 1.515 -0.003 0.000 0.778 0.514 0.139 1.274 0.801 1.260 1.600 0.000 1.495 0.976 0.988 0.676 0.921 1.010 0.943
|
|
||||||
0 1.816 -0.515 0.171 0.980 -0.454 0.870 0.202 -1.399 2.173 1.130 1.066 -1.593 0.000 0.844 0.735 1.275 2.548 1.125 -1.133 0.348 0.000 0.837 0.693 0.988 1.112 0.784 1.009 0.974
|
|
||||||
1 0.364 0.694 0.445 1.862 0.159 0.963 -1.356 1.260 1.087 0.887 -0.540 -1.533 2.215 0.658 -2.544 -1.236 0.000 0.516 -0.807 0.039 0.000 0.891 1.004 0.991 1.092 0.976 1.000 0.953
|
|
||||||
1 0.790 -1.175 0.475 1.846 0.094 0.999 -1.090 0.257 0.000 1.422 0.854 1.112 2.215 1.302 1.004 -1.702 1.274 2.557 -0.787 -1.048 0.000 0.890 1.429 0.993 2.807 0.840 2.248 1.821
|
|
||||||
1 0.765 -0.500 -0.603 1.843 -0.560 1.068 0.007 0.746 2.173 1.154 -0.017 1.329 0.000 1.165 1.791 -1.585 0.000 1.116 0.441 -0.886 0.000 0.774 0.982 0.989 1.102 0.633 1.178 1.021
|
|
||||||
1 1.407 1.293 -1.418 0.502 -1.527 2.005 -2.122 0.622 0.000 1.699 1.508 -0.649 2.215 1.665 0.748 -0.755 0.000 2.555 0.811 1.423 1.551 7.531 5.520 0.985 1.115 1.881 4.487 3.379
|
|
||||||
1 0.772 -0.186 -1.372 0.823 -0.140 0.781 0.763 0.046 2.173 1.128 0.516 1.380 0.000 0.797 -0.640 -0.134 2.548 2.019 -0.972 -1.670 0.000 2.022 1.466 0.989 0.856 0.808 1.230 0.991
|
|
||||||
1 0.546 -0.954 0.715 1.335 -1.689 0.783 -0.443 -1.735 2.173 1.081 0.185 -0.435 0.000 1.433 -0.662 -0.389 0.000 0.969 0.924 1.099 0.000 0.910 0.879 0.988 0.683 0.753 0.878 0.865
|
|
||||||
1 0.596 0.276 -1.054 1.358 1.355 1.444 1.813 -0.208 0.000 1.175 -0.949 -1.573 0.000 0.855 -1.228 -0.925 2.548 1.837 -0.400 0.913 0.000 0.637 0.901 1.028 0.553 0.790 0.679 0.677
|
|
||||||
0 0.458 2.292 1.530 0.291 1.283 0.749 -0.930 -0.198 0.000 0.300 -1.560 0.990 0.000 0.811 -0.176 0.995 2.548 1.085 -0.178 -1.213 3.102 0.891 0.648 0.999 0.732 0.655 0.619 0.620
|
|
||||||
0 0.638 -0.575 -1.048 0.125 0.178 0.846 -0.753 -0.339 1.087 0.799 -0.727 1.182 0.000 0.888 0.283 0.717 0.000 1.051 -1.046 -1.557 3.102 0.889 0.871 0.989 0.884 0.923 0.836 0.779
|
|
||||||
1 0.434 -1.119 -0.313 2.427 0.461 0.497 0.261 -1.177 2.173 0.618 -0.737 -0.688 0.000 1.150 -1.237 -1.652 2.548 0.757 -0.054 1.700 0.000 0.809 0.741 0.982 1.450 0.936 1.086 0.910
|
|
||||||
1 0.431 -1.144 -1.030 0.778 -0.655 0.490 0.047 -1.546 0.000 1.583 -0.014 0.891 2.215 0.516 0.956 0.567 2.548 0.935 -1.123 -0.082 0.000 0.707 0.995 0.995 0.700 0.602 0.770 0.685
|
|
||||||
1 1.894 0.222 1.224 1.578 1.715 0.966 2.890 -0.013 0.000 0.922 -0.703 -0.844 0.000 0.691 2.056 1.039 0.000 0.900 -0.733 -1.240 3.102 1.292 1.992 1.026 0.881 0.684 1.759 1.755
|
|
||||||
0 0.985 -0.316 0.141 1.067 -0.946 0.819 -1.177 1.307 2.173 1.080 -0.429 0.557 1.107 1.726 1.435 -1.075 0.000 1.100 1.547 -0.647 0.000 0.873 1.696 1.179 1.146 1.015 1.538 1.270
|
|
||||||
0 0.998 -0.187 -0.236 0.882 0.755 0.468 0.950 -0.439 2.173 0.579 -0.550 -0.624 0.000 1.847 1.196 1.384 1.274 0.846 1.273 -1.072 0.000 1.194 0.797 1.013 1.319 1.174 0.963 0.898
|
|
||||||
0 0.515 0.246 -0.593 1.082 1.591 0.912 -0.623 -0.957 2.173 0.858 0.418 0.844 0.000 0.948 2.519 1.599 0.000 1.158 1.385 -0.095 3.102 0.973 1.033 0.988 0.998 1.716 1.054 0.901
|
|
||||||
0 0.919 -1.001 1.506 1.389 0.653 0.507 -0.616 -0.689 2.173 0.808 0.536 -0.467 2.215 0.496 2.187 -0.859 0.000 0.822 0.807 1.163 0.000 0.876 0.861 1.088 0.947 0.614 0.911 1.087
|
|
||||||
0 0.794 0.051 1.477 1.504 -1.695 0.716 0.315 0.264 1.087 0.879 -0.135 -1.094 2.215 1.433 -0.741 0.201 0.000 1.566 0.534 -0.989 0.000 0.627 0.882 0.974 0.807 1.130 0.929 0.925
|
|
||||||
1 0.455 -0.946 -1.175 1.453 -0.580 0.763 -0.856 0.840 0.000 0.829 1.223 1.174 2.215 0.714 0.638 -0.466 0.000 1.182 0.223 -1.333 0.000 0.977 0.938 0.986 0.713 0.714 0.796 0.843
|
|
||||||
1 0.662 -0.296 -1.287 1.212 -0.707 0.641 1.457 0.222 0.000 0.600 0.525 -1.700 2.215 0.784 -0.835 -0.961 2.548 0.865 1.131 1.162 0.000 0.854 0.877 0.978 0.740 0.734 0.888 0.811
|
|
||||||
0 0.390 0.698 -1.629 1.888 0.298 0.990 1.614 -1.572 0.000 1.666 0.170 0.719 2.215 1.590 1.064 -0.886 1.274 0.952 0.305 -1.216 0.000 1.048 0.897 1.173 0.891 1.936 1.273 1.102
|
|
||||||
0 1.014 0.117 1.384 0.686 -1.047 0.609 -1.245 -0.850 0.000 1.076 -1.158 0.814 1.107 1.598 -0.389 -0.111 0.000 0.907 1.688 -1.673 0.000 1.333 0.866 0.989 0.975 0.442 0.797 0.788
|
|
||||||
0 1.530 -1.408 -0.207 0.440 -1.357 0.902 -0.647 1.325 1.087 1.320 -0.819 0.246 1.107 0.503 1.407 -1.683 0.000 1.189 -0.972 -0.925 0.000 0.386 1.273 0.988 0.829 1.335 1.173 1.149
|
|
||||||
1 1.689 -0.590 0.915 2.076 1.202 0.644 -0.478 -0.238 0.000 0.809 -1.660 -1.184 0.000 1.227 -0.224 -0.808 2.548 1.655 1.047 -0.623 0.000 0.621 1.192 0.988 1.309 0.866 0.924 1.012
|
|
||||||
0 1.102 0.402 -1.622 1.262 1.022 0.576 0.271 -0.269 0.000 0.591 0.495 -1.278 0.000 1.271 0.209 0.575 2.548 0.941 0.964 -0.685 3.102 0.989 0.963 1.124 0.857 0.858 0.716 0.718
|
|
||||||
0 2.491 0.825 0.581 1.593 0.205 0.782 -0.815 1.499 0.000 1.179 -0.999 -1.509 0.000 0.926 0.920 -0.522 2.548 2.068 -1.021 -1.050 3.102 0.874 0.943 0.980 0.945 1.525 1.570 1.652
|
|
||||||
0 0.666 0.254 1.601 1.303 -0.250 1.236 -1.929 0.793 0.000 1.074 0.447 -0.871 0.000 0.991 1.059 -0.342 0.000 1.703 -0.393 -1.419 3.102 0.921 0.945 1.285 0.931 0.462 0.770 0.729
|
|
||||||
0 0.937 -1.126 1.424 1.395 1.743 0.760 0.428 -0.238 2.173 0.846 0.494 1.320 2.215 0.872 -1.826 -0.507 0.000 0.612 1.860 1.403 0.000 3.402 2.109 0.985 1.298 1.165 1.404 1.240
|
|
||||||
1 0.881 -1.086 -0.870 0.513 0.266 2.049 -1.870 1.160 0.000 2.259 -0.428 -0.935 2.215 1.321 -0.655 -0.449 2.548 1.350 -1.766 -0.108 0.000 0.911 1.852 0.987 1.167 0.820 1.903 1.443
|
|
||||||
0 0.410 0.835 -0.819 1.257 1.112 0.871 -1.737 -0.401 0.000 0.927 0.158 1.253 0.000 1.183 0.405 -1.570 0.000 0.807 -0.704 -0.438 3.102 0.932 0.962 0.987 0.653 0.315 0.616 0.648
|
|
||||||
1 0.634 0.196 -1.679 1.379 -0.967 2.260 -0.273 1.114 0.000 1.458 1.070 -0.278 1.107 1.195 0.110 -0.688 2.548 0.907 0.298 -1.359 0.000 0.949 1.129 0.984 0.675 0.877 0.938 0.824
|
|
||||||
1 0.632 -1.254 1.201 0.496 -0.106 0.235 2.731 -0.955 0.000 0.615 -0.805 0.600 0.000 0.633 -0.934 1.641 0.000 1.407 -0.483 -0.962 1.551 0.778 0.797 0.989 0.578 0.722 0.576 0.539
|
|
||||||
0 0.714 1.122 1.566 2.399 -1.431 1.665 0.299 0.323 0.000 1.489 1.087 -0.861 2.215 1.174 0.140 1.083 2.548 0.404 -0.968 1.105 0.000 0.867 0.969 0.981 1.039 1.552 1.157 1.173
|
|
||||||
1 0.477 -0.321 -0.471 1.966 1.034 2.282 1.359 -0.874 0.000 1.672 -0.258 1.109 0.000 1.537 0.604 0.231 2.548 1.534 -0.640 0.827 0.000 0.746 1.337 1.311 0.653 0.721 0.795 0.742
|
|
||||||
1 1.351 0.460 0.031 1.194 -1.185 0.670 -1.157 -1.637 2.173 0.599 -0.823 0.680 0.000 0.478 0.373 1.716 0.000 0.809 -0.919 0.010 1.551 0.859 0.839 1.564 0.994 0.777 0.971 0.826
|
|
||||||
1 0.520 -1.442 -0.348 0.840 1.654 1.273 -0.760 1.317 0.000 0.861 2.579 -0.791 0.000 1.779 0.257 -0.703 0.000 2.154 1.928 0.457 0.000 1.629 3.194 0.992 0.730 1.107 2.447 2.747
|
|
||||||
0 0.700 -0.308 0.920 0.438 -0.879 0.516 1.409 1.101 0.000 0.960 0.701 -0.049 2.215 1.442 -0.416 -1.439 2.548 0.628 1.009 -0.364 0.000 0.848 0.817 0.987 0.759 1.421 0.937 0.920
|
|
||||||
1 0.720 1.061 -0.546 0.798 -1.521 1.066 0.173 0.271 1.087 1.453 0.114 1.336 1.107 0.702 0.616 -0.367 0.000 0.543 -0.386 -1.301 0.000 0.653 0.948 0.989 1.031 1.500 0.965 0.790
|
|
||||||
1 0.735 -0.416 0.588 1.308 -0.382 1.042 0.344 1.609 0.000 0.926 0.163 -0.520 1.107 1.050 -0.427 1.159 2.548 0.834 0.613 0.948 0.000 0.848 1.189 1.042 0.844 1.099 0.829 0.843
|
|
||||||
1 0.777 -0.396 1.540 1.608 0.638 0.955 0.040 0.918 2.173 1.315 1.116 -0.823 0.000 0.781 -0.762 0.564 2.548 0.945 -0.573 1.379 0.000 0.679 0.706 1.124 0.608 0.593 0.515 0.493
|
|
||||||
1 0.934 0.319 -0.257 0.970 -0.980 0.726 0.774 0.731 0.000 0.896 0.038 -1.465 1.107 0.773 -0.055 -0.831 2.548 1.439 -0.229 0.698 0.000 0.964 1.031 0.995 0.845 0.480 0.810 0.762
|
|
||||||
0 0.461 0.771 0.019 2.055 -1.288 1.043 0.147 0.261 2.173 0.833 -0.156 1.425 0.000 0.832 0.805 -0.491 2.548 0.589 1.252 1.414 0.000 0.850 0.906 1.245 1.364 0.850 0.908 0.863
|
|
||||||
1 0.858 -0.116 -0.937 0.966 1.167 0.825 -0.108 1.111 1.087 0.733 1.163 -0.634 0.000 0.894 0.771 0.020 0.000 0.846 -1.124 -1.195 3.102 0.724 1.194 1.195 0.813 0.969 0.985 0.856
|
|
||||||
0 0.720 -0.335 -0.307 1.445 0.540 1.108 -0.034 -1.691 1.087 0.883 -1.356 -0.678 2.215 0.440 1.093 0.253 0.000 0.389 -1.582 -1.097 0.000 1.113 1.034 0.988 1.256 1.572 1.062 0.904
|
|
||||||
1 0.750 -0.811 -0.542 0.985 0.408 0.471 0.477 0.355 0.000 1.347 -0.875 -1.556 2.215 0.564 1.082 -0.724 0.000 0.793 -0.958 -0.020 3.102 0.836 0.825 0.986 1.066 0.924 0.927 0.883
|
|
||||||
0 0.392 -0.468 -0.216 0.680 1.565 1.086 -0.765 -0.581 1.087 1.264 -1.035 1.189 2.215 0.986 -0.338 0.747 0.000 0.884 -1.328 -0.965 0.000 1.228 0.988 0.982 1.135 1.741 1.108 0.956
|
|
||||||
1 0.434 -1.269 0.643 0.713 0.608 0.597 0.832 1.627 0.000 0.708 -0.422 0.079 2.215 1.533 -0.823 -1.127 2.548 0.408 -1.357 -0.828 0.000 1.331 1.087 0.999 1.075 1.015 0.875 0.809
|
|
||||||
0 0.828 -1.803 0.342 0.847 -0.162 1.585 -1.128 -0.272 2.173 1.974 0.039 -1.717 0.000 0.900 0.764 -1.741 0.000 1.349 -0.079 1.035 3.102 0.984 0.815 0.985 0.780 1.661 1.403 1.184
|
|
||||||
1 1.089 -0.350 -0.747 1.472 0.792 1.087 -0.069 -1.192 0.000 0.512 -0.841 -1.284 0.000 2.162 -0.821 0.545 2.548 1.360 2.243 -0.183 0.000 0.977 0.628 1.725 1.168 0.635 0.823 0.822
|
|
||||||
1 0.444 0.451 -1.332 1.176 -0.247 0.898 0.194 0.007 0.000 1.958 0.576 -1.618 2.215 0.584 1.203 0.268 0.000 0.939 1.033 1.264 3.102 0.829 0.886 0.985 1.265 0.751 1.032 0.948
|
|
||||||
0 0.629 0.114 1.177 0.917 -1.204 0.845 0.828 -0.088 0.000 0.962 -1.302 0.823 2.215 0.732 0.358 -1.334 2.548 0.538 0.582 1.561 0.000 1.028 0.834 0.988 0.904 1.205 1.039 0.885
|
|
||||||
1 1.754 -1.259 -0.573 0.959 -1.483 0.358 0.448 -1.452 0.000 0.711 0.313 0.499 2.215 1.482 -0.390 1.474 2.548 1.879 -1.540 0.668 0.000 0.843 0.825 1.313 1.315 0.939 1.048 0.871
|
|
||||||
1 0.549 0.706 -1.437 0.894 0.891 0.680 -0.762 -1.568 0.000 0.981 0.499 -0.425 2.215 1.332 0.678 0.485 1.274 0.803 0.022 -0.893 0.000 0.793 1.043 0.987 0.761 0.899 0.915 0.794
|
|
||||||
0 0.475 0.542 -0.987 1.569 0.069 0.551 1.543 -1.488 0.000 0.608 0.301 1.734 2.215 0.277 0.499 -0.522 0.000 1.375 1.212 0.696 3.102 0.652 0.756 0.987 0.828 0.830 0.715 0.679
|
|
||||||
1 0.723 0.049 -1.153 1.300 0.083 0.723 -0.749 0.630 0.000 1.126 0.412 -0.384 0.000 1.272 1.256 1.358 2.548 3.108 0.777 -1.486 3.102 0.733 1.096 1.206 1.269 0.899 1.015 0.903
|
|
||||||
1 1.062 0.296 0.725 0.285 -0.531 0.819 1.277 -0.667 0.000 0.687 0.829 -0.092 0.000 1.158 0.447 1.047 2.548 1.444 -0.186 -1.491 3.102 0.863 1.171 0.986 0.769 0.828 0.919 0.840
|
|
||||||
0 0.572 -0.349 1.396 2.023 0.795 0.577 0.457 -0.533 0.000 1.351 0.701 -1.091 0.000 0.724 -1.012 -0.182 2.548 0.923 -0.012 0.789 3.102 0.936 1.025 0.985 1.002 0.600 0.828 0.909
|
|
||||||
1 0.563 0.387 0.412 0.553 1.050 0.723 -0.992 -0.447 0.000 0.748 0.948 0.546 2.215 1.761 -0.559 -1.183 0.000 1.114 -0.251 1.192 3.102 0.936 0.912 0.976 0.578 0.722 0.829 0.892
|
|
||||||
1 1.632 1.577 -0.697 0.708 -1.263 0.863 0.012 1.197 2.173 0.498 0.990 -0.806 0.000 0.627 2.387 -1.283 0.000 0.607 1.290 -0.174 3.102 0.916 1.328 0.986 0.557 0.971 0.935 0.836
|
|
||||||
1 0.562 -0.360 0.399 0.803 -1.334 1.443 -0.116 1.628 2.173 0.750 0.987 0.135 1.107 0.795 0.298 -0.556 0.000 1.150 -0.113 -0.093 0.000 0.493 1.332 0.985 1.001 1.750 1.013 0.886
|
|
||||||
1 0.987 0.706 -0.492 0.861 0.607 0.593 0.088 -0.184 0.000 0.802 0.894 1.608 2.215 0.782 -0.471 1.500 2.548 0.521 0.772 -0.960 0.000 0.658 0.893 1.068 0.877 0.664 0.709 0.661
|
|
||||||
1 1.052 0.883 -0.581 1.566 0.860 0.931 1.515 -0.873 0.000 0.493 0.145 -0.672 0.000 1.133 0.935 1.581 2.548 1.630 0.695 0.923 3.102 1.105 1.087 1.713 0.948 0.590 0.872 0.883
|
|
||||||
1 2.130 -0.516 -0.291 0.776 -1.230 0.689 -0.257 0.800 2.173 0.730 -0.274 -1.437 0.000 0.615 0.241 1.083 0.000 0.834 0.757 1.613 3.102 0.836 0.806 1.333 1.061 0.730 0.889 0.783
|
|
||||||
1 0.742 0.797 1.628 0.311 -0.418 0.620 0.685 -1.457 0.000 0.683 1.774 -1.082 0.000 1.700 1.104 0.225 2.548 0.382 -2.184 -1.307 0.000 0.945 1.228 0.984 0.864 0.931 0.988 0.838
|
|
||||||
0 0.311 -1.249 -0.927 1.272 -1.262 0.642 -1.228 -0.136 0.000 1.220 -0.804 -1.558 2.215 0.950 -0.828 0.495 1.274 2.149 -1.672 0.634 0.000 1.346 0.887 0.981 0.856 1.101 1.001 1.106
|
|
||||||
0 0.660 -1.834 -0.667 0.601 1.236 0.932 -0.933 -0.135 2.173 1.373 -0.122 1.429 0.000 0.654 -0.034 -0.847 2.548 0.711 0.911 0.703 0.000 1.144 0.942 0.984 0.822 0.739 0.992 0.895
|
|
||||||
0 3.609 -0.590 0.851 0.615 0.455 1.280 0.003 -0.866 1.087 1.334 0.708 -1.131 0.000 0.669 0.480 0.092 0.000 0.975 0.983 -1.429 3.102 1.301 1.089 0.987 1.476 0.934 1.469 1.352
|
|
||||||
1 0.905 -0.403 1.567 2.651 0.953 1.194 -0.241 -0.567 1.087 0.308 -0.384 -0.007 0.000 0.608 -0.175 -1.163 2.548 0.379 0.941 1.662 0.000 0.580 0.721 1.126 0.895 0.544 1.097 0.836
|
|
||||||
1 0.983 0.255 1.093 0.905 -0.874 0.863 0.060 -0.368 0.000 0.824 -0.747 -0.633 0.000 0.614 0.961 1.052 0.000 0.792 -0.260 1.632 3.102 0.874 0.883 1.280 0.663 0.406 0.592 0.645
|
|
||||||
1 1.160 -1.027 0.274 0.460 0.322 2.085 -1.623 -0.840 0.000 1.634 -1.046 1.182 2.215 0.492 -0.367 1.174 0.000 0.824 -0.998 1.617 0.000 0.943 0.884 1.001 1.209 1.313 1.034 0.866
|
|
||||||
0 0.299 0.028 -1.372 1.930 -0.661 0.840 -0.979 0.664 1.087 0.535 -2.041 1.434 0.000 1.087 -1.797 0.344 0.000 0.485 -0.560 -1.105 3.102 0.951 0.890 0.980 0.483 0.684 0.730 0.706
|
|
||||||
0 0.293 1.737 -1.418 2.074 0.794 0.679 1.024 -1.457 0.000 1.034 1.094 -0.168 1.107 0.506 1.680 -0.661 0.000 0.523 -0.042 -1.274 3.102 0.820 0.944 0.987 0.842 0.694 0.761 0.750
|
|
||||||
0 0.457 -0.393 1.560 0.738 -0.007 0.475 -0.230 0.246 0.000 0.776 -1.264 -0.606 2.215 0.865 -0.731 -1.576 2.548 1.153 0.343 1.436 0.000 1.060 0.883 0.988 0.972 0.703 0.758 0.720
|
|
||||||
0 0.935 -0.582 0.240 2.401 0.818 1.231 -0.618 -1.289 0.000 0.799 0.544 -0.228 2.215 0.525 -1.494 -0.969 0.000 0.609 -1.123 1.168 3.102 0.871 0.767 1.035 1.154 0.919 0.868 1.006
|
|
||||||
1 0.902 -0.745 -1.215 1.174 -0.501 1.215 0.167 1.162 0.000 0.896 1.217 -0.976 0.000 0.585 -0.429 1.036 0.000 1.431 -0.416 0.151 3.102 0.524 0.952 0.990 0.707 0.271 0.592 0.826
|
|
||||||
1 0.653 0.337 -0.320 1.118 -0.934 1.050 0.745 0.529 1.087 1.075 1.742 -1.538 0.000 0.585 1.090 0.973 0.000 1.091 -0.187 1.160 1.551 1.006 1.108 0.978 1.121 0.838 0.947 0.908
|
|
||||||
0 1.157 1.401 0.340 0.395 -1.218 0.945 1.928 -0.876 0.000 1.384 0.320 1.002 1.107 1.900 1.177 -0.462 2.548 1.122 1.316 1.720 0.000 1.167 1.096 0.989 0.937 1.879 1.307 1.041
|
|
||||||
0 0.960 0.355 -0.152 0.872 -0.338 0.391 0.348 0.956 1.087 0.469 2.664 1.409 0.000 0.756 -1.561 1.500 0.000 0.525 1.436 1.728 3.102 1.032 0.946 0.996 0.929 0.470 0.698 0.898
|
|
||||||
1 1.038 0.274 0.825 1.198 0.963 1.078 -0.496 -1.014 2.173 0.739 -0.727 -0.151 2.215 1.035 -0.799 0.398 0.000 1.333 -0.872 -1.498 0.000 0.849 1.033 0.985 0.886 0.936 0.975 0.823
|
|
||||||
0 0.490 0.277 0.318 1.303 0.694 1.333 -1.620 -0.563 0.000 1.459 -1.326 1.140 0.000 0.779 -0.673 -1.324 2.548 0.860 -1.247 0.043 0.000 0.857 0.932 0.992 0.792 0.278 0.841 1.498
|
|
||||||
0 1.648 -0.688 -1.386 2.790 0.995 1.087 1.359 -0.687 0.000 1.050 -0.223 -0.261 2.215 0.613 -0.889 1.335 0.000 1.204 0.827 0.309 3.102 0.464 0.973 2.493 1.737 0.827 1.319 1.062
|
|
||||||
0 1.510 -0.662 1.668 0.860 0.280 0.705 0.974 -1.647 1.087 0.662 -0.393 -0.225 0.000 0.610 -0.996 0.532 2.548 0.464 1.305 0.102 0.000 0.859 1.057 1.498 0.799 1.260 0.946 0.863
|
|
||||||
1 0.850 -1.185 -0.117 0.943 -0.449 1.142 0.875 -0.030 0.000 2.223 -0.461 1.627 2.215 0.767 -1.761 -1.692 0.000 1.012 -0.727 0.639 3.102 3.649 2.062 0.985 1.478 1.087 1.659 1.358
|
|
||||||
0 0.933 1.259 0.130 0.326 -0.890 0.306 1.136 1.142 0.000 0.964 0.705 -1.373 2.215 0.546 -0.196 -0.001 0.000 0.578 -1.169 1.004 3.102 0.830 0.836 0.988 0.837 1.031 0.749 0.655
|
|
||||||
0 0.471 0.697 1.570 1.109 0.201 1.248 0.348 -1.448 0.000 2.103 0.773 0.686 2.215 1.451 -0.087 -0.453 2.548 1.197 -0.045 -1.026 0.000 0.793 1.094 0.987 0.851 1.804 1.378 1.089
|
|
||||||
1 2.446 -0.701 -1.568 0.059 0.822 1.401 -0.600 -0.044 2.173 0.324 -0.001 1.344 2.215 0.913 -0.818 1.049 0.000 0.442 -1.088 -0.005 0.000 0.611 1.062 0.979 0.562 0.988 0.998 0.806
|
|
||||||
0 0.619 2.029 0.933 0.528 -0.903 0.974 0.760 -0.311 2.173 0.825 0.658 -1.466 1.107 0.894 1.594 0.370 0.000 0.882 -0.258 1.661 0.000 1.498 1.088 0.987 0.867 1.139 0.900 0.779
|
|
||||||
1 0.674 -0.131 -0.362 0.518 -1.574 0.876 0.442 0.145 1.087 0.497 -1.526 -1.704 0.000 0.680 2.514 -1.374 0.000 0.792 -0.479 0.773 1.551 0.573 1.198 0.984 0.800 0.667 0.987 0.832
|
|
||||||
1 1.447 1.145 -0.937 0.307 -1.458 0.478 1.264 0.816 1.087 0.558 1.015 -0.101 2.215 0.937 -0.190 1.177 0.000 0.699 0.954 -1.512 0.000 0.877 0.838 0.990 0.873 0.566 0.646 0.713
|
|
||||||
1 0.976 0.308 -0.844 0.436 0.610 1.253 0.149 -1.585 2.173 1.415 0.568 0.096 2.215 0.953 -0.855 0.441 0.000 0.867 -0.650 1.643 0.000 0.890 1.234 0.988 0.796 2.002 1.179 0.977
|
|
||||||
0 0.697 0.401 -0.718 0.920 0.735 0.958 -0.172 0.168 2.173 0.872 -0.097 -1.335 0.000 0.513 -1.192 -1.710 1.274 0.426 -1.637 1.368 0.000 0.997 1.227 1.072 0.800 1.013 0.786 0.749
|
|
||||||
1 1.305 -2.157 1.740 0.661 -0.912 0.705 -0.516 0.759 2.173 0.989 -0.716 -0.300 2.215 0.627 -1.052 -1.736 0.000 0.467 -2.467 0.568 0.000 0.807 0.964 0.988 1.427 1.012 1.165 0.926
|
|
||||||
0 1.847 1.663 -0.618 0.280 1.258 1.462 -0.054 1.371 0.000 0.900 0.309 -0.544 0.000 0.331 -2.149 -0.341 0.000 1.091 -0.833 0.710 3.102 1.496 0.931 0.989 1.549 0.115 1.140 1.150
|
|
||||||
0 0.410 -0.323 1.069 2.160 0.010 0.892 0.942 -1.640 2.173 0.946 0.938 1.314 0.000 1.213 -1.099 -0.794 2.548 0.650 0.053 0.056 0.000 1.041 0.916 1.063 0.985 1.910 1.246 1.107
|
|
||||||
1 0.576 1.092 -0.088 0.777 -1.579 0.757 0.271 0.109 0.000 0.819 0.827 -1.554 2.215 1.313 2.341 -1.568 0.000 2.827 0.239 -0.338 0.000 0.876 0.759 0.986 0.692 0.457 0.796 0.791
|
|
||||||
1 0.537 0.925 -1.406 0.306 -0.050 0.906 1.051 0.037 0.000 1.469 -0.177 -1.320 2.215 1.872 0.723 1.158 0.000 1.313 0.227 -0.501 3.102 0.953 0.727 0.978 0.755 0.892 0.932 0.781
|
|
||||||
0 0.716 -0.065 -0.484 1.313 -1.563 0.596 -0.242 0.678 2.173 0.426 -1.909 0.616 0.000 0.885 -0.406 -1.343 2.548 0.501 -1.327 -0.340 0.000 0.470 0.728 1.109 0.919 0.881 0.665 0.692
|
|
||||||
1 0.624 -0.389 0.128 1.636 -1.110 1.025 0.573 -0.843 2.173 0.646 -0.697 1.064 0.000 0.632 -1.442 0.961 0.000 0.863 -0.106 1.717 0.000 0.825 0.917 1.257 0.983 0.713 0.890 0.824
|
|
||||||
0 0.484 2.101 1.714 1.131 -0.823 0.750 0.583 -1.304 1.087 0.894 0.421 0.559 2.215 0.921 -0.063 0.282 0.000 0.463 -0.474 -1.387 0.000 0.742 0.886 0.995 0.993 1.201 0.806 0.754
|
|
||||||
0 0.570 0.339 -1.478 0.528 0.439 0.978 1.479 -1.411 2.173 0.763 1.541 -0.734 0.000 1.375 0.840 0.903 0.000 0.965 1.599 0.364 0.000 0.887 1.061 0.992 1.322 1.453 1.013 0.969
|
|
||||||
0 0.940 1.303 1.636 0.851 -1.732 0.803 -0.030 -0.177 0.000 0.480 -0.125 -0.954 0.000 0.944 0.709 0.296 2.548 1.342 -0.418 1.197 3.102 0.853 0.989 0.979 0.873 0.858 0.719 0.786
|
|
||||||
1 0.599 0.544 -0.238 0.816 1.043 0.857 0.660 1.128 2.173 0.864 -0.624 -0.843 0.000 1.159 0.367 0.174 0.000 1.520 -0.543 -1.508 0.000 0.842 0.828 0.984 0.759 0.895 0.918 0.791
|
|
||||||
1 1.651 1.897 -0.914 0.423 0.315 0.453 0.619 -1.607 2.173 0.532 -0.424 0.209 1.107 0.369 2.479 0.034 0.000 0.701 0.217 0.984 0.000 0.976 0.951 1.035 0.879 0.825 0.915 0.798
|
|
||||||
1 0.926 -0.574 -0.763 0.285 1.094 0.672 2.314 1.545 0.000 1.124 0.415 0.809 0.000 1.387 0.270 -0.949 2.548 1.547 -0.631 -0.200 3.102 0.719 0.920 0.986 0.889 0.933 0.797 0.777
|
|
||||||
0 0.677 1.698 -0.890 0.641 -0.449 0.607 1.754 1.720 0.000 0.776 0.372 0.782 2.215 0.511 1.491 -0.480 0.000 0.547 -0.341 0.853 3.102 0.919 1.026 0.997 0.696 0.242 0.694 0.687
|
|
||||||
0 1.266 0.602 0.958 0.487 1.256 0.709 0.843 -1.196 0.000 0.893 1.303 -0.594 1.107 1.090 1.320 0.354 0.000 0.797 1.846 1.139 0.000 0.780 0.896 0.986 0.661 0.709 0.790 0.806
|
|
||||||
1 0.628 -0.616 -0.329 0.764 -1.150 0.477 -0.715 1.187 2.173 1.250 0.607 1.026 2.215 0.983 -0.023 -0.583 0.000 0.377 1.344 -1.015 0.000 0.744 0.954 0.987 0.837 0.841 0.795 0.694
|
|
||||||
1 1.035 -0.828 -1.358 1.870 -1.060 1.075 0.130 0.448 2.173 0.660 0.697 0.641 0.000 0.425 1.006 -1.035 0.000 0.751 1.055 1.364 3.102 0.826 0.822 0.988 0.967 0.901 1.077 0.906
|
|
||||||
1 0.830 0.265 -0.150 0.660 1.105 0.592 -0.557 0.908 2.173 0.670 -1.419 -0.671 0.000 1.323 -0.409 1.644 2.548 0.850 -0.033 -0.615 0.000 0.760 0.967 0.984 0.895 0.681 0.747 0.770
|
|
||||||
1 1.395 1.100 1.167 1.088 0.218 0.400 -0.132 0.024 2.173 0.743 0.530 -1.361 2.215 0.341 -0.691 -0.238 0.000 0.396 -1.426 -0.933 0.000 0.363 0.472 1.287 0.922 0.810 0.792 0.656
|
|
||||||
1 1.070 1.875 -1.298 1.215 -0.106 0.767 0.795 0.514 1.087 0.401 2.780 1.276 0.000 0.686 1.127 1.721 2.548 0.391 -0.259 -1.167 0.000 1.278 1.113 1.389 0.852 0.824 0.838 0.785
|
|
||||||
0 1.114 -0.071 1.719 0.399 -1.383 0.849 0.254 0.481 0.000 0.958 -0.579 0.742 0.000 1.190 -0.140 -0.862 2.548 0.479 1.390 0.856 0.000 0.952 0.988 0.985 0.764 0.419 0.835 0.827
|
|
||||||
0 0.714 0.376 -0.568 1.578 -1.165 0.648 0.141 0.639 2.173 0.472 0.569 1.449 1.107 0.783 1.483 0.361 0.000 0.540 -0.790 0.032 0.000 0.883 0.811 0.982 0.775 0.572 0.760 0.745
|
|
||||||
0 0.401 -1.731 0.765 0.974 1.648 0.652 -1.024 0.191 0.000 0.544 -0.366 -1.246 2.215 0.627 0.140 1.008 2.548 0.810 0.409 0.429 0.000 0.950 0.934 0.977 0.621 0.580 0.677 0.650
|
|
||||||
1 0.391 1.679 -1.298 0.605 -0.832 0.549 1.338 0.522 2.173 1.244 0.884 1.070 0.000 1.002 0.846 -1.345 2.548 0.783 -2.464 -0.237 0.000 4.515 2.854 0.981 0.877 0.939 1.942 1.489
|
|
||||||
1 0.513 -0.220 -0.444 1.699 0.479 1.109 0.181 -0.999 2.173 0.883 -0.335 -1.716 2.215 1.075 -0.380 1.352 0.000 0.857 0.048 0.147 0.000 0.937 0.758 0.986 1.206 0.958 0.949 0.876
|
|
||||||
0 1.367 -0.388 0.798 1.158 1.078 0.811 -1.024 -1.628 0.000 1.504 0.097 -0.999 2.215 1.652 -0.860 0.054 2.548 0.573 -0.142 -1.401 0.000 0.869 0.833 1.006 1.412 1.641 1.214 1.041
|
|
||||||
1 1.545 -0.533 -1.517 1.177 1.289 2.331 -0.370 -0.073 0.000 1.295 -0.358 -0.891 2.215 0.476 0.756 0.985 0.000 1.945 -0.016 -1.651 3.102 1.962 1.692 1.073 0.656 0.941 1.312 1.242
|
|
||||||
0 0.858 0.978 -1.258 0.286 0.161 0.729 1.230 1.087 2.173 0.561 2.670 -0.109 0.000 0.407 2.346 0.938 0.000 1.078 0.729 -0.658 3.102 0.597 0.921 0.982 0.579 0.954 0.733 0.769
|
|
||||||
1 1.454 -1.384 0.870 0.067 0.394 1.033 -0.673 0.318 0.000 1.166 -0.763 -1.533 2.215 2.848 -0.045 -0.856 2.548 0.697 -0.140 1.134 0.000 0.931 1.293 0.977 1.541 1.326 1.201 1.078
|
|
||||||
1 0.559 -0.913 0.486 1.104 -0.321 1.073 -0.348 1.345 0.000 0.901 -0.827 -0.842 0.000 0.739 0.047 -0.415 2.548 0.433 -1.132 1.268 0.000 0.797 0.695 0.985 0.868 0.346 0.674 0.623
|
|
||||||
1 1.333 0.780 -0.964 0.916 1.202 1.822 -0.071 0.742 2.173 1.486 -0.399 -0.824 0.000 0.740 0.568 -0.134 0.000 0.971 -0.070 -1.589 3.102 1.278 0.929 1.421 1.608 1.214 1.215 1.137
|
|
||||||
1 2.417 0.631 -0.317 0.323 0.581 0.841 1.524 -1.738 0.000 0.543 1.176 -0.325 0.000 0.827 0.700 0.866 0.000 0.834 -0.262 -1.702 3.102 0.932 0.820 0.988 0.646 0.287 0.595 0.589
|
|
||||||
0 0.955 -1.242 0.938 1.104 0.474 0.798 -0.743 1.535 0.000 1.356 -1.357 -1.080 2.215 1.320 -1.396 -0.132 2.548 0.728 -0.529 -0.633 0.000 0.832 0.841 0.988 0.923 1.077 0.988 0.816
|
|
||||||
1 1.305 -1.918 0.391 1.161 0.063 0.724 2.593 1.481 0.000 0.592 -1.207 -0.329 0.000 0.886 -0.836 -1.168 2.548 1.067 -1.481 -1.440 0.000 0.916 0.688 0.991 0.969 0.550 0.665 0.638
|
|
||||||
0 1.201 0.071 -1.123 2.242 -1.533 0.702 -0.256 0.688 0.000 0.967 0.491 1.040 2.215 1.271 -0.558 0.095 0.000 1.504 0.676 -0.383 3.102 0.917 1.006 0.985 1.017 1.057 0.928 1.057
|
|
||||||
0 0.994 -1.607 1.596 0.774 -1.391 0.625 -0.134 -0.862 2.173 0.746 -0.765 -0.316 2.215 1.131 -0.320 0.869 0.000 0.607 0.826 0.301 0.000 0.798 0.967 0.999 0.880 0.581 0.712 0.774
|
|
||||||
1 0.482 -0.467 0.729 1.419 1.458 0.824 0.376 -0.242 0.000 1.368 0.023 1.459 2.215 0.826 0.669 -1.079 2.548 0.936 2.215 -0.309 0.000 1.883 1.216 0.997 1.065 0.946 1.224 1.526
|
|
||||||
1 0.383 1.588 1.611 0.748 1.194 0.866 -0.279 -0.636 0.000 0.707 0.536 0.801 2.215 1.647 -1.155 0.367 0.000 1.292 0.303 -1.681 3.102 2.016 1.581 0.986 0.584 0.684 1.107 0.958
|
|
||||||
0 0.629 0.203 0.736 0.671 -0.271 1.350 -0.486 0.761 2.173 0.496 -0.805 -1.718 0.000 2.393 0.044 -1.046 1.274 0.651 -0.116 -0.541 0.000 0.697 1.006 0.987 1.069 2.317 1.152 0.902
|
|
||||||
0 0.905 -0.564 -0.570 0.263 1.096 1.219 -1.397 -1.414 1.087 1.164 -0.533 -0.208 0.000 1.459 1.965 0.784 0.000 2.220 -1.421 0.452 0.000 0.918 1.360 0.993 0.904 0.389 2.118 1.707
|
|
||||||
1 1.676 1.804 1.171 0.529 1.175 1.664 0.354 -0.530 0.000 1.004 0.691 -1.280 2.215 0.838 0.373 0.626 2.548 1.094 1.774 0.501 0.000 0.806 1.100 0.991 0.769 0.976 0.807 0.740
|
|
||||||
1 1.364 -1.936 0.020 1.327 0.428 1.021 -1.665 -0.907 2.173 0.818 -2.701 1.303 0.000 0.716 -0.590 -1.629 2.548 0.895 -2.280 -1.602 0.000 1.211 0.849 0.989 1.320 0.864 1.065 0.949
|
|
||||||
0 0.629 -0.626 0.609 1.828 1.280 0.644 -0.856 -0.873 2.173 0.555 1.066 -0.640 0.000 0.477 -1.364 -1.021 2.548 1.017 0.036 0.380 0.000 0.947 0.941 0.994 1.128 0.241 0.793 0.815
|
|
||||||
1 1.152 -0.843 0.926 1.802 0.800 2.493 -1.449 -1.127 0.000 1.737 0.833 0.488 0.000 1.026 0.929 -0.990 2.548 1.408 0.689 1.142 3.102 1.171 0.956 0.993 2.009 0.867 1.499 1.474
|
|
||||||
0 2.204 0.081 0.008 1.021 -0.679 2.676 0.090 1.163 0.000 2.210 -1.686 -1.195 0.000 1.805 0.891 -0.148 2.548 0.450 -0.502 -1.295 3.102 6.959 3.492 1.205 0.908 0.845 2.690 2.183
|
|
||||||
1 0.957 0.954 1.702 0.043 -0.503 1.113 0.033 -0.308 0.000 0.757 -0.363 -1.129 2.215 1.635 0.068 1.048 1.274 0.415 -2.098 0.061 0.000 1.010 0.979 0.992 0.704 1.125 0.761 0.715
|
|
||||||
0 1.222 0.418 1.059 1.303 1.442 0.282 -1.499 -1.286 0.000 1.567 0.016 -0.164 2.215 0.451 2.229 -1.229 0.000 0.660 -0.513 -0.296 3.102 2.284 1.340 0.985 1.531 0.314 1.032 1.094
|
|
||||||
1 0.603 1.675 -0.973 0.703 -1.709 1.023 0.652 1.296 2.173 1.078 0.363 -0.263 0.000 0.734 -0.457 -0.745 1.274 0.561 1.434 -0.042 0.000 0.888 0.771 0.984 0.847 1.234 0.874 0.777
|
|
||||||
0 0.897 0.949 -0.848 1.115 -0.085 0.522 -1.267 -1.418 0.000 0.684 -0.599 1.474 0.000 1.176 0.922 0.641 2.548 0.470 0.103 0.148 3.102 0.775 0.697 0.984 0.839 0.358 0.847 1.008
|
|
||||||
1 0.987 1.013 -1.504 0.468 -0.259 1.160 0.476 -0.971 2.173 1.266 0.919 0.780 0.000 0.634 1.695 0.233 0.000 0.487 -0.082 0.719 3.102 0.921 0.641 0.991 0.730 0.828 0.952 0.807
|
|
||||||
1 0.847 1.581 -1.397 1.629 1.529 1.053 0.816 -0.344 2.173 0.895 0.779 0.332 0.000 0.750 1.311 0.419 2.548 1.604 0.844 1.367 0.000 1.265 0.798 0.989 1.328 0.783 0.930 0.879
|
|
||||||
1 0.805 1.416 -1.327 0.397 0.589 0.488 0.982 0.843 0.000 0.664 -0.999 0.129 0.000 0.624 0.613 -0.558 0.000 1.431 -0.667 -1.561 3.102 0.959 1.103 0.989 0.590 0.632 0.926 0.798
|
|
||||||
0 1.220 -0.313 -0.489 1.759 0.201 1.698 -0.220 0.241 2.173 1.294 1.390 -1.682 0.000 1.447 -1.623 -1.296 0.000 1.710 0.872 -1.356 3.102 1.198 0.981 1.184 0.859 2.165 1.807 1.661
|
|
||||||
0 0.772 -0.611 -0.549 0.465 -1.528 1.103 -0.140 0.001 2.173 0.854 -0.406 1.655 0.000 0.733 -1.250 1.072 0.000 0.883 0.627 -1.132 3.102 0.856 0.927 0.987 1.094 1.013 0.938 0.870
|
|
||||||
1 1.910 0.771 0.828 0.231 1.267 1.398 1.455 -0.295 2.173 0.837 -2.564 0.770 0.000 0.540 2.189 1.287 0.000 1.345 1.311 -1.151 0.000 0.861 0.869 0.984 1.359 1.562 1.105 0.963
|
|
||||||
1 0.295 0.832 1.399 1.222 -0.517 2.480 0.013 1.591 0.000 2.289 0.436 0.287 2.215 1.995 -0.367 -0.409 1.274 0.375 1.367 -1.716 0.000 1.356 2.171 0.990 1.467 1.664 1.855 1.705
|
|
||||||
1 1.228 0.339 -0.575 0.417 1.474 0.480 -1.416 -1.498 2.173 0.614 -0.933 -0.961 0.000 1.189 1.690 1.003 0.000 1.690 -1.065 0.106 3.102 0.963 1.147 0.987 1.086 0.948 0.930 0.866
|
|
||||||
0 2.877 -1.014 1.440 0.782 0.483 1.134 -0.735 -0.196 2.173 1.123 0.084 -0.596 0.000 1.796 -0.356 1.044 2.548 1.406 1.582 -0.991 0.000 0.939 1.178 1.576 0.996 1.629 1.216 1.280
|
|
||||||
1 2.178 0.259 1.107 0.256 1.222 0.979 -0.440 -0.538 1.087 0.496 -0.760 -0.049 0.000 1.471 1.683 -1.486 0.000 0.646 0.695 -1.577 3.102 1.093 1.070 0.984 0.608 0.889 0.962 0.866
|
|
||||||
1 0.604 0.592 1.295 0.964 0.348 1.178 -0.016 0.832 2.173 1.626 -0.420 -0.760 0.000 0.748 0.461 -0.906 0.000 0.728 0.309 -1.269 1.551 0.852 0.604 0.989 0.678 0.949 1.021 0.878
|
|
||||||
0 0.428 -1.352 -0.912 1.713 0.797 1.894 -1.452 0.191 2.173 2.378 2.113 -1.190 0.000 0.860 2.174 0.949 0.000 1.693 0.759 1.426 3.102 0.885 1.527 1.186 1.090 3.294 4.492 3.676
|
|
||||||
0 0.473 0.485 0.154 1.433 -1.504 0.766 1.257 -1.302 2.173 0.414 0.119 0.238 0.000 0.805 0.242 -0.691 2.548 0.734 0.749 0.753 0.000 0.430 0.893 1.137 0.686 0.724 0.618 0.608
|
|
||||||
1 0.763 -0.601 0.876 0.182 -1.678 0.818 0.599 0.481 2.173 0.658 -0.737 -0.553 0.000 0.857 -1.138 -1.435 0.000 1.540 -1.466 -0.447 0.000 0.870 0.566 0.989 0.728 0.658 0.821 0.726
|
|
||||||
0 0.619 -0.273 -0.143 0.992 -1.267 0.566 0.876 -1.396 2.173 0.515 0.892 0.618 0.000 0.434 -0.902 0.862 2.548 0.490 -0.539 0.549 0.000 0.568 0.794 0.984 0.667 0.867 0.597 0.578
|
|
||||||
0 0.793 0.970 0.324 0.570 0.816 0.761 -0.550 1.519 2.173 1.150 0.496 -0.447 0.000 0.925 0.724 1.008 1.274 1.135 -0.275 -0.843 0.000 0.829 1.068 0.978 1.603 0.892 1.041 1.059
|
|
||||||
1 0.480 0.364 -0.067 1.906 -1.582 1.397 1.159 0.140 0.000 0.639 0.398 -1.102 0.000 1.597 -0.668 1.607 2.548 1.306 -0.797 0.288 3.102 0.856 1.259 1.297 1.022 1.032 1.049 0.939
|
|
||||||
0 0.514 1.304 1.490 1.741 -0.220 0.648 0.155 0.535 0.000 0.562 -1.016 0.837 0.000 0.863 -0.780 -0.815 2.548 1.688 -0.130 -1.545 3.102 0.887 0.980 1.309 1.269 0.654 1.044 1.035
|
|
||||||
0 1.225 0.333 0.656 0.893 0.859 1.037 -0.876 1.603 1.087 1.769 0.272 -0.227 2.215 1.000 0.579 -1.690 0.000 1.385 0.471 -0.860 0.000 0.884 1.207 0.995 1.097 2.336 1.282 1.145
|
|
||||||
0 2.044 -1.472 -0.294 0.392 0.369 0.927 0.718 1.492 1.087 1.619 -0.736 0.047 2.215 1.884 -0.101 -1.540 0.000 0.548 -0.441 1.117 0.000 0.798 0.877 0.981 0.750 2.272 1.469 1.276
|
|
||||||
0 1.037 -0.276 0.735 3.526 1.156 2.498 0.401 -0.590 1.087 0.714 -1.203 1.393 2.215 0.681 0.629 1.534 0.000 0.719 -0.355 -0.706 0.000 0.831 0.857 0.988 2.864 2.633 1.988 1.466
|
|
||||||
1 0.651 -1.218 -0.791 0.770 -1.449 0.610 -0.535 0.960 2.173 0.380 -1.072 -0.031 2.215 0.415 2.123 -1.100 0.000 0.776 0.217 0.420 0.000 0.986 1.008 1.001 0.853 0.588 0.799 0.776
|
|
||||||
0 1.586 -0.409 0.085 3.258 0.405 1.647 -0.674 -1.519 0.000 0.640 -1.027 -1.681 0.000 1.452 -0.444 -0.957 2.548 0.927 -0.017 1.215 3.102 0.519 0.866 0.992 0.881 0.847 1.018 1.278
|
|
||||||
0 0.712 0.092 -0.466 0.688 1.236 0.921 -1.217 -1.022 2.173 2.236 -1.167 0.868 2.215 0.851 -1.892 -0.753 0.000 0.475 -1.216 -0.383 0.000 0.668 0.758 0.988 1.180 2.093 1.157 0.934
|
|
||||||
0 0.419 0.471 0.974 2.805 0.235 1.473 -0.198 1.255 1.087 0.931 1.083 -0.712 0.000 1.569 1.358 -1.179 2.548 2.506 0.199 -0.842 0.000 0.929 0.991 0.992 1.732 2.367 1.549 1.430
|
|
||||||
1 0.667 1.003 1.504 0.368 1.061 0.885 -0.318 -0.353 0.000 1.438 -1.939 0.710 0.000 1.851 0.277 -1.460 2.548 1.403 0.517 -0.157 0.000 0.883 1.019 1.000 0.790 0.859 0.938 0.841
|
|
||||||
1 1.877 -0.492 0.372 0.441 0.955 1.034 -1.220 -0.846 1.087 0.952 -0.320 1.125 0.000 0.542 0.308 -1.261 2.548 1.018 -1.415 -1.547 0.000 1.280 0.932 0.991 1.273 0.878 0.921 0.906
|
|
||||||
0 1.052 0.901 1.176 1.280 1.517 0.562 -1.150 -0.079 2.173 1.228 -0.308 -0.354 0.000 0.790 -1.492 -0.963 0.000 0.942 -0.672 -1.588 3.102 1.116 0.902 0.988 1.993 0.765 1.375 1.325
|
|
||||||
1 0.518 -0.254 1.642 0.865 0.725 0.980 0.734 0.023 0.000 1.448 0.780 -1.736 2.215 0.955 0.513 -0.519 0.000 0.365 -0.444 -0.243 3.102 0.833 0.555 0.984 0.827 0.795 0.890 0.786
|
|
||||||
0 0.870 0.815 -0.506 0.663 -0.518 0.935 0.289 -1.675 2.173 1.188 0.005 0.635 0.000 0.580 0.066 -1.455 2.548 0.580 -0.634 -0.199 0.000 0.852 0.788 0.979 1.283 0.208 0.856 0.950
|
|
||||||
0 0.628 1.382 0.135 0.683 0.571 1.097 0.564 -0.950 2.173 0.617 -0.326 0.371 0.000 1.093 0.918 1.667 2.548 0.460 1.221 0.708 0.000 0.743 0.861 0.975 1.067 1.007 0.843 0.762
|
|
||||||
0 4.357 0.816 -1.609 1.845 -1.288 3.292 0.726 0.324 2.173 1.528 0.583 -0.801 2.215 0.605 0.572 1.406 0.000 0.794 -0.791 0.122 0.000 0.967 1.132 1.124 3.602 2.811 2.460 1.861
|
|
||||||
0 0.677 -1.265 1.559 0.866 -0.618 0.823 0.260 0.185 0.000 1.133 0.337 1.589 2.215 0.563 -0.830 0.510 0.000 0.777 0.117 -0.941 3.102 0.839 0.763 0.986 1.182 0.649 0.796 0.851
|
|
||||||
0 2.466 -1.838 -1.648 1.717 1.533 1.676 -1.553 -0.109 2.173 0.670 -0.666 0.284 0.000 0.334 -2.480 0.316 0.000 0.366 -0.804 -1.298 3.102 0.875 0.894 0.997 0.548 0.770 1.302 1.079
|
|
||||||
1 1.403 0.129 -1.307 0.688 0.306 0.579 0.753 0.814 1.087 0.474 0.694 -1.400 0.000 0.520 1.995 0.185 0.000 0.929 -0.504 1.270 3.102 0.972 0.998 1.353 0.948 0.650 0.688 0.724
|
|
||||||
1 0.351 1.188 -0.360 0.254 -0.346 1.129 0.545 1.691 0.000 0.652 -0.039 -0.258 2.215 1.089 0.655 0.472 2.548 0.554 -0.493 1.366 0.000 0.808 1.045 0.992 0.570 0.649 0.809 0.744
|
|
||||||
0 1.875 -0.013 -0.128 0.236 1.163 0.902 0.426 0.590 2.173 1.251 -1.210 -0.616 0.000 1.035 1.534 0.912 0.000 1.944 1.789 -1.691 0.000 0.974 1.113 0.990 0.925 1.120 0.956 0.912
|
|
||||||
0 0.298 0.750 -0.507 1.555 1.463 0.804 1.200 -0.665 0.000 0.439 -0.829 -0.252 1.107 0.770 -1.090 0.947 2.548 1.165 -0.166 -0.763 0.000 1.140 0.997 0.988 1.330 0.555 1.005 1.012
|
|
||||||
0 0.647 0.342 0.245 4.340 -0.157 2.229 0.068 1.170 2.173 2.133 -0.201 -1.441 0.000 1.467 0.697 -0.532 1.274 1.457 0.583 -1.640 0.000 0.875 1.417 0.976 2.512 2.390 1.794 1.665
|
|
||||||
1 1.731 -0.803 -1.013 1.492 -0.020 1.646 -0.541 1.121 2.173 0.459 -1.251 -1.495 2.215 0.605 -1.711 -0.232 0.000 0.658 0.634 -0.068 0.000 1.214 0.886 1.738 1.833 1.024 1.192 1.034
|
|
||||||
0 0.515 1.416 -1.089 1.697 1.426 1.414 0.941 0.027 0.000 1.480 0.133 -1.595 2.215 1.110 0.752 0.760 2.548 1.062 0.697 -0.492 0.000 0.851 0.955 0.994 1.105 1.255 1.175 1.095
|
|
||||||
0 1.261 0.858 1.465 0.757 0.305 2.310 0.679 1.080 2.173 1.544 2.518 -0.464 0.000 2.326 0.270 -0.841 0.000 2.163 0.839 -0.500 3.102 0.715 0.825 1.170 0.980 2.371 1.527 1.221
|
|
||||||
1 1.445 1.509 1.471 0.414 -1.285 0.767 0.864 -0.677 2.173 0.524 1.388 0.171 0.000 0.826 0.190 0.121 2.548 0.572 1.691 -1.603 0.000 0.870 0.935 0.994 0.968 0.735 0.783 0.777
|
|
||||||
1 0.919 -0.264 -1.245 0.681 -1.722 1.022 1.010 0.097 2.173 0.685 0.403 -1.351 0.000 1.357 -0.429 1.262 1.274 0.687 1.021 -0.563 0.000 0.953 0.796 0.991 0.873 1.749 1.056 0.917
|
|
||||||
1 0.293 -2.258 -1.427 1.191 1.202 0.394 -2.030 1.438 0.000 0.723 0.596 -0.024 2.215 0.525 -1.678 -0.290 0.000 0.788 -0.824 -1.029 3.102 0.821 0.626 0.976 1.080 0.810 0.842 0.771
|
|
||||||
0 3.286 0.386 1.688 1.619 -1.620 1.392 -0.009 0.280 0.000 1.179 -0.776 -0.110 2.215 1.256 0.248 -1.114 2.548 0.777 0.825 -0.156 0.000 1.026 1.065 0.964 0.909 1.249 1.384 1.395
|
|
||||||
1 1.075 0.603 0.561 0.656 -0.685 0.985 0.175 0.979 2.173 1.154 0.584 -0.886 0.000 1.084 -0.354 -1.004 2.548 0.865 1.224 1.269 0.000 1.346 1.073 1.048 0.873 1.310 1.003 0.865
|
|
||||||
1 1.098 -0.091 1.466 1.558 0.915 0.649 1.314 -1.182 2.173 0.791 0.073 0.351 0.000 0.517 0.940 1.195 0.000 1.150 1.187 -0.692 3.102 0.866 0.822 0.980 1.311 0.394 1.119 0.890
|
|
||||||
1 0.481 -1.042 0.148 1.135 -1.249 1.202 -0.344 0.308 1.087 0.779 -1.431 1.581 0.000 0.860 -0.860 -1.125 0.000 0.785 0.303 1.199 3.102 0.878 0.853 0.988 1.072 0.827 0.936 0.815
|
|
||||||
0 1.348 0.497 0.318 0.806 0.976 1.393 -0.152 0.632 2.173 2.130 0.515 -1.054 0.000 0.908 0.062 -0.780 0.000 1.185 0.687 1.668 1.551 0.720 0.898 0.985 0.683 1.292 1.320 1.131
|
|
||||||
0 2.677 -0.420 -1.685 1.828 1.433 2.040 -0.718 -0.039 0.000 0.400 -0.873 0.472 0.000 0.444 0.340 -0.830 2.548 0.431 0.768 -1.417 3.102 0.869 0.917 0.996 0.707 0.193 0.728 1.154
|
|
||||||
1 1.300 0.586 -0.122 1.306 0.609 0.727 -0.556 -1.652 2.173 0.636 0.720 1.393 2.215 0.328 1.280 -0.390 0.000 0.386 0.752 -0.905 0.000 0.202 0.751 1.106 0.864 0.799 0.928 0.717
|
|
||||||
0 0.637 -0.176 1.737 1.322 -0.414 0.702 -0.964 -0.680 0.000 1.054 -0.461 0.889 2.215 0.861 -0.267 0.225 0.000 1.910 -1.888 1.027 0.000 0.919 0.899 1.186 0.993 1.109 0.862 0.775
|
|
||||||
1 0.723 -0.104 1.572 0.428 -0.840 0.655 0.544 1.401 2.173 1.522 -0.154 -0.452 2.215 0.996 0.190 0.273 0.000 1.906 -0.176 0.966 0.000 0.945 0.894 0.990 0.981 1.555 0.988 0.893
|
|
||||||
0 2.016 -0.570 1.612 0.798 0.441 0.334 0.191 -0.909 0.000 0.939 0.146 0.021 2.215 0.553 -0.444 1.156 2.548 0.781 -1.545 -0.520 0.000 0.922 0.956 1.528 0.722 0.699 0.778 0.901
|
|
||||||
0 1.352 -0.707 1.284 0.665 0.580 0.694 -1.040 -0.899 2.173 0.692 -2.048 0.029 0.000 0.545 -2.042 1.259 0.000 0.661 -0.808 -1.251 3.102 0.845 0.991 0.979 0.662 0.225 0.685 0.769
|
|
||||||
1 1.057 -1.561 -0.411 0.952 -0.681 1.236 -1.107 1.045 2.173 1.288 -2.521 -0.521 0.000 1.361 -1.239 1.546 0.000 0.373 -1.540 0.028 0.000 0.794 0.782 0.987 0.889 0.832 0.972 0.828
|
|
||||||
0 1.118 -0.017 -1.227 1.077 1.256 0.714 0.624 -0.811 0.000 0.800 0.704 0.387 1.107 0.604 0.234 0.986 0.000 1.306 -0.456 0.094 3.102 0.828 0.984 1.195 0.987 0.672 0.774 0.748
|
|
||||||
1 0.602 2.201 0.212 0.119 0.182 0.474 2.130 1.270 0.000 0.370 2.088 -0.573 0.000 0.780 -0.725 -1.033 0.000 1.642 0.598 0.303 3.102 0.886 0.988 0.985 0.644 0.756 0.651 0.599
|
|
||||||
0 1.677 -0.844 1.581 0.585 0.887 1.012 -2.315 0.752 0.000 1.077 0.748 -0.195 0.000 0.718 0.832 -1.337 1.274 1.181 -0.557 -1.006 3.102 1.018 1.247 0.988 0.908 0.651 1.311 1.120
|
|
||||||
1 1.695 0.259 1.224 1.344 1.067 0.718 -1.752 -0.215 0.000 0.473 0.991 -0.993 0.000 0.891 1.285 -1.500 2.548 0.908 -0.131 0.288 0.000 0.945 0.824 0.979 1.009 0.951 0.934 0.833
|
|
||||||
0 0.793 0.628 0.432 1.707 0.302 0.919 1.045 -0.784 0.000 1.472 0.175 -1.284 2.215 1.569 0.155 0.971 2.548 0.435 0.735 1.625 0.000 0.801 0.907 0.992 0.831 1.446 1.082 1.051
|
|
||||||
1 0.537 -0.664 -0.244 1.104 1.272 1.154 0.394 1.633 0.000 1.527 0.963 0.559 2.215 1.744 0.650 -0.912 0.000 1.097 0.730 -0.368 3.102 1.953 1.319 1.045 1.309 0.869 1.196 1.126
|
|
||||||
1 0.585 -1.469 1.005 0.749 -1.060 1.224 -0.717 -0.323 2.173 1.012 -0.201 1.268 0.000 0.359 -0.567 0.476 0.000 1.117 -1.124 1.557 3.102 0.636 1.281 0.986 0.616 1.289 0.890 0.881
|
|
||||||
1 0.354 -1.517 0.667 2.534 -1.298 1.020 -0.375 1.254 0.000 1.119 -0.060 -1.538 2.215 1.059 -0.395 -0.140 0.000 2.609 0.199 -0.778 1.551 0.957 0.975 1.286 1.666 1.003 1.224 1.135
|
|
||||||
1 0.691 -1.619 -1.380 0.361 1.727 1.493 -1.093 -0.289 0.000 1.447 -0.640 1.341 0.000 1.453 -0.617 -1.456 1.274 1.061 -1.481 -0.091 0.000 0.744 0.649 0.987 0.596 0.727 0.856 0.797
|
|
||||||
0 1.336 1.293 -1.359 0.357 0.067 1.110 -0.058 -0.515 0.000 0.976 1.498 1.207 0.000 1.133 0.437 1.053 2.548 0.543 1.374 0.171 0.000 0.764 0.761 0.984 0.827 0.553 0.607 0.612
|
|
||||||
0 0.417 -1.111 1.661 2.209 -0.683 1.931 -0.642 0.959 1.087 1.514 -2.032 -0.686 0.000 1.521 -0.539 1.344 0.000 0.978 -0.866 0.363 1.551 2.813 1.850 1.140 1.854 0.799 1.600 1.556
|
|
||||||
0 1.058 0.390 -0.591 0.134 1.149 0.346 -1.550 0.186 0.000 1.108 -0.999 0.843 1.107 1.124 0.415 -1.514 0.000 1.067 -0.426 -1.000 3.102 1.744 1.050 0.985 1.006 1.010 0.883 0.789
|
|
||||||
1 1.655 0.253 1.216 0.270 1.703 0.500 -0.006 -1.418 2.173 0.690 -0.350 0.170 2.215 1.045 -0.924 -0.774 0.000 0.996 -0.745 -0.123 0.000 0.839 0.820 0.993 0.921 0.869 0.725 0.708
|
|
||||||
0 1.603 -0.850 0.564 0.829 0.093 1.270 -1.113 -1.155 2.173 0.853 -1.021 1.248 2.215 0.617 -1.270 1.733 0.000 0.935 -0.092 0.136 0.000 1.011 1.074 0.977 0.823 1.269 1.054 0.878
|
|
||||||
0 1.568 -0.792 1.005 0.545 0.896 0.895 -1.698 -0.988 0.000 0.608 -1.634 1.705 0.000 0.826 0.208 0.618 1.274 2.063 -1.743 -0.520 0.000 0.939 0.986 0.990 0.600 0.435 1.033 1.087
|
|
||||||
0 0.489 -1.335 -1.102 1.738 1.028 0.628 -0.992 -0.627 0.000 0.652 -0.064 -0.215 0.000 1.072 0.173 -1.251 2.548 1.042 0.057 0.841 3.102 0.823 0.895 1.200 1.164 0.770 0.837 0.846
|
|
||||||
1 1.876 0.870 1.234 0.556 -1.262 1.764 0.855 -0.467 2.173 1.079 1.351 0.852 0.000 0.773 0.383 0.874 0.000 1.292 0.829 -1.228 3.102 0.707 0.969 1.102 1.601 1.017 1.112 1.028
|
|
||||||
0 1.033 0.407 -0.374 0.705 -1.254 0.690 -0.231 1.502 2.173 0.433 -2.009 -0.057 0.000 0.861 1.151 0.334 0.000 0.960 -0.839 1.299 3.102 2.411 1.480 0.982 0.995 0.377 1.012 0.994
|
|
||||||
0 1.092 0.653 -0.801 0.463 0.426 0.529 -1.055 0.040 0.000 0.663 0.999 1.255 1.107 0.749 -1.106 1.185 2.548 0.841 -0.745 -1.029 0.000 0.841 0.743 0.988 0.750 1.028 0.831 0.868
|
|
||||||
1 0.799 -0.285 -0.011 0.531 1.392 1.063 0.854 0.494 2.173 1.187 -1.065 -0.851 0.000 0.429 -0.296 1.072 0.000 0.942 -1.985 1.172 0.000 0.873 0.693 0.992 0.819 0.689 1.131 0.913
|
|
||||||
0 0.503 1.973 -0.377 1.515 -1.514 0.708 1.081 -0.313 2.173 1.110 -0.417 0.839 0.000 0.712 -1.153 1.165 0.000 0.675 -0.303 -0.930 1.551 0.709 0.761 1.032 0.986 0.698 0.963 1.291
|
|
||||||
0 0.690 -0.574 -1.608 1.182 1.118 0.557 -2.243 0.144 0.000 0.969 0.216 -1.383 1.107 1.054 0.888 -0.709 2.548 0.566 1.663 -0.550 0.000 0.752 1.528 0.987 1.408 0.740 1.290 1.123
|
|
||||||
1 0.890 1.501 0.786 0.779 -0.615 1.126 0.716 1.541 2.173 0.887 0.728 -0.673 2.215 1.216 0.332 -0.020 0.000 0.965 1.828 0.101 0.000 0.827 0.715 1.099 1.088 1.339 0.924 0.878
|
|
||||||
0 0.566 0.883 0.655 1.600 0.034 1.155 2.028 -1.499 0.000 0.723 -0.871 0.763 0.000 1.286 -0.696 -0.676 2.548 1.134 -0.113 1.207 3.102 4.366 2.493 0.984 0.960 0.962 1.843 1.511
|
|
||||||
0 1.146 1.086 -0.911 0.838 1.298 0.821 0.127 -0.145 0.000 1.352 0.474 -1.580 2.215 1.619 -0.081 0.675 2.548 1.382 -0.748 0.127 0.000 0.958 0.976 1.239 0.876 1.481 1.116 1.076
|
|
||||||
0 1.739 -0.326 -1.661 0.420 -1.705 1.193 -0.031 -1.212 2.173 1.783 -0.442 0.522 0.000 1.064 -0.692 0.027 0.000 1.314 0.359 -0.037 3.102 0.968 0.897 0.986 0.907 1.196 1.175 1.112
|
|
||||||
1 0.669 0.194 -0.703 0.657 -0.260 0.899 -2.511 0.311 0.000 1.482 0.773 0.974 2.215 3.459 0.037 -1.299 1.274 2.113 0.067 1.516 0.000 0.740 0.871 0.979 1.361 2.330 1.322 1.046
|
|
||||||
1 1.355 -1.033 -1.173 0.552 -0.048 0.899 -0.482 -1.287 2.173 1.422 -1.227 0.390 1.107 1.937 -0.028 0.914 0.000 0.849 -0.230 -1.734 0.000 0.986 1.224 1.017 1.051 1.788 1.150 1.009
|
|
||||||
1 0.511 -0.202 1.029 0.780 1.154 0.816 0.532 -0.731 0.000 0.757 0.517 0.749 2.215 1.302 0.289 -1.188 0.000 0.584 1.211 -0.350 0.000 0.876 0.943 0.995 0.963 0.256 0.808 0.891
|
|
||||||
1 1.109 0.572 1.484 0.753 1.543 1.711 -0.145 -0.746 1.087 1.759 0.631 0.845 2.215 0.945 0.542 0.003 0.000 0.378 -1.150 -0.044 0.000 0.764 1.042 0.992 1.045 2.736 1.441 1.140
|
|
||||||
0 0.712 -0.025 0.553 0.928 -0.711 1.304 0.045 -0.300 0.000 0.477 0.720 0.969 0.000 1.727 -0.474 1.328 1.274 1.282 2.222 1.684 0.000 0.819 0.765 1.023 0.961 0.657 0.799 0.744
|
|
||||||
1 1.131 -0.302 1.079 0.901 0.236 0.904 -0.249 1.694 2.173 1.507 -0.702 -1.128 0.000 0.774 0.565 0.284 2.548 1.802 1.446 -0.192 0.000 3.720 2.108 0.986 0.930 1.101 1.484 1.238
|
|
||||||
0 1.392 1.253 0.118 0.864 -1.358 0.922 -0.447 -1.243 1.087 1.969 1.031 0.774 2.215 1.333 -0.359 -0.681 0.000 1.099 -0.257 1.473 0.000 1.246 0.909 1.475 1.234 2.531 1.449 1.306
|
|
||||||
0 1.374 2.291 -0.479 1.339 -0.243 0.687 2.345 1.310 0.000 0.467 1.081 0.772 0.000 0.656 1.155 -1.636 2.548 0.592 0.536 -1.269 3.102 0.981 0.821 1.010 0.877 0.217 0.638 0.758
|
|
||||||
1 0.401 -1.516 0.909 2.738 0.519 0.887 0.566 -1.202 0.000 0.909 -0.176 1.682 0.000 2.149 -0.878 -0.514 2.548 0.929 -0.563 -1.555 3.102 1.228 0.803 0.980 1.382 0.884 1.025 1.172
|
|
||||||
1 0.430 -1.589 1.417 2.158 1.226 1.180 -0.829 -0.781 2.173 0.798 1.400 -0.111 0.000 0.939 -0.878 1.076 2.548 0.576 1.335 -0.826 0.000 0.861 0.970 0.982 1.489 1.308 1.015 0.992
|
|
||||||
1 1.943 -0.391 -0.840 0.621 -1.613 2.026 1.734 1.025 0.000 0.930 0.573 -0.912 0.000 1.326 0.847 -0.220 1.274 1.181 0.079 0.709 3.102 1.164 1.007 0.987 1.094 0.821 0.857 0.786
|
|
||||||
1 0.499 0.436 0.887 0.859 1.509 0.733 -0.559 1.111 1.087 1.011 -0.796 0.279 2.215 1.472 -0.510 -0.982 0.000 1.952 0.379 -0.733 0.000 1.076 1.358 0.991 0.589 0.879 1.068 0.922
|
|
||||||
0 0.998 -0.407 -1.711 0.139 0.652 0.810 -0.331 -0.721 0.000 0.471 -0.533 0.442 0.000 0.531 -1.405 0.120 2.548 0.707 0.098 -1.176 1.551 1.145 0.809 0.988 0.529 0.612 0.562 0.609
|
|
||||||
1 1.482 0.872 0.638 1.288 0.362 0.856 0.900 -0.511 1.087 1.072 1.061 -1.432 2.215 1.770 -2.292 -1.547 0.000 1.131 1.374 0.783 0.000 6.316 4.381 1.002 1.317 1.048 2.903 2.351
|
|
||||||
1 2.084 -0.422 1.289 1.125 0.735 1.104 -0.518 -0.326 2.173 0.413 -0.719 -0.699 0.000 0.857 0.108 -1.631 0.000 0.527 0.641 -1.362 3.102 0.791 0.952 1.016 0.776 0.856 0.987 0.836
|
|
||||||
0 0.464 0.674 0.025 0.430 -1.703 0.982 -1.311 -0.808 2.173 1.875 1.060 0.821 2.215 0.954 -0.480 -1.677 0.000 0.567 0.702 -0.939 0.000 0.781 1.076 0.989 1.256 3.632 1.652 1.252
|
|
||||||
1 0.457 -1.944 -1.010 1.409 0.931 1.098 -0.742 -0.415 0.000 1.537 -0.834 0.945 2.215 1.752 -0.287 -1.269 2.548 0.692 -1.537 -0.223 0.000 0.801 1.192 1.094 1.006 1.659 1.175 1.122
|
|
||||||
0 3.260 -0.943 1.737 0.920 1.309 0.946 -0.139 -0.271 2.173 0.994 -0.952 -0.311 0.000 0.563 -0.136 -0.881 0.000 1.236 -0.507 0.906 1.551 0.747 0.869 0.985 1.769 1.034 1.179 1.042
|
|
||||||
0 0.615 -0.778 0.246 1.861 1.619 0.560 -0.943 -0.204 2.173 0.550 -0.759 -1.342 2.215 0.578 0.076 -0.973 0.000 0.939 0.035 0.680 0.000 0.810 0.747 1.401 0.772 0.702 0.719 0.662
|
|
||||||
1 2.370 -0.064 -0.237 1.737 0.154 2.319 -1.838 -1.673 0.000 1.053 -1.305 -0.075 0.000 0.925 0.149 0.318 1.274 0.851 -0.922 0.981 3.102 0.919 0.940 0.989 0.612 0.598 1.219 1.626
|
|
||||||
1 1.486 0.311 -1.262 1.354 -0.847 0.886 -0.158 1.213 2.173 1.160 -0.218 0.239 0.000 1.166 0.494 0.278 2.548 0.575 1.454 -1.701 0.000 0.429 1.129 0.983 1.111 1.049 1.006 0.920
|
|
||||||
1 1.294 1.587 -0.864 0.487 -0.312 0.828 1.051 -0.031 1.087 2.443 1.216 1.609 2.215 1.167 0.813 0.921 0.000 1.751 -0.415 0.119 0.000 1.015 1.091 0.974 1.357 2.093 1.178 1.059
|
|
||||||
1 0.984 0.465 -1.661 0.379 -0.554 0.977 0.237 0.365 0.000 0.510 0.143 1.101 0.000 1.099 -0.662 -1.593 2.548 1.104 -0.197 -0.648 3.102 0.925 0.922 0.986 0.642 0.667 0.806 0.722
|
|
||||||
1 0.930 -0.009 0.047 0.667 1.367 1.065 -0.231 0.815 0.000 1.199 -1.114 -0.877 2.215 0.940 0.824 -1.583 0.000 1.052 -0.407 -0.076 1.551 1.843 1.257 1.013 1.047 0.751 1.158 0.941
|
|
||||||
0 0.767 -0.011 -0.637 0.341 -1.437 1.438 -0.425 -0.450 2.173 1.073 -0.718 1.341 2.215 0.633 -1.394 0.486 0.000 0.603 -1.945 -1.626 0.000 0.703 0.790 0.984 1.111 1.848 1.129 1.072
|
|
||||||
1 1.779 0.017 0.432 0.402 1.022 0.959 1.480 1.595 2.173 1.252 1.365 0.006 0.000 1.188 -0.174 -1.107 0.000 1.181 0.518 -0.258 0.000 1.057 0.910 0.991 1.616 0.779 1.158 1.053
|
|
||||||
0 0.881 0.630 1.029 1.990 0.508 1.102 0.742 -1.298 2.173 1.565 1.085 0.686 2.215 2.691 1.391 -0.904 0.000 0.499 1.388 -1.199 0.000 0.347 0.861 0.997 0.881 1.920 1.233 1.310
|
|
||||||
0 1.754 -0.266 0.389 0.347 -0.030 0.462 -1.408 -0.957 2.173 0.515 -2.341 -1.700 0.000 0.588 -0.797 1.355 2.548 0.608 0.329 -1.389 0.000 1.406 0.909 0.988 0.760 0.593 0.768 0.847
|
|
||||||
0 1.087 0.311 -1.447 0.173 0.567 0.854 0.362 0.584 0.000 1.416 -0.716 -1.211 2.215 0.648 -0.358 -0.692 1.274 0.867 -0.513 0.206 0.000 0.803 0.813 0.984 1.110 0.491 0.921 0.873
|
|
||||||
0 0.279 1.114 -1.190 3.004 -0.738 1.233 0.896 1.092 2.173 0.454 -0.374 0.117 2.215 0.357 0.119 1.270 0.000 0.458 1.343 0.316 0.000 0.495 0.540 0.988 1.715 1.139 1.618 1.183
|
|
||||||
1 1.773 -0.694 -1.518 2.306 -1.200 3.104 0.749 0.362 0.000 1.871 0.230 -1.686 2.215 0.805 -0.179 -0.871 1.274 0.910 0.607 -0.246 0.000 1.338 1.598 0.984 1.050 0.919 1.678 1.807
|
|
||||||
0 0.553 0.683 0.827 0.973 -0.706 1.488 0.149 1.140 2.173 1.788 0.447 -0.478 0.000 0.596 1.043 1.607 0.000 0.373 -0.868 -1.308 1.551 1.607 1.026 0.998 1.134 0.808 1.142 0.936
|
|
||||||
1 0.397 1.101 -1.139 1.688 0.146 0.972 0.541 1.518 0.000 1.549 -0.873 -1.012 0.000 2.282 -0.151 0.314 2.548 1.174 0.033 -1.368 0.000 0.937 0.776 1.039 1.143 0.959 0.986 1.013
|
|
||||||
1 0.840 1.906 -0.959 0.869 0.576 0.642 0.554 -1.351 0.000 0.756 0.923 -0.823 2.215 1.251 1.130 0.545 2.548 1.513 0.410 1.073 0.000 1.231 0.985 1.163 0.812 0.987 0.816 0.822
|
|
||||||
1 0.477 1.665 0.814 0.763 -0.382 0.828 -0.008 0.280 2.173 1.213 -0.001 1.560 0.000 1.136 0.311 -1.289 0.000 0.797 1.091 -0.616 3.102 1.026 0.964 0.992 0.772 0.869 0.916 0.803
|
|
||||||
0 2.655 0.020 0.273 1.464 0.482 1.709 -0.107 -1.456 2.173 0.825 0.141 -0.386 0.000 1.342 -0.592 1.635 1.274 0.859 -0.175 -0.874 0.000 0.829 0.946 1.003 2.179 0.836 1.505 1.176
|
|
||||||
0 0.771 -1.992 -0.720 0.732 -1.464 0.869 -1.290 0.388 2.173 0.926 -1.072 -1.489 2.215 0.640 -1.232 0.840 0.000 0.528 -2.440 -0.446 0.000 0.811 0.868 0.993 0.995 1.317 0.809 0.714
|
|
||||||
0 1.357 1.302 0.076 0.283 -1.060 0.783 1.559 -0.994 0.000 0.947 1.212 1.617 0.000 1.127 0.311 0.442 2.548 0.582 -0.052 1.186 1.551 1.330 0.995 0.985 0.846 0.404 0.858 0.815
|
|
||||||
0 0.442 -0.381 -0.424 1.244 0.591 0.731 0.605 -0.713 2.173 0.629 2.762 1.040 0.000 0.476 2.693 -0.617 0.000 0.399 0.442 1.486 3.102 0.839 0.755 0.988 0.869 0.524 0.877 0.918
|
|
||||||
0 0.884 0.422 0.055 0.818 0.624 0.950 -0.763 1.624 0.000 0.818 -0.609 -1.166 0.000 1.057 -0.528 1.070 2.548 1.691 -0.124 -0.335 3.102 1.104 0.933 0.985 0.913 1.000 0.863 1.056
|
|
||||||
0 1.276 0.156 1.714 1.053 -1.189 0.672 -0.464 -0.030 2.173 0.469 -2.483 0.442 0.000 0.564 2.580 -0.253 0.000 0.444 -0.628 1.080 1.551 5.832 2.983 0.985 1.162 0.494 1.809 1.513
|
|
||||||
0 1.106 -0.556 0.406 0.573 -1.400 0.769 -0.518 1.457 2.173 0.743 -0.352 -0.010 0.000 1.469 -0.550 -0.930 2.548 0.540 1.236 -0.571 0.000 0.962 0.970 1.101 0.805 1.107 0.873 0.773
|
|
||||||
0 0.539 -0.964 -0.464 1.371 -1.606 0.667 -0.160 0.655 0.000 0.952 0.352 -0.740 2.215 0.952 0.007 1.123 0.000 1.061 -0.505 1.389 3.102 1.063 0.991 1.019 0.633 0.967 0.732 0.799
|
|
||||||
1 0.533 -0.989 -1.608 0.462 -1.723 1.204 -0.598 -0.098 2.173 1.343 -0.460 1.632 2.215 0.577 0.221 -0.492 0.000 0.628 -0.073 0.472 0.000 0.518 0.880 0.988 1.179 1.874 1.041 0.813
|
|
||||||
1 1.024 1.075 -0.795 0.286 -1.436 1.365 0.857 -0.309 2.173 0.804 1.532 1.435 0.000 1.511 0.722 1.494 0.000 1.778 0.903 0.753 1.551 0.686 0.810 0.999 0.900 1.360 1.133 0.978
|
|
||||||
1 2.085 -0.269 -1.423 0.789 1.298 0.281 1.652 0.187 0.000 0.658 -0.760 -0.042 2.215 0.663 0.024 0.120 0.000 0.552 -0.299 -0.428 3.102 0.713 0.811 1.130 0.705 0.218 0.675 0.743
|
|
||||||
1 0.980 -0.443 0.813 0.785 -1.253 0.719 0.448 -1.458 0.000 1.087 0.595 0.635 1.107 1.428 0.029 -0.995 0.000 1.083 1.562 -0.092 0.000 0.834 0.891 1.165 0.967 0.661 0.880 0.817
|
|
||||||
1 0.903 -0.733 -0.980 0.634 -0.639 0.780 0.266 -0.287 2.173 1.264 -0.936 1.004 0.000 1.002 -0.056 -1.344 2.548 1.183 -0.098 1.169 0.000 0.733 1.002 0.985 0.711 0.916 0.966 0.875
|
|
||||||
0 0.734 -0.304 -1.175 2.851 1.674 0.904 -0.634 0.412 2.173 1.363 -1.050 -0.282 0.000 1.476 -1.603 0.103 0.000 2.231 -0.718 1.708 3.102 0.813 0.896 1.088 0.686 1.392 1.033 1.078
|
|
||||||
1 1.680 0.591 -0.243 0.111 -0.478 0.326 -0.079 -1.555 2.173 0.711 0.714 0.922 2.215 0.355 0.858 1.682 0.000 0.727 1.620 1.360 0.000 0.334 0.526 1.001 0.862 0.633 0.660 0.619
|
|
||||||
1 1.163 0.225 -0.202 0.501 -0.979 1.609 -0.938 1.424 0.000 1.224 -0.118 -1.274 0.000 2.034 1.241 -0.254 0.000 1.765 0.536 0.237 3.102 0.894 0.838 0.988 0.693 0.579 0.762 0.726
|
|
||||||
0 1.223 1.232 1.471 0.489 1.728 0.703 -0.111 0.411 0.000 1.367 1.014 -1.294 1.107 1.524 -0.414 -0.164 2.548 1.292 0.833 0.316 0.000 0.861 0.752 0.994 0.836 1.814 1.089 0.950
|
|
||||||
0 0.816 1.637 -1.557 1.036 -0.342 0.913 1.333 0.949 2.173 0.812 0.756 -0.628 2.215 1.333 0.470 1.495 0.000 1.204 -2.222 -1.675 0.000 1.013 0.924 1.133 0.758 1.304 0.855 0.860
|
|
||||||
0 0.851 -0.564 -0.691 0.692 1.345 1.219 1.014 0.318 0.000 1.422 -0.262 -1.635 2.215 0.531 1.802 0.008 0.000 0.508 0.515 -1.267 3.102 0.821 0.787 1.026 0.783 0.432 1.149 1.034
|
|
||||||
0 0.800 -0.599 0.204 0.552 -0.484 0.974 0.413 0.961 2.173 1.269 -0.984 -1.039 2.215 0.380 -1.213 1.371 0.000 0.551 0.332 -0.659 0.000 0.694 0.852 0.984 1.057 2.037 1.096 0.846
|
|
||||||
0 0.744 -0.071 -0.255 0.638 0.512 1.125 0.407 0.844 2.173 0.860 -0.481 -0.677 0.000 1.102 0.181 -1.194 0.000 1.011 -1.081 -1.713 3.102 0.854 0.862 0.982 1.111 1.372 1.042 0.920
|
|
||||||
1 0.400 1.049 -0.625 0.880 -0.407 1.040 2.150 -1.359 0.000 0.747 -0.144 0.847 2.215 0.560 -1.829 0.698 0.000 1.663 -0.668 0.267 0.000 0.845 0.964 0.996 0.820 0.789 0.668 0.668
|
|
||||||
0 1.659 -0.705 -1.057 1.803 -1.436 1.008 0.693 0.005 0.000 0.895 -0.007 0.681 1.107 1.085 0.125 1.476 2.548 1.214 1.068 0.486 0.000 0.867 0.919 0.986 1.069 0.692 1.026 1.313
|
|
||||||
0 0.829 -0.153 0.861 0.615 -0.548 0.589 1.077 -0.041 2.173 1.056 0.763 -1.737 0.000 0.639 0.970 0.725 0.000 0.955 1.227 -0.799 3.102 1.020 1.024 0.985 0.750 0.525 0.685 0.671
|
|
||||||
1 0.920 -0.806 -0.840 1.048 0.278 0.973 -0.077 -1.364 2.173 1.029 0.309 0.133 0.000 1.444 1.484 1.618 1.274 1.419 -0.482 0.417 0.000 0.831 1.430 1.151 1.829 1.560 1.343 1.224
|
|
||||||
1 0.686 0.249 -0.905 0.343 -1.731 0.724 -2.823 -0.901 0.000 0.982 0.303 1.312 1.107 1.016 0.245 0.610 0.000 1.303 -0.557 -0.360 3.102 1.384 1.030 0.984 0.862 1.144 0.866 0.779
|
|
||||||
0 1.603 0.444 0.508 0.586 0.401 0.610 0.467 -1.735 2.173 0.914 0.626 -1.019 0.000 0.812 0.422 -0.408 2.548 0.902 1.679 1.490 0.000 1.265 0.929 0.990 1.004 0.816 0.753 0.851
|
|
||||||
1 0.623 0.780 -0.203 0.056 0.015 0.899 0.793 1.326 1.087 0.803 1.478 -1.499 2.215 1.561 1.492 -0.120 0.000 0.904 0.795 0.137 0.000 0.548 1.009 0.850 0.924 0.838 0.914 0.860
|
|
||||||
0 1.654 -2.032 -1.160 0.859 -1.583 0.689 -1.965 0.891 0.000 0.646 -1.014 -0.288 2.215 0.630 -0.815 0.402 0.000 0.638 0.316 0.655 3.102 0.845 0.879 0.993 1.067 0.625 1.041 0.958
|
|
||||||
1 0.828 -1.269 -1.203 0.744 -0.213 0.626 -1.017 -0.404 0.000 1.281 -0.931 1.733 2.215 0.699 -0.351 1.287 0.000 1.251 -1.171 0.197 0.000 0.976 1.186 0.987 0.646 0.655 0.733 0.671
|
|
||||||
1 0.677 0.111 1.090 1.580 1.591 1.560 0.654 -0.341 2.173 0.794 -0.266 0.702 0.000 0.823 0.651 -1.239 2.548 0.730 1.467 -1.530 0.000 1.492 1.023 0.983 1.909 1.022 1.265 1.127
|
|
||||||
1 0.736 0.882 -1.060 0.589 0.168 1.663 0.781 1.022 2.173 2.025 1.648 -1.292 0.000 1.240 0.924 -0.421 1.274 1.354 0.065 0.501 0.000 0.316 0.925 0.988 0.664 1.736 0.992 0.807
|
|
||||||
1 1.040 -0.822 1.638 0.974 -0.674 0.393 0.830 0.011 2.173 0.770 -0.140 -0.402 0.000 0.294 -0.133 0.030 0.000 1.220 0.807 0.638 0.000 0.826 1.063 1.216 1.026 0.705 0.934 0.823
|
|
||||||
1 0.711 0.602 0.048 1.145 0.966 0.934 0.263 -1.589 2.173 0.971 -0.496 -0.421 1.107 0.628 -0.865 0.845 0.000 0.661 -0.008 -0.565 0.000 0.893 0.705 0.988 0.998 1.339 0.908 0.872
|
|
||||||
1 0.953 -1.651 -0.167 0.885 1.053 1.013 -1.239 0.133 0.000 1.884 -1.122 1.222 2.215 1.906 -0.860 -1.184 1.274 1.413 -0.668 -1.647 0.000 1.873 1.510 1.133 1.050 1.678 1.246 1.061
|
|
||||||
1 0.986 -0.892 -1.380 0.917 1.134 0.950 -1.162 -0.469 0.000 0.569 -1.393 0.215 0.000 0.320 2.667 1.712 0.000 1.570 -0.375 1.457 3.102 0.925 1.128 1.011 0.598 0.824 0.913 0.833
|
|
||||||
1 1.067 0.099 1.154 0.527 -0.789 1.085 0.623 -1.602 2.173 1.511 -0.230 0.022 2.215 0.269 -0.377 0.883 0.000 0.571 -0.540 -0.512 0.000 0.414 0.803 1.022 0.959 2.053 1.041 0.780
|
|
||||||
0 0.825 -2.118 0.217 1.453 -0.493 0.819 0.313 -0.942 0.000 2.098 -0.725 1.096 2.215 0.484 1.336 1.458 0.000 0.482 0.100 1.163 0.000 0.913 0.536 0.990 1.679 0.957 1.095 1.143
|
|
||||||
1 1.507 0.054 1.120 0.698 -1.340 0.912 0.384 0.015 1.087 0.720 0.247 -0.820 0.000 0.286 0.154 1.578 2.548 0.629 1.582 -0.576 0.000 0.828 0.893 1.136 0.514 0.632 0.699 0.709
|
|
||||||
1 0.610 1.180 -0.993 0.816 0.301 0.932 0.758 1.539 0.000 0.726 -0.830 0.248 2.215 0.883 0.857 -1.305 0.000 1.338 1.009 -0.252 3.102 0.901 1.074 0.987 0.875 1.159 1.035 0.858
|
|
||||||
1 1.247 -1.360 1.502 1.525 -1.332 0.618 1.063 0.755 0.000 0.582 -0.155 0.473 2.215 1.214 -0.422 -0.551 2.548 0.838 -1.171 -1.166 0.000 2.051 1.215 1.062 1.091 0.725 0.896 1.091
|
|
||||||
0 0.373 -0.600 1.291 2.573 0.207 0.765 -0.209 1.667 0.000 0.668 0.724 -1.499 0.000 1.045 -0.338 -0.754 2.548 0.558 -0.469 0.029 3.102 0.868 0.939 1.124 0.519 0.383 0.636 0.838
|
|
||||||
0 0.791 0.336 -0.307 0.494 1.213 1.158 0.336 1.081 2.173 0.918 1.289 -0.449 0.000 0.735 -0.521 -0.969 0.000 1.052 0.499 -1.188 3.102 0.699 1.013 0.987 0.622 1.050 0.712 0.661
|
|
||||||
0 1.321 0.856 0.464 0.202 0.901 1.144 0.120 -1.651 0.000 0.803 0.577 -0.509 2.215 0.695 -0.114 0.423 2.548 0.621 1.852 -0.420 0.000 0.697 0.964 0.983 0.527 0.659 0.719 0.729
|
|
||||||
0 0.563 2.081 0.913 0.982 -0.533 0.549 -0.481 -1.730 0.000 0.962 0.921 0.569 2.215 0.731 1.184 -0.679 1.274 0.918 0.931 -1.432 0.000 1.008 0.919 0.993 0.895 0.819 0.810 0.878
|
|
||||||
1 1.148 0.345 0.953 0.921 0.617 0.991 1.103 -0.484 0.000 0.970 1.978 1.525 0.000 1.150 0.689 -0.757 2.548 0.517 0.995 1.245 0.000 1.093 1.140 0.998 1.006 0.756 0.864 0.838
|
|
||||||
1 1.400 0.128 -1.695 1.169 1.070 1.094 -0.345 -0.249 0.000 1.224 0.364 -0.036 2.215 1.178 0.530 -1.544 0.000 1.334 0.933 1.604 0.000 0.560 1.267 1.073 0.716 0.780 0.832 0.792
|
|
||||||
0 0.330 -2.133 1.403 0.628 0.379 1.686 -0.995 0.030 1.087 2.071 0.127 -0.457 0.000 4.662 -0.855 1.477 0.000 2.072 -0.917 -1.416 3.102 5.403 3.074 0.977 0.936 1.910 2.325 1.702
|
|
||||||
0 0.989 0.473 0.968 1.970 1.368 0.844 0.574 -0.290 2.173 0.866 -0.345 -1.019 0.000 1.130 0.605 -0.752 0.000 0.956 -0.888 0.870 3.102 0.885 0.886 0.982 1.157 1.201 1.100 1.068
|
|
||||||
1 0.773 0.418 0.753 1.388 1.070 1.104 -0.378 -0.758 0.000 1.027 0.397 -0.496 2.215 1.234 0.027 1.084 2.548 0.936 0.209 1.677 0.000 1.355 1.020 0.983 0.550 1.206 0.916 0.931
|
|
||||||
0 0.319 2.015 1.534 0.570 -1.134 0.632 0.124 0.757 0.000 0.477 0.598 -1.109 1.107 0.449 0.438 -0.755 2.548 0.574 -0.659 0.691 0.000 0.440 0.749 0.985 0.517 0.158 0.505 0.522
|
|
||||||
0 1.215 1.453 -1.386 1.276 1.298 0.643 0.570 -0.196 2.173 0.588 2.104 0.498 0.000 0.617 -0.296 -0.801 2.548 0.452 0.110 0.313 0.000 0.815 0.953 1.141 1.166 0.547 0.892 0.807
|
|
||||||
1 1.257 -1.869 -0.060 0.265 0.653 1.527 -0.346 1.163 2.173 0.758 -2.119 -0.604 0.000 1.473 -1.133 -1.290 2.548 0.477 -0.428 -0.066 0.000 0.818 0.841 0.984 1.446 1.729 1.211 1.054
|
|
||||||
1 1.449 0.464 1.585 1.418 -1.488 1.540 0.942 0.087 0.000 0.898 0.402 -0.631 2.215 0.753 0.039 -1.729 0.000 0.859 0.849 -1.054 0.000 0.791 0.677 0.995 0.687 0.527 0.703 0.606
|
|
||||||
1 1.084 -1.997 0.900 1.333 1.024 0.872 -0.864 -1.500 2.173 1.072 -0.813 -0.421 2.215 0.924 0.478 0.304 0.000 0.992 -0.398 -1.022 0.000 0.741 1.085 0.980 1.221 1.176 1.032 0.961
|
|
||||||
0 1.712 1.129 0.125 1.120 -1.402 1.749 0.951 -1.575 2.173 1.711 0.445 0.578 0.000 1.114 0.234 -1.011 0.000 1.577 -0.088 0.086 3.102 2.108 1.312 1.882 1.597 2.009 1.441 1.308
|
|
||||||
0 0.530 0.248 1.622 1.450 -1.012 1.221 -1.154 -0.763 2.173 1.698 -0.586 0.733 0.000 0.889 1.042 1.038 1.274 0.657 0.008 0.701 0.000 0.430 1.005 0.983 0.930 2.264 1.357 1.146
|
|
||||||
1 0.921 1.735 0.883 0.699 -1.614 0.821 1.463 0.319 1.087 1.099 0.814 -1.600 2.215 1.375 0.702 -0.691 0.000 0.869 1.326 -0.790 0.000 0.980 0.900 0.988 0.832 1.452 0.816 0.709
|
|
||||||
0 2.485 -0.823 -0.297 0.886 -1.404 0.989 0.835 1.615 2.173 0.382 0.588 -0.224 0.000 1.029 -0.456 1.546 2.548 0.613 -0.359 -0.789 0.000 0.768 0.977 1.726 2.007 0.913 1.338 1.180
|
|
||||||
1 0.657 -0.069 -0.078 1.107 1.549 0.804 1.335 -1.630 2.173 1.271 0.481 0.153 1.107 1.028 0.144 -0.762 0.000 1.098 0.132 1.570 0.000 0.830 0.979 1.175 1.069 1.624 1.000 0.868
|
|
||||||
1 2.032 0.329 -1.003 0.493 -0.136 1.159 -0.224 0.750 1.087 0.396 0.546 0.587 0.000 0.620 1.805 0.982 0.000 1.236 0.744 -1.621 0.000 0.930 1.200 0.988 0.482 0.771 0.887 0.779
|
|
||||||
0 0.524 -1.319 0.634 0.471 1.221 0.599 -0.588 -0.461 0.000 1.230 -1.504 -1.517 1.107 1.436 -0.035 0.104 2.548 0.629 1.997 -1.282 0.000 2.084 1.450 0.984 1.084 1.827 1.547 1.213
|
|
||||||
1 0.871 0.618 -1.544 0.718 0.186 1.041 -1.180 0.434 2.173 1.133 1.558 -1.301 0.000 0.452 -0.595 0.522 0.000 0.665 0.567 0.130 3.102 1.872 1.114 1.095 1.398 0.979 1.472 1.168
|
|
||||||
1 3.308 1.037 -0.634 0.690 -0.619 1.975 0.949 1.280 0.000 0.826 0.546 -0.139 2.215 0.635 -0.045 0.427 0.000 1.224 0.112 1.339 3.102 1.756 1.050 0.992 0.738 0.903 0.968 1.238
|
|
||||||
0 0.588 2.104 -0.872 1.136 1.743 0.842 0.638 0.015 0.000 0.481 0.928 1.000 2.215 0.595 0.125 1.429 0.000 0.951 -1.140 -0.511 3.102 1.031 1.057 0.979 0.673 1.064 1.001 0.891
|
|
||||||
0 0.289 0.823 0.013 0.615 -1.601 0.177 2.403 -0.015 0.000 0.258 1.151 1.036 2.215 0.694 0.553 -1.326 2.548 0.411 0.366 0.106 0.000 0.482 0.562 0.989 0.670 0.404 0.516 0.561
|
|
||||||
1 0.294 -0.660 -1.162 1.752 0.384 0.860 0.513 1.119 0.000 2.416 0.107 -1.342 0.000 1.398 0.361 -0.350 2.548 1.126 -0.902 0.040 1.551 0.650 1.125 0.988 0.531 0.843 0.912 0.911
|
|
||||||
0 0.599 -0.616 1.526 1.381 0.507 0.955 -0.646 -0.085 2.173 0.775 -0.533 1.116 2.215 0.789 -0.136 -1.176 0.000 2.449 1.435 -1.433 0.000 1.692 1.699 1.000 0.869 1.119 1.508 1.303
|
|
||||||
1 1.100 -1.174 -1.114 1.601 -1.576 1.056 -1.343 0.547 2.173 0.555 0.367 0.592 2.215 0.580 -1.862 -0.914 0.000 0.904 0.508 -0.444 0.000 1.439 1.105 0.986 1.408 1.104 1.190 1.094
|
|
||||||
1 2.237 -0.701 1.470 0.719 -0.199 0.745 -0.132 -0.737 1.087 0.976 -0.227 0.093 2.215 0.699 0.057 1.133 0.000 0.661 0.573 -0.679 0.000 0.785 0.772 1.752 1.235 0.856 0.990 0.825
|
|
||||||
1 0.455 -0.880 -1.482 1.260 -0.178 1.499 0.158 1.022 0.000 1.867 -0.435 -0.675 2.215 1.234 0.783 1.586 0.000 0.641 -0.454 -0.409 3.102 1.002 0.964 0.986 0.761 0.240 1.190 0.995
|
|
||||||
1 1.158 -0.778 -0.159 0.823 1.641 1.341 -0.830 -1.169 2.173 0.840 -1.554 0.934 0.000 0.693 0.488 -1.218 2.548 1.042 1.395 0.276 0.000 0.946 0.785 1.350 1.079 0.893 1.267 1.151
|
|
||||||
1 0.902 -0.078 -0.055 0.872 -0.012 0.843 1.276 1.739 2.173 0.838 1.492 0.918 0.000 0.626 0.904 -0.648 2.548 0.412 -2.027 -0.883 0.000 2.838 1.664 0.988 1.803 0.768 1.244 1.280
|
|
||||||
1 0.649 -1.028 -1.521 1.097 0.774 1.216 -0.383 -0.318 2.173 1.643 -0.285 -1.705 0.000 0.911 -0.091 0.341 0.000 0.592 0.537 0.732 3.102 0.911 0.856 1.027 1.160 0.874 0.986 0.893
|
|
||||||
1 1.192 1.846 -0.781 1.326 -0.747 1.550 1.177 1.366 0.000 1.196 0.151 0.387 2.215 0.527 2.261 -0.190 0.000 0.390 1.474 0.381 0.000 0.986 1.025 1.004 1.392 0.761 0.965 1.043
|
|
||||||
0 0.438 -0.358 -1.549 0.836 0.436 0.818 0.276 -0.708 2.173 0.707 0.826 0.392 0.000 1.050 1.741 -1.066 0.000 1.276 -1.583 0.842 0.000 1.475 1.273 0.986 0.853 1.593 1.255 1.226
|
|
||||||
1 1.083 0.142 1.701 0.605 -0.253 1.237 0.791 1.183 2.173 0.842 2.850 -0.082 0.000 0.724 -0.464 -0.694 0.000 1.499 0.456 -0.226 3.102 0.601 0.799 1.102 0.995 1.389 1.013 0.851
|
|
||||||
0 0.828 1.897 -0.615 0.572 -0.545 0.572 0.461 0.464 2.173 0.393 0.356 1.069 2.215 1.840 0.088 1.500 0.000 0.407 -0.663 -0.787 0.000 0.950 0.965 0.979 0.733 0.363 0.618 0.733
|
|
||||||
0 0.735 1.438 1.197 1.123 -0.214 0.641 0.949 0.858 0.000 1.162 0.524 -0.896 2.215 0.992 0.454 -1.475 2.548 0.902 1.079 0.019 0.000 0.822 0.917 1.203 1.032 0.569 0.780 0.764
|
|
||||||
0 0.437 -2.102 0.044 1.779 -1.042 1.231 -0.181 -0.515 1.087 2.666 0.863 1.466 2.215 1.370 0.345 -1.371 0.000 0.906 0.363 1.611 0.000 1.140 1.362 1.013 3.931 3.004 2.724 2.028
|
|
||||||
1 0.881 1.814 -0.987 0.384 0.800 2.384 1.422 0.640 0.000 1.528 0.292 -0.962 1.107 2.126 -0.371 -1.401 2.548 0.700 0.109 0.203 0.000 0.450 0.813 0.985 0.956 1.013 0.993 0.774
|
|
||||||
1 0.630 0.408 0.152 0.194 0.316 0.710 -0.824 -0.358 2.173 0.741 0.535 -0.851 2.215 0.933 0.406 1.148 0.000 0.523 -0.479 -0.625 0.000 0.873 0.960 0.988 0.830 0.921 0.711 0.661
|
|
||||||
1 0.870 -0.448 -1.134 0.616 0.135 0.600 0.649 -0.622 2.173 0.768 0.709 -0.123 0.000 1.308 0.500 1.468 0.000 1.973 -0.286 1.462 3.102 0.909 0.944 0.990 0.835 1.250 0.798 0.776
|
|
||||||
0 1.290 0.552 1.330 0.615 -1.353 0.661 0.240 -0.393 0.000 0.531 0.053 -1.588 0.000 0.675 0.839 -0.345 1.274 1.597 0.020 0.536 3.102 1.114 0.964 0.987 0.783 0.675 0.662 0.675
|
|
||||||
1 0.943 0.936 1.068 1.373 0.671 2.170 -2.011 -1.032 0.000 0.640 0.361 -0.806 0.000 2.239 -0.083 0.590 2.548 1.224 0.646 -1.723 0.000 0.879 0.834 0.981 1.436 0.568 0.916 0.931
|
|
||||||
1 0.431 1.686 -1.053 0.388 1.739 0.457 -0.471 -0.743 2.173 0.786 1.432 -0.547 2.215 0.537 -0.413 1.256 0.000 0.413 2.311 -0.408 0.000 1.355 1.017 0.982 0.689 1.014 0.821 0.715
|
|
||||||
0 1.620 -0.055 -0.862 1.341 -1.571 0.634 -0.906 0.935 2.173 0.501 -2.198 -0.525 0.000 0.778 -0.708 -0.060 0.000 0.988 -0.621 0.489 3.102 0.870 0.956 1.216 0.992 0.336 0.871 0.889
|
|
||||||
1 0.549 0.304 -1.443 1.309 -0.312 1.116 0.644 1.519 2.173 1.078 -0.303 -0.736 0.000 1.261 0.387 0.628 2.548 0.945 -0.190 0.090 0.000 0.893 1.043 1.000 1.124 1.077 1.026 0.886
|
|
||||||
0 0.412 -0.618 -1.486 1.133 -0.665 0.646 0.436 1.520 0.000 0.993 0.976 0.106 2.215 0.832 0.091 0.164 2.548 0.672 -0.650 1.256 0.000 0.695 1.131 0.991 1.017 0.455 1.226 1.087
|
|
||||||
0 1.183 -0.084 1.644 1.389 0.967 0.843 0.938 -0.670 0.000 0.480 0.256 0.123 2.215 0.437 1.644 0.491 0.000 0.501 -0.416 0.101 3.102 1.060 0.804 1.017 0.775 0.173 0.535 0.760
|
|
||||||
0 1.629 -1.486 -0.683 2.786 -0.492 1.347 -2.638 1.453 0.000 1.857 0.208 0.873 0.000 0.519 -1.265 -1.602 1.274 0.903 -1.102 -0.329 1.551 6.892 3.522 0.998 0.570 0.477 2.039 2.006
|
|
||||||
1 2.045 -0.671 -1.235 0.490 -0.952 0.525 -1.252 1.289 0.000 1.088 -0.993 0.648 2.215 0.975 -0.109 -0.254 2.548 0.556 -1.095 -0.194 0.000 0.803 0.861 0.980 1.282 0.945 0.925 0.811
|
|
||||||
0 0.448 -0.058 -0.974 0.945 -1.633 1.181 -1.139 0.266 2.173 1.118 -0.761 1.502 1.107 1.706 0.585 -0.680 0.000 0.487 -1.951 0.945 0.000 2.347 1.754 0.993 1.161 1.549 1.414 1.176
|
|
||||||
0 0.551 0.519 0.448 2.183 1.293 1.220 0.628 -0.627 2.173 1.019 -0.002 -0.652 0.000 1.843 -0.386 1.042 2.548 0.400 -1.102 -1.014 0.000 0.648 0.792 1.049 0.888 2.132 1.262 1.096
|
|
||||||
0 1.624 0.488 1.403 0.760 0.559 0.812 0.777 -1.244 2.173 0.613 0.589 -0.030 2.215 0.692 1.058 0.683 0.000 1.054 1.165 -0.765 0.000 0.915 0.875 1.059 0.821 0.927 0.792 0.721
|
|
||||||
1 0.774 0.444 1.257 0.515 -0.689 0.515 1.448 -1.271 0.000 0.793 0.118 0.811 1.107 0.679 0.326 -0.426 0.000 1.066 -0.865 -0.049 3.102 0.960 1.046 0.986 0.716 0.772 0.855 0.732
|
|
||||||
1 2.093 -1.240 1.615 0.918 -1.202 1.412 -0.541 0.640 1.087 2.019 0.872 -0.639 0.000 0.672 -0.936 0.972 0.000 0.896 0.235 0.212 0.000 0.810 0.700 1.090 0.797 0.862 1.049 0.874
|
|
||||||
1 0.908 1.069 0.283 0.400 1.293 0.609 1.452 -1.136 0.000 0.623 0.417 -0.098 2.215 1.023 0.775 1.054 1.274 0.706 2.346 -1.305 0.000 0.744 1.006 0.991 0.606 0.753 0.796 0.753
|
|
||||||
0 0.403 -1.328 -0.065 0.901 1.052 0.708 -0.354 -0.718 2.173 0.892 0.633 1.684 2.215 0.999 -1.205 0.941 0.000 0.930 1.072 -0.809 0.000 2.105 1.430 0.989 0.838 1.147 1.042 0.883
|
|
||||||
0 1.447 0.453 0.118 1.731 0.650 0.771 0.446 -1.564 0.000 0.973 -2.014 0.354 0.000 1.949 -0.643 -1.531 1.274 1.106 -0.334 -1.163 0.000 0.795 0.821 1.013 1.699 0.918 1.118 1.018
|
|
||||||
1 1.794 0.123 -0.454 0.057 1.489 0.966 -1.190 1.090 1.087 0.539 -0.535 1.035 0.000 1.096 -1.069 -1.236 2.548 0.659 -1.196 -0.283 0.000 0.803 0.756 0.985 1.343 1.109 0.993 0.806
|
|
||||||
0 1.484 -2.047 0.813 0.591 -0.295 0.923 0.312 -1.164 2.173 0.654 -0.316 0.752 2.215 0.599 1.966 -1.128 0.000 0.626 -0.304 -1.431 0.000 1.112 0.910 1.090 0.986 1.189 1.350 1.472
|
|
||||||
0 0.417 -2.016 0.849 1.817 0.040 1.201 -1.676 -1.394 0.000 0.792 0.537 0.641 2.215 0.794 -1.222 0.187 0.000 0.825 -0.217 1.334 3.102 1.470 0.931 0.987 1.203 0.525 0.833 0.827
|
|
||||||
1 0.603 1.009 0.033 0.486 1.225 0.884 -0.617 -1.058 0.000 0.500 -1.407 -0.567 0.000 1.476 -0.876 0.605 2.548 0.970 0.560 1.092 3.102 0.853 1.153 0.988 0.846 0.920 0.944 0.835
|
|
||||||
1 1.381 -0.326 0.552 0.417 -0.027 1.030 -0.835 -1.287 2.173 0.941 -0.421 1.519 2.215 0.615 -1.650 0.377 0.000 0.606 0.644 0.650 0.000 1.146 0.970 0.990 1.191 0.884 0.897 0.826
|
|
||||||
1 0.632 1.200 -0.703 0.438 -1.700 0.779 -0.731 0.958 1.087 0.605 0.393 -1.376 0.000 0.670 -0.827 -1.315 2.548 0.626 -0.501 0.417 0.000 0.904 0.903 0.998 0.673 0.803 0.722 0.640
|
|
||||||
1 1.561 -0.569 1.580 0.329 0.237 1.059 0.731 0.415 2.173 0.454 0.016 -0.828 0.000 0.587 0.008 -0.291 1.274 0.597 1.119 1.191 0.000 0.815 0.908 0.988 0.733 0.690 0.892 0.764
|
|
||||||
1 2.102 0.087 0.449 1.164 -0.390 1.085 -0.408 -1.116 2.173 0.578 0.197 -0.137 0.000 1.202 0.917 1.523 0.000 0.959 -0.832 1.404 3.102 1.380 1.109 1.486 1.496 0.886 1.066 1.025
|
|
||||||
1 1.698 -0.489 -0.552 0.976 -1.009 1.620 -0.721 0.648 1.087 1.481 -1.860 -1.354 0.000 1.142 -1.140 1.401 2.548 1.000 -1.274 -0.158 0.000 1.430 1.130 0.987 1.629 1.154 1.303 1.223
|
|
||||||
1 1.111 -0.249 -1.457 0.421 0.939 0.646 -2.076 0.362 0.000 1.315 0.796 -1.436 2.215 0.780 0.130 0.055 0.000 1.662 -0.834 0.461 0.000 0.920 0.948 0.990 1.046 0.905 1.493 1.169
|
|
||||||
1 0.945 0.390 -1.159 1.675 0.437 0.356 0.261 0.543 1.087 0.574 0.838 1.599 2.215 0.496 -1.220 -0.022 0.000 0.558 -2.454 1.440 0.000 0.763 0.983 1.728 1.000 0.578 0.922 1.003
|
|
||||||
1 2.076 0.014 -1.314 0.854 -0.306 3.446 1.341 0.598 0.000 2.086 0.227 -0.747 2.215 1.564 -0.216 1.649 2.548 0.965 -0.857 -1.062 0.000 0.477 0.734 1.456 1.003 1.660 1.001 0.908
|
|
||||||
1 1.992 0.192 -0.103 0.108 -1.599 0.938 0.595 -1.360 2.173 0.869 -1.012 1.432 0.000 1.302 0.850 0.436 2.548 0.487 1.051 -1.027 0.000 0.502 0.829 0.983 1.110 1.394 0.904 0.836
|
|
||||||
0 0.460 1.625 1.485 1.331 1.242 0.675 -0.329 -1.039 1.087 0.671 -1.028 -0.514 0.000 1.265 -0.788 0.415 1.274 0.570 -0.683 -1.738 0.000 0.725 0.758 1.004 1.024 1.156 0.944 0.833
|
|
||||||
0 0.871 0.839 -1.536 0.428 1.198 0.875 -1.256 -0.466 1.087 0.684 -0.768 0.150 0.000 0.556 -1.793 0.389 0.000 0.942 -1.126 1.339 1.551 0.624 0.734 0.986 1.357 0.960 1.474 1.294
|
|
||||||
1 0.951 1.651 0.576 1.273 1.495 0.834 0.048 -0.578 2.173 0.386 -0.056 -1.448 0.000 0.597 -0.196 0.162 2.548 0.524 1.649 1.625 0.000 0.737 0.901 1.124 1.014 0.556 1.039 0.845
|
|
||||||
1 1.049 -0.223 0.685 0.256 -1.191 2.506 0.238 -0.359 0.000 1.510 -0.904 1.158 1.107 2.733 -0.902 1.679 2.548 0.407 -0.474 -1.572 0.000 1.513 2.472 0.982 1.238 0.978 1.985 1.510
|
|
||||||
0 0.455 -0.028 0.265 1.286 1.373 0.459 0.331 -0.922 0.000 0.343 0.634 0.430 0.000 0.279 -0.084 -0.272 0.000 0.475 0.926 -0.123 3.102 0.803 0.495 0.987 0.587 0.211 0.417 0.445
|
|
||||||
1 2.074 0.388 0.878 1.110 1.557 1.077 -0.226 -0.295 2.173 0.865 -0.319 -1.116 2.215 0.707 -0.835 0.722 0.000 0.632 -0.608 -0.728 0.000 0.715 0.802 1.207 1.190 0.960 1.143 0.926
|
|
||||||
1 1.390 0.265 1.196 0.919 -1.371 1.858 0.506 0.786 0.000 1.280 -1.367 -0.720 2.215 1.483 -0.441 -0.675 2.548 1.076 0.294 -0.539 0.000 1.126 0.830 1.155 1.551 0.702 1.103 0.933
|
|
||||||
1 1.014 -0.079 1.597 1.038 -0.281 1.135 -0.722 -0.177 2.173 0.544 -1.475 -1.501 0.000 1.257 -1.315 1.212 0.000 0.496 -0.060 1.180 1.551 0.815 0.611 1.411 1.110 0.792 0.846 0.853
|
|
||||||
0 0.335 1.267 -1.154 2.011 -0.574 0.753 0.618 1.411 0.000 0.474 0.748 0.681 2.215 0.608 -0.446 -0.354 2.548 0.399 1.295 -0.581 0.000 0.911 0.882 0.975 0.832 0.598 0.580 0.678
|
|
||||||
1 0.729 -0.189 1.182 0.293 1.310 0.412 0.459 -0.632 0.000 0.869 -1.128 -0.625 2.215 1.173 -0.893 0.478 2.548 0.584 -2.394 -1.727 0.000 2.016 1.272 0.995 1.034 0.905 0.966 1.038
|
|
||||||
1 1.225 -1.215 -0.088 0.881 -0.237 0.600 -0.976 1.462 2.173 0.876 0.506 1.583 2.215 0.718 1.228 -0.031 0.000 0.653 -1.292 1.216 0.000 0.838 1.108 0.981 1.805 0.890 1.251 1.197
|
|
||||||
1 2.685 -0.444 0.847 0.253 0.183 0.641 -1.541 -0.873 2.173 0.417 2.874 -0.551 0.000 0.706 -1.431 0.764 0.000 1.390 -0.596 -1.397 0.000 0.894 0.829 0.993 0.789 0.654 0.883 0.746
|
|
||||||
0 0.638 -0.481 0.683 1.457 -1.024 0.707 -1.338 1.498 0.000 0.980 0.518 0.289 2.215 0.964 -0.531 -0.423 0.000 0.694 -0.654 -1.314 3.102 0.807 1.283 1.335 0.658 0.907 0.797 0.772
|
|
||||||
1 1.789 -0.765 -0.732 0.421 -0.020 1.142 -1.353 1.439 2.173 0.725 -1.518 -1.261 0.000 0.812 -2.597 -0.463 0.000 1.203 -0.120 1.001 0.000 0.978 0.673 0.985 1.303 1.400 1.078 0.983
|
|
||||||
1 0.784 -1.431 1.724 0.848 0.559 0.615 -1.643 -1.456 0.000 1.339 -0.513 0.040 2.215 0.394 -2.483 1.304 0.000 0.987 0.889 -0.339 0.000 0.732 0.713 0.987 0.973 0.705 0.875 0.759
|
|
||||||
1 0.911 1.098 -1.289 0.421 0.823 1.218 -0.503 0.431 0.000 0.775 0.432 -1.680 0.000 0.855 -0.226 -0.460 2.548 0.646 -0.947 -1.243 1.551 2.201 1.349 0.985 0.730 0.451 0.877 0.825
|
|
||||||
1 0.959 0.372 -0.269 1.255 0.702 1.151 0.097 0.805 2.173 0.993 1.011 0.767 2.215 1.096 0.185 0.381 0.000 1.001 -0.205 0.059 0.000 0.979 0.997 1.168 0.796 0.771 0.839 0.776
|
|
||||||
0 0.283 -1.864 -1.663 0.219 1.624 0.955 -1.213 0.932 2.173 0.889 0.395 -0.268 0.000 0.597 -1.083 -0.921 2.548 0.584 1.325 -1.072 0.000 0.856 0.927 0.996 0.937 0.936 1.095 0.892
|
|
||||||
0 2.017 -0.488 -0.466 1.029 -0.870 3.157 0.059 -0.343 2.173 3.881 0.872 1.502 1.107 3.631 1.720 0.963 0.000 0.633 -1.264 -1.734 0.000 4.572 3.339 1.005 1.407 5.590 3.614 3.110
|
|
||||||
1 1.088 0.414 -0.841 0.485 0.605 0.860 1.110 -0.568 0.000 1.152 -0.325 1.203 2.215 0.324 1.652 -0.104 0.000 0.510 1.095 -1.728 0.000 0.880 0.722 0.989 0.977 0.711 0.888 0.762
|
|
||||||
0 0.409 -1.717 0.712 0.809 -1.301 0.701 -1.529 -1.411 0.000 1.191 -0.582 0.438 2.215 1.147 0.813 -0.571 2.548 1.039 0.543 0.892 0.000 0.636 0.810 0.986 0.861 1.411 0.907 0.756
|
|
||||||
1 1.094 1.577 -0.988 0.497 -0.149 0.891 -2.459 1.034 0.000 0.646 0.792 -1.022 0.000 1.573 0.254 -0.053 2.548 1.428 0.190 -1.641 3.102 4.322 2.687 0.985 0.881 1.135 1.907 1.831
|
|
||||||
1 0.613 1.993 -0.280 0.544 0.931 0.909 1.526 1.559 0.000 0.840 1.473 -0.483 2.215 0.856 0.352 0.408 2.548 1.058 1.733 -1.396 0.000 0.801 1.066 0.984 0.639 0.841 0.871 0.748
|
|
||||||
0 0.958 -1.202 0.600 0.434 0.170 0.783 -0.214 1.319 0.000 0.835 -0.454 -0.615 2.215 0.658 -1.858 -0.891 0.000 0.640 0.172 -1.204 3.102 1.790 1.086 0.997 0.804 0.403 0.793 0.756
|
|
||||||
1 1.998 -0.238 0.972 0.058 0.266 0.759 1.576 -0.357 2.173 1.004 -0.349 -0.747 2.215 0.962 0.490 -0.453 0.000 1.592 0.661 -1.405 0.000 0.874 1.086 0.990 1.436 1.527 1.177 0.993
|
|
||||||
1 0.796 -0.171 -0.818 0.574 -1.625 1.201 -0.737 1.451 2.173 0.651 0.404 -0.452 0.000 1.150 -0.652 -0.120 0.000 1.008 -0.093 0.531 3.102 0.884 0.706 0.979 1.193 0.937 0.943 0.881
|
|
||||||
1 0.773 1.023 0.527 1.537 -0.201 2.967 -0.574 -1.534 2.173 2.346 -0.307 0.394 2.215 1.393 0.135 -0.027 0.000 3.015 0.187 0.516 0.000 0.819 1.260 0.982 2.552 3.862 2.179 1.786
|
|
||||||
0 1.823 1.008 -1.489 0.234 -0.962 0.591 0.461 0.996 2.173 0.568 -1.297 -0.410 0.000 0.887 2.157 1.194 0.000 2.079 0.369 -0.085 3.102 0.770 0.945 0.995 1.179 0.971 0.925 0.983
|
|
||||||
0 0.780 0.640 0.490 0.680 -1.301 0.715 -0.137 0.152 2.173 0.616 -0.831 1.668 0.000 1.958 0.528 -0.982 2.548 0.966 -1.551 0.462 0.000 1.034 1.079 1.008 0.827 1.369 1.152 0.983
|
|
||||||
1 0.543 0.801 1.543 1.134 -0.772 0.954 -0.849 0.410 1.087 0.851 -1.988 1.686 0.000 0.799 -0.912 -1.156 0.000 0.479 0.097 1.334 0.000 0.923 0.597 0.989 1.231 0.759 0.975 0.867
|
|
||||||
0 1.241 -0.014 0.129 1.158 0.670 0.445 -0.732 1.739 2.173 0.918 0.659 -1.340 2.215 0.557 2.410 -1.404 0.000 0.966 -1.545 -1.120 0.000 0.874 0.918 0.987 1.001 0.798 0.904 0.937
|
|
||||||
0 1.751 -0.266 -1.575 0.489 1.292 1.112 1.533 0.137 2.173 1.204 -0.414 -0.928 0.000 0.879 1.237 -0.415 2.548 1.479 1.469 0.913 0.000 2.884 1.747 0.989 1.742 0.600 1.363 1.293
|
|
||||||
1 1.505 1.208 -1.476 0.995 -0.836 2.800 -1.600 0.111 0.000 2.157 1.241 1.110 2.215 1.076 2.619 -0.913 0.000 1.678 2.204 -1.575 0.000 0.849 1.224 0.990 1.412 0.976 1.271 1.105
|
|
||||||
0 0.816 0.611 0.779 1.694 0.278 0.575 -0.787 1.592 2.173 1.148 1.076 -0.831 2.215 0.421 1.316 0.632 0.000 0.589 0.452 -1.466 0.000 0.779 0.909 0.990 1.146 1.639 1.236 0.949
|
|
||||||
1 0.551 -0.808 0.330 1.188 -0.294 0.447 -0.035 -0.993 0.000 0.432 -0.276 -0.481 2.215 1.959 -0.288 1.195 2.548 0.638 0.583 1.107 0.000 0.832 0.924 0.993 0.723 0.976 0.968 0.895
|
|
||||||
0 1.316 -0.093 0.995 0.860 -0.621 0.593 -0.560 -1.599 2.173 0.524 -0.318 -0.240 2.215 0.566 0.759 -0.368 0.000 0.483 -2.030 -1.104 0.000 1.468 1.041 1.464 0.811 0.778 0.690 0.722
|
|
||||||
1 1.528 0.067 -0.855 0.959 -1.464 1.143 -0.082 1.023 0.000 0.702 -0.763 -0.244 0.000 0.935 -0.881 0.206 2.548 0.614 -0.831 1.657 3.102 1.680 1.105 0.983 1.078 0.559 0.801 0.809
|
|
||||||
0 0.558 -0.833 -0.598 1.436 -1.724 1.316 -0.661 1.593 2.173 1.148 -0.503 -0.132 1.107 1.584 -0.125 0.380 0.000 1.110 -1.216 -0.181 0.000 1.258 0.860 1.053 0.790 1.814 1.159 1.007
|
|
||||||
1 0.819 0.879 1.221 0.598 -1.450 0.754 0.417 -0.369 2.173 0.477 1.199 0.274 0.000 1.073 0.368 0.273 2.548 1.599 2.047 1.690 0.000 0.933 0.984 0.983 0.788 0.613 0.728 0.717
|
|
||||||
0 0.981 -1.007 0.489 0.923 1.261 0.436 -0.698 -0.506 2.173 0.764 -1.105 -1.241 2.215 0.577 -2.573 -0.036 0.000 0.565 -1.628 1.610 0.000 0.688 0.801 0.991 0.871 0.554 0.691 0.656
|
|
||||||
0 2.888 0.568 -1.416 1.461 -1.157 1.756 -0.900 0.522 0.000 0.657 0.409 1.076 2.215 1.419 0.672 -0.019 0.000 1.436 -0.184 -0.980 3.102 0.946 0.919 0.995 1.069 0.890 0.834 0.856
|
|
||||||
1 0.522 1.805 -0.963 1.136 0.418 0.727 -0.195 -1.695 2.173 0.309 2.559 -0.178 0.000 0.521 1.794 0.919 0.000 0.788 0.174 -0.406 3.102 0.555 0.729 1.011 1.385 0.753 0.927 0.832
|
|
||||||
1 0.793 -0.162 -1.643 0.634 0.337 0.898 -0.633 1.689 0.000 0.806 -0.826 -0.356 2.215 0.890 -0.142 -1.268 0.000 1.293 0.574 0.725 0.000 0.833 1.077 0.988 0.721 0.679 0.867 0.753
|
|
||||||
0 1.298 1.098 0.280 0.371 -0.373 0.855 -0.306 -1.186 0.000 0.977 -0.421 1.003 0.000 0.978 0.956 -1.249 2.548 0.735 0.577 -0.037 3.102 0.974 1.002 0.992 0.549 0.587 0.725 0.954
|
|
||||||
1 0.751 -0.520 -1.653 0.168 -0.419 0.878 -1.023 -1.364 2.173 1.310 -0.667 0.863 0.000 1.196 -0.827 0.358 0.000 1.154 -0.165 -0.360 1.551 0.871 0.950 0.983 0.907 0.955 0.959 0.874
|
|
||||||
0 1.730 0.666 -1.432 0.446 1.302 0.921 -0.203 0.621 0.000 1.171 -0.365 -0.611 1.107 0.585 0.807 1.150 0.000 0.415 -0.843 1.311 0.000 0.968 0.786 0.986 1.059 0.371 0.790 0.848
|
|
||||||
1 0.596 -1.486 0.690 1.045 -1.344 0.928 0.867 0.820 2.173 0.610 0.999 -1.329 2.215 0.883 -0.001 -0.106 0.000 1.145 2.184 -0.808 0.000 2.019 1.256 1.056 1.751 1.037 1.298 1.518
|
|
||||||
1 0.656 -1.993 -0.519 1.643 -0.143 0.815 0.256 1.220 1.087 0.399 -1.184 -1.458 0.000 0.738 1.361 -1.443 0.000 0.842 0.033 0.293 0.000 0.910 0.891 0.993 0.668 0.562 0.958 0.787
|
|
||||||
1 1.127 -0.542 0.645 0.318 -1.496 0.661 -0.640 0.369 2.173 0.992 0.358 1.702 0.000 1.004 0.316 -1.109 0.000 1.616 -0.936 -0.707 1.551 0.875 1.191 0.985 0.651 0.940 0.969 0.834
|
|
||||||
0 0.916 -1.423 -1.490 1.248 -0.538 0.625 -0.535 -0.174 0.000 0.769 -0.389 1.608 2.215 0.667 -1.138 -1.738 1.274 0.877 -0.019 0.482 0.000 0.696 0.917 1.121 0.678 0.347 0.647 0.722
|
|
||||||
1 2.756 -0.637 -1.715 1.331 1.124 0.913 -0.296 -0.491 0.000 0.983 -0.831 0.000 2.215 1.180 -0.428 0.742 0.000 1.113 0.005 -1.157 1.551 1.681 1.096 1.462 0.976 0.917 1.009 1.040
|
|
||||||
0 0.755 1.754 0.701 2.111 0.256 1.243 0.057 -1.502 2.173 0.565 -0.034 -1.078 1.107 0.529 1.696 -1.090 0.000 0.665 0.292 0.107 0.000 0.870 0.780 0.990 2.775 0.465 1.876 1.758
|
|
||||||
1 0.593 -0.762 1.743 0.908 0.442 0.773 -1.357 -0.768 2.173 0.432 1.421 1.236 0.000 0.579 0.291 -0.403 0.000 0.966 -0.309 1.016 3.102 0.893 0.743 0.989 0.857 1.030 0.943 0.854
|
|
||||||
1 0.891 -1.151 -1.269 0.504 -0.622 0.893 -0.549 0.700 0.000 0.828 -0.825 0.154 2.215 1.083 0.632 -1.141 0.000 1.059 -0.557 1.526 3.102 2.117 1.281 0.987 0.819 0.802 0.917 0.828
|
|
||||||
1 2.358 -0.248 0.080 0.747 -0.975 1.019 1.374 1.363 0.000 0.935 0.127 -1.707 2.215 0.312 -0.827 0.017 0.000 0.737 1.059 -0.327 0.000 0.716 0.828 1.495 0.953 0.704 0.880 0.745
|
|
||||||
0 0.660 -0.017 -1.138 0.453 1.002 0.645 0.518 0.703 2.173 0.751 0.705 -0.592 2.215 0.744 -0.909 -1.596 0.000 0.410 -1.135 0.481 0.000 0.592 0.922 0.989 0.897 0.948 0.777 0.701
|
|
||||||
1 0.718 0.518 0.225 1.710 -0.022 1.888 -0.424 1.092 0.000 4.134 0.185 -1.366 0.000 1.415 1.293 0.242 2.548 2.351 0.264 -0.057 3.102 0.830 1.630 0.976 1.215 0.890 1.422 1.215
|
|
||||||
1 1.160 0.203 0.941 0.594 0.212 0.636 -0.556 0.679 2.173 1.089 -0.481 -1.008 1.107 1.245 -0.056 -1.357 0.000 0.587 1.007 0.056 0.000 1.106 0.901 0.987 0.786 1.224 0.914 0.837
|
|
||||||
1 0.697 0.542 0.619 0.985 1.481 0.745 0.415 1.644 2.173 0.903 0.495 -0.958 2.215 1.165 1.195 0.346 0.000 1.067 -0.881 -0.264 0.000 0.830 1.025 0.987 0.690 0.863 0.894 0.867
|
|
||||||
0 1.430 0.190 -0.700 0.246 0.518 1.302 0.660 -0.247 2.173 1.185 -0.539 1.504 0.000 1.976 -0.401 1.079 0.000 0.855 -0.958 -1.110 3.102 0.886 0.953 0.993 0.889 1.400 1.376 1.119
|
|
||||||
1 1.122 -0.795 0.202 0.397 -1.553 0.597 -1.459 -0.734 2.173 0.522 1.044 1.027 2.215 0.783 -1.243 1.701 0.000 0.371 1.737 0.199 0.000 1.719 1.176 0.988 0.723 1.583 1.063 0.914
|
|
||||||
0 1.153 0.526 1.236 0.266 0.001 1.139 -1.236 -0.585 2.173 1.337 -0.215 -1.356 2.215 1.780 1.129 0.902 0.000 1.608 -0.391 -0.161 0.000 1.441 1.633 0.990 1.838 1.516 1.635 1.373
|
|
||||||
1 0.760 1.012 0.758 0.937 0.051 0.941 0.687 -1.247 2.173 1.288 -0.743 0.822 0.000 1.552 1.782 -1.533 0.000 0.767 1.349 0.168 0.000 0.716 0.862 0.988 0.595 0.359 0.697 0.623
|
|
||||||
1 1.756 -1.469 1.395 1.345 -1.595 0.817 0.017 -0.741 2.173 0.483 -0.008 0.293 0.000 1.768 -0.663 0.438 1.274 1.202 -1.387 -0.222 0.000 1.022 1.058 0.992 1.407 1.427 1.356 1.133
|
|
||||||
0 0.397 0.582 -0.758 1.260 -1.735 0.889 -0.515 1.139 2.173 0.973 1.616 0.460 0.000 1.308 1.001 -0.709 2.548 0.858 0.995 -0.231 0.000 0.749 0.888 0.979 1.487 1.804 1.208 1.079
|
|
||||||
0 0.515 -0.984 0.425 1.114 -0.439 1.999 0.818 1.561 0.000 1.407 0.009 -0.380 0.000 1.332 0.230 0.397 0.000 1.356 -0.616 -1.057 3.102 0.978 1.017 0.990 1.118 0.862 0.835 0.919
|
|
||||||
1 1.368 -0.921 -0.866 0.842 -0.598 0.456 -1.176 1.219 1.087 0.419 -1.974 -0.819 0.000 0.791 -1.640 0.881 0.000 1.295 -0.782 0.442 3.102 0.945 0.761 0.974 0.915 0.535 0.733 0.651
|
|
||||||
0 2.276 0.134 0.399 2.525 0.376 1.111 -1.078 -1.571 0.000 0.657 2.215 -0.900 0.000 1.183 -0.662 -0.508 2.548 1.436 -0.517 0.960 3.102 0.569 0.931 0.993 1.170 0.967 0.879 1.207
|
|
||||||
0 0.849 0.907 0.124 0.652 1.585 0.715 0.355 -1.200 0.000 0.599 -0.892 1.301 0.000 1.106 1.151 0.582 0.000 1.895 -0.279 -0.568 3.102 0.881 0.945 0.998 0.559 0.649 0.638 0.660
|
|
||||||
1 2.105 0.248 -0.797 0.530 0.206 1.957 -2.175 0.797 0.000 1.193 0.637 -1.646 2.215 0.881 1.111 -1.046 0.000 0.872 -0.185 1.085 1.551 0.986 1.343 1.151 1.069 0.714 2.063 1.951
|
|
||||||
1 1.838 1.060 1.637 1.017 1.370 0.913 0.461 -0.609 1.087 0.766 -0.461 0.303 2.215 0.724 -0.061 0.886 0.000 0.941 1.123 -0.745 0.000 0.858 0.847 0.979 1.313 1.083 1.094 0.910
|
|
||||||
0 0.364 1.274 1.066 1.570 -0.394 0.485 0.012 -1.716 0.000 0.317 -1.233 0.534 2.215 0.548 -2.165 0.762 0.000 0.729 0.169 -0.318 3.102 0.892 0.944 1.013 0.594 0.461 0.688 0.715
|
|
||||||
1 0.503 1.343 -0.031 1.134 -1.204 0.590 -0.309 0.174 2.173 0.408 2.372 -0.628 0.000 1.850 0.400 1.147 2.548 0.664 -0.458 -0.885 0.000 1.445 1.283 0.989 1.280 1.118 1.127 1.026
|
|
||||||
0 1.873 0.258 0.103 2.491 0.530 1.678 0.644 -1.738 2.173 1.432 0.848 -1.340 0.000 0.621 1.323 -1.316 0.000 0.628 0.789 -0.206 1.551 0.426 0.802 1.125 0.688 1.079 1.338 1.239
|
|
||||||
1 0.826 -0.732 1.587 0.582 -1.236 0.495 0.757 -0.741 2.173 0.940 1.474 0.354 2.215 0.474 1.055 -1.657 0.000 0.415 1.758 0.841 0.000 0.451 0.578 0.984 0.757 0.922 0.860 0.696
|
|
||||||
0 0.935 -1.614 -0.597 0.299 1.223 0.707 -0.853 -1.026 0.000 0.751 0.007 -1.691 0.000 1.062 -0.125 0.976 2.548 0.877 1.275 0.646 0.000 0.962 1.074 0.980 0.608 0.726 0.741 0.662
|
|
||||||
1 0.643 0.542 -1.285 0.474 -0.366 0.667 -0.446 1.195 2.173 1.076 0.145 -0.126 0.000 0.970 -0.661 0.394 1.274 1.218 -0.184 -1.722 0.000 1.331 1.019 0.985 1.192 0.677 0.973 0.910
|
|
||||||
0 0.713 0.164 1.080 1.427 -0.460 0.960 -0.152 -0.940 2.173 1.427 -0.901 1.036 1.107 0.440 -1.269 -0.194 0.000 0.452 1.932 -0.532 0.000 1.542 1.210 1.374 1.319 1.818 1.220 1.050
|
|
||||||
0 0.876 -0.463 -1.224 2.458 -1.689 1.007 -0.752 0.398 0.000 2.456 -1.285 -0.152 1.107 1.641 1.838 1.717 0.000 0.458 0.194 0.488 3.102 4.848 2.463 0.986 1.981 0.974 2.642 2.258
|
|
||||||
1 0.384 -0.275 0.387 1.403 -0.994 0.620 -1.529 1.685 0.000 1.091 -1.644 1.078 0.000 0.781 -1.311 0.326 2.548 1.228 -0.728 -0.633 1.551 0.920 0.854 0.987 0.646 0.609 0.740 0.884
|
|
||||||
0 0.318 -1.818 -1.008 0.977 1.268 0.457 2.451 -1.522 0.000 0.881 1.351 0.461 2.215 0.929 0.239 -0.380 2.548 0.382 -0.613 1.330 0.000 1.563 1.193 0.994 0.829 0.874 0.901 1.026
|
|
||||||
1 0.612 -1.120 1.098 0.402 -0.480 0.818 0.188 1.511 0.000 0.800 -0.253 0.977 0.000 1.175 0.271 -1.289 1.274 2.531 0.226 -0.409 3.102 0.889 0.947 0.979 1.486 0.940 1.152 1.119
|
|
||||||
1 0.587 -0.737 -0.228 0.970 1.119 0.823 0.184 1.594 0.000 1.104 0.301 -0.818 2.215 0.819 0.712 -0.560 0.000 2.240 -0.419 0.340 3.102 1.445 1.103 0.988 0.715 1.363 1.019 0.926
|
|
||||||
0 1.030 -0.694 -1.638 0.893 -1.074 1.160 -0.766 0.485 0.000 1.632 -0.698 -1.142 2.215 1.050 -1.092 0.952 0.000 1.475 0.286 0.125 3.102 0.914 1.075 0.982 0.732 1.493 1.219 1.079
|
|
||||||
1 2.142 0.617 1.517 0.387 -0.862 0.345 1.203 -1.014 2.173 0.609 1.092 0.275 0.000 1.331 0.582 -0.183 2.548 0.557 1.540 -1.642 0.000 0.801 0.737 1.060 0.715 0.626 0.749 0.674
|
|
||||||
0 1.076 0.240 -0.246 0.871 -1.241 0.496 0.282 0.746 2.173 1.095 -0.648 1.100 2.215 0.446 -1.756 0.764 0.000 0.434 0.788 -0.991 0.000 1.079 0.868 1.047 0.818 0.634 0.795 0.733
|
|
||||||
0 1.400 0.901 -1.617 0.625 -0.163 0.661 -0.411 -1.616 2.173 0.685 0.524 0.425 0.000 0.881 -0.766 0.312 0.000 0.979 0.255 -0.667 3.102 0.898 1.105 1.253 0.730 0.716 0.738 0.795
|
|
||||||
0 3.302 1.132 1.051 0.658 0.768 1.308 0.251 -0.374 1.087 1.673 0.015 -0.898 0.000 0.688 -0.535 1.363 1.274 0.871 1.325 -1.583 0.000 1.646 1.249 0.995 1.919 1.288 1.330 1.329
|
|
||||||
0 1.757 0.202 0.750 0.767 -0.362 0.932 -1.033 -1.366 0.000 1.529 -1.012 -0.771 0.000 1.161 -0.287 0.059 0.000 2.185 1.147 1.099 3.102 0.795 0.529 1.354 1.144 1.491 1.319 1.161
|
|
||||||
0 1.290 0.905 -1.711 1.017 -0.695 1.008 -1.038 0.693 2.173 1.202 -0.595 0.187 0.000 1.011 0.139 -1.607 0.000 0.789 -0.613 -1.041 3.102 1.304 0.895 1.259 1.866 0.955 1.211 1.200
|
|
||||||
1 1.125 -0.004 1.694 0.373 0.329 0.978 0.640 -0.391 0.000 1.122 -0.376 1.521 2.215 0.432 2.413 -1.259 0.000 0.969 0.730 0.512 3.102 0.716 0.773 0.991 0.624 0.977 0.981 0.875
|
|
||||||
0 1.081 0.861 1.252 1.621 1.474 1.293 0.600 0.630 0.000 1.991 -0.090 -0.675 2.215 0.861 1.105 -0.201 0.000 1.135 2.489 -1.659 0.000 1.089 0.657 0.991 2.179 0.412 1.334 1.071
|
|
||||||
1 0.652 -0.294 1.241 1.034 0.490 1.033 0.551 -0.963 2.173 0.661 1.031 -1.654 2.215 1.376 -0.018 0.843 0.000 0.943 -0.329 -0.269 0.000 1.085 1.067 0.991 1.504 0.773 1.135 0.993
|
|
||||||
1 1.408 -1.028 -1.018 0.252 -0.242 0.465 -0.364 -0.200 0.000 1.466 0.669 0.739 1.107 1.031 0.415 -1.468 2.548 0.457 -1.091 -1.722 0.000 0.771 0.811 0.979 1.459 1.204 1.041 0.866
|
|
||||||
1 0.781 -1.143 -0.659 0.961 1.266 1.183 -0.686 0.119 2.173 1.126 -0.064 1.447 0.000 0.730 1.430 -1.535 0.000 1.601 0.513 1.658 0.000 0.871 1.345 1.184 1.058 0.620 1.107 0.978
|
|
||||||
1 1.300 -0.616 1.032 0.751 -0.731 0.961 -0.716 1.592 0.000 2.079 -1.063 -0.271 2.215 0.475 0.518 1.695 1.274 0.395 -2.204 0.349 0.000 1.350 0.983 1.369 1.265 1.428 1.135 0.982
|
|
||||||
1 0.833 0.809 1.657 1.637 1.019 0.705 1.077 -0.968 2.173 1.261 0.114 -0.298 1.107 1.032 0.017 0.236 0.000 0.640 -0.026 -1.598 0.000 0.894 0.982 0.981 1.250 1.054 1.018 0.853
|
|
||||||
1 1.686 -1.090 -0.301 0.890 0.557 1.304 -0.284 -1.393 2.173 0.388 2.118 0.513 0.000 0.514 -0.015 0.891 0.000 0.460 0.547 0.627 3.102 0.942 0.524 1.186 1.528 0.889 1.015 1.122
|
|
||||||
1 0.551 0.911 0.879 0.379 -0.796 1.154 -0.808 -0.966 0.000 1.168 -0.513 0.355 2.215 0.646 -1.309 0.773 0.000 0.544 -0.283 1.301 3.102 0.847 0.705 0.990 0.772 0.546 0.790 0.719
|
|
||||||
1 1.597 0.793 -1.119 0.691 -1.455 0.370 0.337 1.354 0.000 0.646 -1.005 0.732 2.215 1.019 0.040 0.209 0.000 0.545 0.958 0.239 3.102 0.962 0.793 0.994 0.719 0.745 0.812 0.739
|
|
||||||
0 1.033 -1.193 -0.452 0.247 0.970 0.503 -1.424 1.362 0.000 1.062 -0.416 -1.156 2.215 0.935 -0.023 0.555 2.548 0.410 -1.766 0.379 0.000 0.590 0.953 0.991 0.717 1.081 0.763 0.690
|
|
||||||
1 0.859 -1.004 1.521 0.781 -0.993 0.677 0.643 -0.338 2.173 0.486 0.409 1.283 0.000 0.679 0.110 0.285 0.000 0.715 -0.735 -0.157 1.551 0.702 0.773 0.984 0.627 0.633 0.694 0.643
|
|
||||||
0 0.612 -1.127 1.074 1.225 -0.426 0.927 -2.141 -0.473 0.000 1.290 -0.927 -1.085 2.215 1.183 1.981 -1.687 0.000 2.176 0.406 -1.581 0.000 0.945 0.651 1.170 0.895 1.604 1.179 1.142
|
|
||||||
1 0.535 0.321 -1.095 0.281 -0.960 0.876 -0.709 -0.076 0.000 1.563 -0.666 1.536 2.215 0.773 -0.321 0.435 0.000 0.682 -0.801 -0.952 3.102 0.711 0.667 0.985 0.888 0.741 0.872 0.758
|
|
||||||
1 0.745 1.586 1.578 0.863 -1.423 0.530 1.714 1.085 0.000 1.174 0.679 1.015 0.000 1.158 0.609 -1.186 2.548 1.851 0.832 -0.248 3.102 0.910 1.164 0.983 0.947 0.858 0.928 0.823
|
|
||||||
0 0.677 -1.014 -1.648 1.455 1.461 0.596 -2.358 0.517 0.000 0.800 0.849 -0.743 2.215 1.024 -0.282 -1.004 0.000 1.846 -0.977 0.378 3.102 2.210 1.423 0.982 1.074 1.623 1.417 1.258
|
|
||||||
1 0.815 -1.263 0.057 1.018 -0.208 0.339 -0.347 -1.646 2.173 1.223 0.600 -1.658 2.215 1.435 0.042 0.926 0.000 0.777 1.698 -0.698 0.000 1.022 1.058 1.000 0.784 0.477 0.886 0.836
|
|
||||||
0 3.512 -1.094 -0.220 0.338 -0.328 1.962 -1.099 1.544 1.087 1.461 -1.305 -0.922 2.215 1.219 -1.289 0.400 0.000 0.731 0.155 1.249 0.000 1.173 1.366 0.993 2.259 2.000 1.626 1.349
|
|
||||||
0 0.904 1.248 0.325 0.317 -1.624 0.685 -0.538 1.665 2.173 0.685 -2.145 -1.106 0.000 0.632 -1.460 1.017 0.000 1.085 -0.182 0.162 3.102 0.885 0.801 0.989 0.930 0.904 1.012 0.961
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,270 +0,0 @@
|
|||||||
{
|
|
||||||
"cells": [
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"Copyright (c) Microsoft Corporation. All rights reserved. \n",
|
|
||||||
"Licensed under the MIT License."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
""
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"# Use LightGBM Estimator in Azure Machine Learning\n",
|
|
||||||
"In this notebook we will demonstrate how to run a training job using LightGBM Estimator. [LightGBM](https://lightgbm.readthedocs.io/en/latest/) is a gradient boosting framework that uses tree based learning algorithms. "
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Prerequisites\n",
|
|
||||||
"This notebook uses azureml-contrib-gbdt package, if you don't already have the package, please install by uncommenting below cell."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"#!pip install azureml-contrib-gbdt"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"from azureml.core import Workspace, Run, Experiment\n",
|
|
||||||
"import shutil, os\n",
|
|
||||||
"from azureml.widgets import RunDetails\n",
|
|
||||||
"from azureml.contrib.gbdt import LightGBM\n",
|
|
||||||
"from azureml.train.dnn import Mpi\n",
|
|
||||||
"from azureml.core.compute import AmlCompute, ComputeTarget\n",
|
|
||||||
"from azureml.core.compute_target import ComputeTargetException"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"If you are using an AzureML Compute Instance, you are all set. Otherwise, go through the [configuration.ipynb](../../../configuration.ipynb) notebook to install the Azure Machine Learning Python SDK and create an Azure ML Workspace"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Set up machine learning resources"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"ws = Workspace.from_config()\n",
|
|
||||||
"\n",
|
|
||||||
"print('Workspace name: ' + ws.name, \n",
|
|
||||||
" 'Azure region: ' + ws.location, \n",
|
|
||||||
" 'Subscription id: ' + ws.subscription_id, \n",
|
|
||||||
" 'Resource group: ' + ws.resource_group, sep = '\\n')"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"cluster_vm_size = \"STANDARD_DS14_V2\"\n",
|
|
||||||
"cluster_min_nodes = 0\n",
|
|
||||||
"cluster_max_nodes = 20\n",
|
|
||||||
"cpu_cluster_name = 'TrainingCompute2' \n",
|
|
||||||
"\n",
|
|
||||||
"try:\n",
|
|
||||||
" cpu_cluster = AmlCompute(ws, cpu_cluster_name)\n",
|
|
||||||
" if cpu_cluster and type(cpu_cluster) is AmlCompute:\n",
|
|
||||||
" print('found compute target: ' + cpu_cluster_name)\n",
|
|
||||||
"except ComputeTargetException:\n",
|
|
||||||
" print('creating a new compute target...')\n",
|
|
||||||
" provisioning_config = AmlCompute.provisioning_configuration(vm_size = cluster_vm_size, \n",
|
|
||||||
" vm_priority = 'lowpriority', \n",
|
|
||||||
" min_nodes = cluster_min_nodes, \n",
|
|
||||||
" max_nodes = cluster_max_nodes)\n",
|
|
||||||
" cpu_cluster = ComputeTarget.create(ws, cpu_cluster_name, provisioning_config)\n",
|
|
||||||
" \n",
|
|
||||||
" # can poll for a minimum number of nodes and for a specific timeout. \n",
|
|
||||||
" # if no min node count is provided it will use the scale settings for the cluster\n",
|
|
||||||
" cpu_cluster.wait_for_completion(show_output=True, min_node_count=None, timeout_in_minutes=20)\n",
|
|
||||||
" \n",
|
|
||||||
" # For a more detailed view of current Azure Machine Learning Compute status, use get_status()\n",
|
|
||||||
" print(cpu_cluster.get_status().serialize())"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"From this point, you can either upload training data file directly or use Datastore for training data storage\n",
|
|
||||||
"## Upload training file from local"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"scripts_folder = \"scripts_folder\"\n",
|
|
||||||
"if not os.path.isdir(scripts_folder):\n",
|
|
||||||
" os.mkdir(scripts_folder)\n",
|
|
||||||
"shutil.copy('./train.conf', os.path.join(scripts_folder, 'train.conf'))\n",
|
|
||||||
"shutil.copy('./binary0.train', os.path.join(scripts_folder, 'binary0.train'))\n",
|
|
||||||
"shutil.copy('./binary1.train', os.path.join(scripts_folder, 'binary1.train'))\n",
|
|
||||||
"shutil.copy('./binary0.test', os.path.join(scripts_folder, 'binary0.test'))\n",
|
|
||||||
"shutil.copy('./binary1.test', os.path.join(scripts_folder, 'binary1.test'))"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"training_data_list=[\"binary0.train\", \"binary1.train\"]\n",
|
|
||||||
"validation_data_list = [\"binary0.test\", \"binary1.test\"]\n",
|
|
||||||
"lgbm = LightGBM(source_directory=scripts_folder, \n",
|
|
||||||
" compute_target=cpu_cluster, \n",
|
|
||||||
" distributed_training=Mpi(),\n",
|
|
||||||
" node_count=2,\n",
|
|
||||||
" lightgbm_config='train.conf',\n",
|
|
||||||
" data=training_data_list,\n",
|
|
||||||
" valid=validation_data_list\n",
|
|
||||||
" )\n",
|
|
||||||
"experiment_name = 'lightgbm-estimator-test'\n",
|
|
||||||
"experiment = Experiment(ws, name=experiment_name)\n",
|
|
||||||
"run = experiment.submit(lgbm, tags={\"test public docker image\": None})\n",
|
|
||||||
"RunDetails(run).show()"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"run.wait_for_completion(show_output=True)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Use data reference"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"from azureml.core.datastore import Datastore\n",
|
|
||||||
"from azureml.data.data_reference import DataReference\n",
|
|
||||||
"datastore = ws.get_default_datastore()"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"datastore.upload(src_dir='.',\n",
|
|
||||||
" target_path='.',\n",
|
|
||||||
" show_progress=True)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"training_data_list=[\"binary0.train\", \"binary1.train\"]\n",
|
|
||||||
"validation_data_list = [\"binary0.test\", \"binary1.test\"]\n",
|
|
||||||
"lgbm = LightGBM(source_directory='.', \n",
|
|
||||||
" compute_target=cpu_cluster, \n",
|
|
||||||
" distributed_training=Mpi(),\n",
|
|
||||||
" node_count=2,\n",
|
|
||||||
" inputs=[datastore.as_mount()],\n",
|
|
||||||
" lightgbm_config='train.conf',\n",
|
|
||||||
" data=training_data_list,\n",
|
|
||||||
" valid=validation_data_list\n",
|
|
||||||
" )\n",
|
|
||||||
"experiment_name = 'lightgbm-estimator-test'\n",
|
|
||||||
"experiment = Experiment(ws, name=experiment_name)\n",
|
|
||||||
"run = experiment.submit(lgbm, tags={\"use datastore.as_mount()\": None})\n",
|
|
||||||
"RunDetails(run).show()"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"run.wait_for_completion(show_output=True)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# uncomment below and run if compute resources are no longer needed\n",
|
|
||||||
"# cpu_cluster.delete() "
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"authors": [
|
|
||||||
{
|
|
||||||
"name": "jingywa"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"kernelspec": {
|
|
||||||
"display_name": "Python 3.6",
|
|
||||||
"language": "python",
|
|
||||||
"name": "python36"
|
|
||||||
},
|
|
||||||
"language_info": {
|
|
||||||
"codemirror_mode": {
|
|
||||||
"name": "ipython",
|
|
||||||
"version": 3
|
|
||||||
},
|
|
||||||
"file_extension": ".py",
|
|
||||||
"mimetype": "text/x-python",
|
|
||||||
"name": "python",
|
|
||||||
"nbconvert_exporter": "python",
|
|
||||||
"pygments_lexer": "ipython3",
|
|
||||||
"version": "3.6.9"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"nbformat": 4,
|
|
||||||
"nbformat_minor": 2
|
|
||||||
}
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
name: lightgbm-example
|
|
||||||
dependencies:
|
|
||||||
- pip:
|
|
||||||
- azureml-sdk
|
|
||||||
- azureml-contrib-gbdt
|
|
||||||
- azureml-widgets
|
|
||||||
- azureml-core
|
|
||||||
@@ -1,111 +0,0 @@
|
|||||||
# task type, support train and predict
|
|
||||||
task = train
|
|
||||||
|
|
||||||
# boosting type, support gbdt for now, alias: boosting, boost
|
|
||||||
boosting_type = gbdt
|
|
||||||
|
|
||||||
# application type, support following application
|
|
||||||
# regression , regression task
|
|
||||||
# binary , binary classification task
|
|
||||||
# lambdarank , lambdarank task
|
|
||||||
# alias: application, app
|
|
||||||
objective = binary
|
|
||||||
|
|
||||||
# eval metrics, support multi metric, delimite by ',' , support following metrics
|
|
||||||
# l1
|
|
||||||
# l2 , default metric for regression
|
|
||||||
# ndcg , default metric for lambdarank
|
|
||||||
# auc
|
|
||||||
# binary_logloss , default metric for binary
|
|
||||||
# binary_error
|
|
||||||
metric = binary_logloss,auc
|
|
||||||
|
|
||||||
# frequence for metric output
|
|
||||||
metric_freq = 1
|
|
||||||
|
|
||||||
# true if need output metric for training data, alias: tranining_metric, train_metric
|
|
||||||
is_training_metric = true
|
|
||||||
|
|
||||||
# number of bins for feature bucket, 255 is a recommend setting, it can save memories, and also has good accuracy.
|
|
||||||
max_bin = 255
|
|
||||||
|
|
||||||
# training data
|
|
||||||
# if exsting weight file, should name to "binary.train.weight"
|
|
||||||
# alias: train_data, train
|
|
||||||
data = binary.train
|
|
||||||
|
|
||||||
# validation data, support multi validation data, separated by ','
|
|
||||||
# if exsting weight file, should name to "binary.test.weight"
|
|
||||||
# alias: valid, test, test_data,
|
|
||||||
valid_data = binary.test
|
|
||||||
|
|
||||||
# number of trees(iterations), alias: num_tree, num_iteration, num_iterations, num_round, num_rounds
|
|
||||||
num_trees = 100
|
|
||||||
|
|
||||||
# shrinkage rate , alias: shrinkage_rate
|
|
||||||
learning_rate = 0.1
|
|
||||||
|
|
||||||
# number of leaves for one tree, alias: num_leaf
|
|
||||||
num_leaves = 63
|
|
||||||
|
|
||||||
# type of tree learner, support following types:
|
|
||||||
# serial , single machine version
|
|
||||||
# feature , use feature parallel to train
|
|
||||||
# data , use data parallel to train
|
|
||||||
# voting , use voting based parallel to train
|
|
||||||
# alias: tree
|
|
||||||
tree_learner = feature
|
|
||||||
|
|
||||||
# number of threads for multi-threading. One thread will use one CPU, defalut is setted to #cpu.
|
|
||||||
# num_threads = 8
|
|
||||||
|
|
||||||
# feature sub-sample, will random select 80% feature to train on each iteration
|
|
||||||
# alias: sub_feature
|
|
||||||
feature_fraction = 0.8
|
|
||||||
|
|
||||||
# Support bagging (data sub-sample), will perform bagging every 5 iterations
|
|
||||||
bagging_freq = 5
|
|
||||||
|
|
||||||
# Bagging farction, will random select 80% data on bagging
|
|
||||||
# alias: sub_row
|
|
||||||
bagging_fraction = 0.8
|
|
||||||
|
|
||||||
# minimal number data for one leaf, use this to deal with over-fit
|
|
||||||
# alias : min_data_per_leaf, min_data
|
|
||||||
min_data_in_leaf = 50
|
|
||||||
|
|
||||||
# minimal sum hessians for one leaf, use this to deal with over-fit
|
|
||||||
min_sum_hessian_in_leaf = 5.0
|
|
||||||
|
|
||||||
# save memory and faster speed for sparse feature, alias: is_sparse
|
|
||||||
is_enable_sparse = true
|
|
||||||
|
|
||||||
# when data is bigger than memory size, set this to true. otherwise set false will have faster speed
|
|
||||||
# alias: two_round_loading, two_round
|
|
||||||
use_two_round_loading = false
|
|
||||||
|
|
||||||
# true if need to save data to binary file and application will auto load data from binary file next time
|
|
||||||
# alias: is_save_binary, save_binary
|
|
||||||
is_save_binary_file = false
|
|
||||||
|
|
||||||
# output model file
|
|
||||||
output_model = LightGBM_model.txt
|
|
||||||
|
|
||||||
# support continuous train from trained gbdt model
|
|
||||||
# input_model= trained_model.txt
|
|
||||||
|
|
||||||
# output prediction file for predict task
|
|
||||||
# output_result= prediction.txt
|
|
||||||
|
|
||||||
# support continuous train from initial score file
|
|
||||||
# input_init_score= init_score.txt
|
|
||||||
|
|
||||||
|
|
||||||
# number of machines in parallel training, alias: num_machine
|
|
||||||
num_machines = 2
|
|
||||||
|
|
||||||
# local listening port in parallel training, alias: local_port
|
|
||||||
local_listen_port = 12400
|
|
||||||
|
|
||||||
# machines list file for parallel training, alias: mlist
|
|
||||||
machine_list_file = mlist.txt
|
|
||||||
@@ -4,7 +4,7 @@ Learn how to use Azure Machine Learning services for experimentation and model m
|
|||||||
|
|
||||||
As a pre-requisite, run the [configuration Notebook](../configuration.ipynb) notebook first to set up your Azure ML Workspace. Then, run the notebooks in following recommended order.
|
As a pre-requisite, run the [configuration Notebook](../configuration.ipynb) notebook first to set up your Azure ML Workspace. Then, run the notebooks in following recommended order.
|
||||||
|
|
||||||
* [train-within-notebook](./training/train-within-notebook): Train a model hile tracking run history, and learn how to deploy the model as web service to Azure Container Instance.
|
* [train-within-notebook](./training/train-within-notebook): Train a model while tracking run history, and learn how to deploy the model as web service to Azure Container Instance.
|
||||||
* [train-on-local](./training/train-on-local): Learn how to submit a run to local computer and use Azure ML managed run configuration.
|
* [train-on-local](./training/train-on-local): Learn how to submit a run to local computer and use Azure ML managed run configuration.
|
||||||
* [train-on-amlcompute](./training/train-on-amlcompute): Use a 1-n node Azure ML managed compute cluster for remote runs on Azure CPU or GPU infrastructure.
|
* [train-on-amlcompute](./training/train-on-amlcompute): Use a 1-n node Azure ML managed compute cluster for remote runs on Azure CPU or GPU infrastructure.
|
||||||
* [train-on-remote-vm](./training/train-on-remote-vm): Use Data Science Virtual Machine as a target for remote runs.
|
* [train-on-remote-vm](./training/train-on-remote-vm): Use Data Science Virtual Machine as a target for remote runs.
|
||||||
|
|||||||
@@ -155,7 +155,7 @@ jupyter notebook
|
|||||||
- Continuous retraining using Pipelines and Time-Series TabularDataset
|
- Continuous retraining using Pipelines and Time-Series TabularDataset
|
||||||
|
|
||||||
- [auto-ml-classification-text-dnn.ipynb](classification-text-dnn/auto-ml-classification-text-dnn.ipynb)
|
- [auto-ml-classification-text-dnn.ipynb](classification-text-dnn/auto-ml-classification-text-dnn.ipynb)
|
||||||
- Classification with text data using deep learning in AutoML
|
- Classification with text data using deep learning in automated ML
|
||||||
- AutoML highlights here include using deep neural networks (DNNs) to create embedded features from text data.
|
- AutoML highlights here include using deep neural networks (DNNs) to create embedded features from text data.
|
||||||
- Depending on the compute cluster the user provides, AutoML tried out Bidirectional Encoder Representations from Transformers (BERT) when a GPU compute is used.
|
- Depending on the compute cluster the user provides, AutoML tried out Bidirectional Encoder Representations from Transformers (BERT) when a GPU compute is used.
|
||||||
- Bidirectional Long-Short Term neural network (BiLSTM) when a CPU compute is used, thereby optimizing the choice of DNN for the uesr's setup.
|
- Bidirectional Long-Short Term neural network (BiLSTM) when a CPU compute is used, thereby optimizing the choice of DNN for the uesr's setup.
|
||||||
@@ -230,6 +230,15 @@ You may check the version of tensorflow and uninstall as follows
|
|||||||
2) enter `pip freeze` and look for `tensorflow` , if found, the version listed should be < 1.13
|
2) enter `pip freeze` and look for `tensorflow` , if found, the version listed should be < 1.13
|
||||||
3) If the listed version is a not a supported version, `pip uninstall tensorflow` in the command shell and enter y for confirmation.
|
3) If the listed version is a not a supported version, `pip uninstall tensorflow` in the command shell and enter y for confirmation.
|
||||||
|
|
||||||
|
## KeyError: 'brand' when running AutoML on local compute or Azure Databricks cluster**
|
||||||
|
If a new environment was created after 10 June 2020 using SDK 1.7.0 or lower, training may fail with the above error due to an update in the py-cpuinfo package. (Environments created on or before 10 June 2020 are unaffected, as well as experiments run on remote compute as cached training images are used.) To work around this issue, either of the two following steps can be taken:
|
||||||
|
|
||||||
|
1) Update the SDK version to 1.8.0 or higher (this will also downgrade py-cpuinfo to 5.0.0):
|
||||||
|
`pip install --upgrade azureml-sdk[automl]`
|
||||||
|
|
||||||
|
2) Downgrade the installed version of py-cpuinfo to 5.0.0:
|
||||||
|
`pip install py-cpuinfo==5.0.0`
|
||||||
|
|
||||||
## Remote run: DsvmCompute.create fails
|
## Remote run: DsvmCompute.create fails
|
||||||
There are several reasons why the DsvmCompute.create can fail. The reason is usually in the error message but you have to look at the end of the error message for the detailed reason. Some common reasons are:
|
There are several reasons why the DsvmCompute.create can fail. The reason is usually in the error message but you have to look at the end of the error message for the detailed reason. Some common reasons are:
|
||||||
1) `Compute name is invalid, it should start with a letter, be between 2 and 16 character, and only include letters (a-zA-Z), numbers (0-9) and \'-\'.` Note that underscore is not allowed in the name.
|
1) `Compute name is invalid, it should start with a letter, be between 2 and 16 character, and only include letters (a-zA-Z), numbers (0-9) and \'-\'.` Note that underscore is not allowed in the name.
|
||||||
|
|||||||
@@ -6,26 +6,23 @@ dependencies:
|
|||||||
- python>=3.5.2,<3.6.8
|
- python>=3.5.2,<3.6.8
|
||||||
- nb_conda
|
- nb_conda
|
||||||
- matplotlib==2.1.0
|
- matplotlib==2.1.0
|
||||||
- numpy~=1.16.0
|
- numpy~=1.18.0
|
||||||
- cython
|
- cython
|
||||||
- urllib3<1.24
|
- urllib3<1.24
|
||||||
- scipy==1.4.1
|
- scipy==1.4.1
|
||||||
- scikit-learn>=0.19.0,<=0.20.3
|
- scikit-learn==0.22.1
|
||||||
- pandas>=0.22.0,<=0.23.4
|
- pandas==0.25.1
|
||||||
- py-xgboost<=0.90
|
- py-xgboost<=0.90
|
||||||
- conda-forge::fbprophet==0.5
|
- conda-forge::fbprophet==0.5
|
||||||
|
- holidays==0.9.11
|
||||||
- pytorch::pytorch=1.4.0
|
- pytorch::pytorch=1.4.0
|
||||||
- cudatoolkit=10.1.243
|
- cudatoolkit=10.1.243
|
||||||
|
|
||||||
- pip:
|
- pip:
|
||||||
# Required packages for AzureML execution, history, and data preparation.
|
# Required packages for AzureML execution, history, and data preparation.
|
||||||
- azureml-defaults
|
|
||||||
- azureml-train-automl
|
|
||||||
- azureml-train
|
|
||||||
- azureml-widgets
|
- azureml-widgets
|
||||||
- azureml-pipeline
|
|
||||||
- pytorch-transformers==1.0.0
|
- pytorch-transformers==1.0.0
|
||||||
- spacy==2.1.8
|
- spacy==2.1.8
|
||||||
- pyarrow==0.17.0
|
|
||||||
- https://aka.ms/automl-resources/packages/en_core_web_sm-2.1.0.tar.gz
|
- https://aka.ms/automl-resources/packages/en_core_web_sm-2.1.0.tar.gz
|
||||||
|
- -r https://automlcesdkdataresources.blob.core.windows.net/validated-requirements/1.13.0/validated_win32_requirements.txt [--no-deps]
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,28 @@
|
|||||||
|
name: azure_automl
|
||||||
|
dependencies:
|
||||||
|
# The python interpreter version.
|
||||||
|
# Currently Azure ML only supports 3.5.2 and later.
|
||||||
|
- pip<=19.3.1
|
||||||
|
- python>=3.5.2,<3.6.8
|
||||||
|
- nb_conda
|
||||||
|
- matplotlib==2.1.0
|
||||||
|
- numpy~=1.18.0
|
||||||
|
- cython
|
||||||
|
- urllib3<1.24
|
||||||
|
- scipy==1.4.1
|
||||||
|
- scikit-learn==0.22.1
|
||||||
|
- pandas==0.25.1
|
||||||
|
- py-xgboost<=0.90
|
||||||
|
- conda-forge::fbprophet==0.5
|
||||||
|
- holidays==0.9.11
|
||||||
|
- pytorch::pytorch=1.4.0
|
||||||
|
- cudatoolkit=10.1.243
|
||||||
|
|
||||||
|
- pip:
|
||||||
|
# Required packages for AzureML execution, history, and data preparation.
|
||||||
|
- azureml-widgets
|
||||||
|
- pytorch-transformers==1.0.0
|
||||||
|
- spacy==2.1.8
|
||||||
|
- https://aka.ms/automl-resources/packages/en_core_web_sm-2.1.0.tar.gz
|
||||||
|
- -r https://automlcesdkdataresources.blob.core.windows.net/validated-requirements/1.13.0/validated_linux_requirements.txt [--no-deps]
|
||||||
|
|
||||||
@@ -7,25 +7,22 @@ dependencies:
|
|||||||
- python>=3.5.2,<3.6.8
|
- python>=3.5.2,<3.6.8
|
||||||
- nb_conda
|
- nb_conda
|
||||||
- matplotlib==2.1.0
|
- matplotlib==2.1.0
|
||||||
- numpy~=1.16.0
|
- numpy~=1.18.0
|
||||||
- cython
|
- cython
|
||||||
- urllib3<1.24
|
- urllib3<1.24
|
||||||
- scipy==1.4.1
|
- scipy==1.4.1
|
||||||
- scikit-learn>=0.19.0,<=0.20.3
|
- scikit-learn==0.22.1
|
||||||
- pandas>=0.22.0,<=0.23.4
|
- pandas==0.25.1
|
||||||
- py-xgboost<=0.90
|
- py-xgboost<=0.90
|
||||||
- conda-forge::fbprophet==0.5
|
- conda-forge::fbprophet==0.5
|
||||||
|
- holidays==0.9.11
|
||||||
- pytorch::pytorch=1.4.0
|
- pytorch::pytorch=1.4.0
|
||||||
- cudatoolkit=9.0
|
- cudatoolkit=9.0
|
||||||
|
|
||||||
- pip:
|
- pip:
|
||||||
# Required packages for AzureML execution, history, and data preparation.
|
# Required packages for AzureML execution, history, and data preparation.
|
||||||
- azureml-defaults
|
|
||||||
- azureml-train-automl
|
|
||||||
- azureml-train
|
|
||||||
- azureml-widgets
|
- azureml-widgets
|
||||||
- azureml-pipeline
|
|
||||||
- pytorch-transformers==1.0.0
|
- pytorch-transformers==1.0.0
|
||||||
- spacy==2.1.8
|
- spacy==2.1.8
|
||||||
- pyarrow==0.17.0
|
|
||||||
- https://aka.ms/automl-resources/packages/en_core_web_sm-2.1.0.tar.gz
|
- https://aka.ms/automl-resources/packages/en_core_web_sm-2.1.0.tar.gz
|
||||||
|
- -r https://automlcesdkdataresources.blob.core.windows.net/validated-requirements/1.13.0/validated_darwin_requirements.txt [--no-deps]
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ fi
|
|||||||
|
|
||||||
if [ "$AUTOML_ENV_FILE" == "" ]
|
if [ "$AUTOML_ENV_FILE" == "" ]
|
||||||
then
|
then
|
||||||
AUTOML_ENV_FILE="automl_env.yml"
|
AUTOML_ENV_FILE="automl_env_linux.yml"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ ! -f $AUTOML_ENV_FILE ]; then
|
if [ ! -f $AUTOML_ENV_FILE ]; then
|
||||||
|
|||||||
@@ -89,7 +89,7 @@
|
|||||||
"from azureml.automl.core.featurization import FeaturizationConfig\n",
|
"from azureml.automl.core.featurization import FeaturizationConfig\n",
|
||||||
"from azureml.core.dataset import Dataset\n",
|
"from azureml.core.dataset import Dataset\n",
|
||||||
"from azureml.train.automl import AutoMLConfig\n",
|
"from azureml.train.automl import AutoMLConfig\n",
|
||||||
"from azureml.explain.model._internal.explanation_client import ExplanationClient"
|
"from azureml.interpret._internal.explanation_client import ExplanationClient"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -105,7 +105,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"print(\"This notebook was created using version 1.10.0 of the Azure ML SDK\")\n",
|
"print(\"This notebook was created using version 1.13.0 of the Azure ML SDK\")\n",
|
||||||
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
|
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -362,7 +362,7 @@
|
|||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"Call the `submit` method on the experiment object and pass the run configuration. Execution of local runs is synchronous. Depending on the data and the number of iterations this can run for a while."
|
"Call the `submit` method on the experiment object and pass the run configuration. Execution of local runs is synchronous. Depending on the data and the number of iterations this can run for a while. Validation errors and current status will be shown when setting `show_output=True` and the execution will be synchronous."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -733,24 +733,6 @@
|
|||||||
"print(aci_service.state)"
|
"print(aci_service.state)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### Delete a Web Service\n",
|
|
||||||
"\n",
|
|
||||||
"Deletes the specified web service."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"#aci_service.delete()"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
@@ -775,7 +757,9 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"## Test\n",
|
"## Test\n",
|
||||||
"\n",
|
"\n",
|
||||||
"Now that the model is trained, run the test data through the trained model to get the predicted values."
|
"Now that the model is trained, run the test data through the trained model to get the predicted values. This calls the ACI web service to do the prediction.\n",
|
||||||
|
"\n",
|
||||||
|
"Note that the JSON passed to the ACI web service is an array of rows of data. Each row should either be an array of values in the same order that was used for training or a dictionary where the keys are the same as the column names used for training. The example below uses dictionary rows."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -815,10 +799,27 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"y_pred = fitted_model.predict(X_test)\n",
|
"import json\n",
|
||||||
|
"import requests\n",
|
||||||
|
"\n",
|
||||||
|
"X_test_json = X_test.to_json(orient='records')\n",
|
||||||
|
"data = \"{\\\"data\\\": \" + X_test_json +\"}\"\n",
|
||||||
|
"headers = {'Content-Type': 'application/json'}\n",
|
||||||
|
"\n",
|
||||||
|
"resp = requests.post(aci_service.scoring_uri, data, headers=headers)\n",
|
||||||
|
"\n",
|
||||||
|
"y_pred = json.loads(json.loads(resp.text))['result']"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
"actual = array(y_test)\n",
|
"actual = array(y_test)\n",
|
||||||
"actual = actual[:,0]\n",
|
"actual = actual[:,0]\n",
|
||||||
"print(y_pred.shape, \" \", actual.shape)"
|
"print(len(y_pred), \" \", len(actual))"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -827,8 +828,7 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"### Calculate metrics for the prediction\n",
|
"### Calculate metrics for the prediction\n",
|
||||||
"\n",
|
"\n",
|
||||||
"Now visualize the data on a scatter plot to show what our truth (actual) values are compared to the predicted values \n",
|
"Now visualize the data as a confusion matrix that compared the predicted values against the actual values.\n"
|
||||||
"from the trained model that was returned."
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -838,12 +838,45 @@
|
|||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"%matplotlib notebook\n",
|
"%matplotlib notebook\n",
|
||||||
"test_pred = plt.scatter(actual, y_pred, color='b')\n",
|
"from sklearn.metrics import confusion_matrix\n",
|
||||||
"test_test = plt.scatter(actual, actual, color='g')\n",
|
"import numpy as np\n",
|
||||||
"plt.legend((test_pred, test_test), ('prediction', 'truth'), loc='upper left', fontsize=8)\n",
|
"import itertools\n",
|
||||||
|
"\n",
|
||||||
|
"cf =confusion_matrix(actual,y_pred)\n",
|
||||||
|
"plt.imshow(cf,cmap=plt.cm.Blues,interpolation='nearest')\n",
|
||||||
|
"plt.colorbar()\n",
|
||||||
|
"plt.title('Confusion Matrix')\n",
|
||||||
|
"plt.xlabel('Predicted')\n",
|
||||||
|
"plt.ylabel('Actual')\n",
|
||||||
|
"class_labels = ['no','yes']\n",
|
||||||
|
"tick_marks = np.arange(len(class_labels))\n",
|
||||||
|
"plt.xticks(tick_marks,class_labels)\n",
|
||||||
|
"plt.yticks([-0.5,0,1,1.5],['','no','yes',''])\n",
|
||||||
|
"# plotting text value inside cells\n",
|
||||||
|
"thresh = cf.max() / 2.\n",
|
||||||
|
"for i,j in itertools.product(range(cf.shape[0]),range(cf.shape[1])):\n",
|
||||||
|
" plt.text(j,i,format(cf[i,j],'d'),horizontalalignment='center',color='white' if cf[i,j] >thresh else 'black')\n",
|
||||||
"plt.show()"
|
"plt.show()"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Delete a Web Service\n",
|
||||||
|
"\n",
|
||||||
|
"Deletes the specified web service."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"aci_service.delete()"
|
||||||
|
]
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
|
|||||||
@@ -93,7 +93,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"print(\"This notebook was created using version 1.10.0 of the Azure ML SDK\")\n",
|
"print(\"This notebook was created using version 1.13.0 of the Azure ML SDK\")\n",
|
||||||
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
|
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -232,7 +232,7 @@
|
|||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"Call the `submit` method on the experiment object and pass the run configuration. Depending on the data and the number of iterations this can run for a while."
|
"Call the `submit` method on the experiment object and pass the run configuration. Depending on the data and the number of iterations this can run for a while. Validation errors and current status will be shown when setting `show_output=True` and the execution will be synchronous."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -97,7 +97,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"print(\"This notebook was created using version 1.10.0 of the Azure ML SDK\")\n",
|
"print(\"This notebook was created using version 1.13.0 of the Azure ML SDK\")\n",
|
||||||
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
|
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -151,6 +151,8 @@
|
|||||||
"from azureml.core.compute import ComputeTarget, AmlCompute\n",
|
"from azureml.core.compute import ComputeTarget, AmlCompute\n",
|
||||||
"from azureml.core.compute_target import ComputeTargetException\n",
|
"from azureml.core.compute_target import ComputeTargetException\n",
|
||||||
"\n",
|
"\n",
|
||||||
|
"num_nodes = 2\n",
|
||||||
|
"\n",
|
||||||
"# Choose a name for your cluster.\n",
|
"# Choose a name for your cluster.\n",
|
||||||
"amlcompute_cluster_name = \"dnntext-cluster\"\n",
|
"amlcompute_cluster_name = \"dnntext-cluster\"\n",
|
||||||
"\n",
|
"\n",
|
||||||
@@ -163,7 +165,7 @@
|
|||||||
" # To use BERT (this is recommended for best performance), select a GPU such as \"STANDARD_NC6\" \n",
|
" # To use BERT (this is recommended for best performance), select a GPU such as \"STANDARD_NC6\" \n",
|
||||||
" # or similar GPU option\n",
|
" # or similar GPU option\n",
|
||||||
" # available in your workspace\n",
|
" # available in your workspace\n",
|
||||||
" max_nodes = 1)\n",
|
" max_nodes = num_nodes)\n",
|
||||||
" compute_target = ComputeTarget.create(ws, amlcompute_cluster_name, compute_config)\n",
|
" compute_target = ComputeTarget.create(ws, amlcompute_cluster_name, compute_config)\n",
|
||||||
"\n",
|
"\n",
|
||||||
"compute_target.wait_for_completion(show_output=True)"
|
"compute_target.wait_for_completion(show_output=True)"
|
||||||
@@ -270,7 +272,9 @@
|
|||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"This step requires an Enterprise workspace to gain access to this feature. To learn more about creating an Enterprise workspace or upgrading to an Enterprise workspace from the Azure portal, please visit our [Workspace page](https://docs.microsoft.com/azure/machine-learning/service/concept-workspace#upgrade)."
|
"This step requires an Enterprise workspace to gain access to this feature. To learn more about creating an Enterprise workspace or upgrading to an Enterprise workspace from the Azure portal, please visit our [Workspace page](https://docs.microsoft.com/azure/machine-learning/service/concept-workspace#upgrade).\n",
|
||||||
|
"\n",
|
||||||
|
"This notebook uses the blocked_models parameter to exclude some models that can take a longer time to train on some text datasets. You can choose to remove models from the blocked_models list but you may need to increase the experiment_timeout_hours parameter value to get results."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -282,7 +286,7 @@
|
|||||||
"automl_settings = {\n",
|
"automl_settings = {\n",
|
||||||
" \"experiment_timeout_minutes\": 20,\n",
|
" \"experiment_timeout_minutes\": 20,\n",
|
||||||
" \"primary_metric\": 'accuracy',\n",
|
" \"primary_metric\": 'accuracy',\n",
|
||||||
" \"max_concurrent_iterations\": 4, \n",
|
" \"max_concurrent_iterations\": num_nodes, \n",
|
||||||
" \"max_cores_per_iteration\": -1,\n",
|
" \"max_cores_per_iteration\": -1,\n",
|
||||||
" \"enable_dnn\": True,\n",
|
" \"enable_dnn\": True,\n",
|
||||||
" \"enable_early_stopping\": True,\n",
|
" \"enable_early_stopping\": True,\n",
|
||||||
@@ -297,6 +301,7 @@
|
|||||||
" compute_target=compute_target,\n",
|
" compute_target=compute_target,\n",
|
||||||
" training_data=train_dataset,\n",
|
" training_data=train_dataset,\n",
|
||||||
" label_column_name=target_column_name,\n",
|
" label_column_name=target_column_name,\n",
|
||||||
|
" blocked_models = ['LightGBM'],\n",
|
||||||
" **automl_settings\n",
|
" **automl_settings\n",
|
||||||
" )"
|
" )"
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
import pandas as pd
|
import pandas as pd
|
||||||
from azureml.core import Environment
|
from azureml.core import Environment
|
||||||
from azureml.core.conda_dependencies import CondaDependencies
|
|
||||||
from azureml.train.estimator import Estimator
|
from azureml.train.estimator import Estimator
|
||||||
from azureml.core.run import Run
|
from azureml.core.run import Run
|
||||||
|
|
||||||
@@ -8,13 +7,7 @@ from azureml.core.run import Run
|
|||||||
def run_inference(test_experiment, compute_target, script_folder, train_run,
|
def run_inference(test_experiment, compute_target, script_folder, train_run,
|
||||||
train_dataset, test_dataset, target_column_name, model_name):
|
train_dataset, test_dataset, target_column_name, model_name):
|
||||||
|
|
||||||
train_run.download_file('outputs/conda_env_v_1_0_0.yml',
|
inference_env = train_run.get_environment()
|
||||||
'inference/condafile.yml')
|
|
||||||
|
|
||||||
inference_env = Environment("myenv")
|
|
||||||
inference_env.docker.enabled = True
|
|
||||||
inference_env.python.conda_dependencies = CondaDependencies(
|
|
||||||
conda_dependencies_file_path='inference/condafile.yml')
|
|
||||||
|
|
||||||
est = Estimator(source_directory=script_folder,
|
est = Estimator(source_directory=script_folder,
|
||||||
entry_script='infer.py',
|
entry_script='infer.py',
|
||||||
|
|||||||
@@ -88,7 +88,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"print(\"This notebook was created using version 1.10.0 of the Azure ML SDK\")\n",
|
"print(\"This notebook was created using version 1.13.0 of the Azure ML SDK\")\n",
|
||||||
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
|
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -204,7 +204,6 @@
|
|||||||
"cd = CondaDependencies.create(pip_packages=['azureml-sdk[automl]', 'applicationinsights', 'azureml-opendatasets', 'azureml-defaults'], \n",
|
"cd = CondaDependencies.create(pip_packages=['azureml-sdk[automl]', 'applicationinsights', 'azureml-opendatasets', 'azureml-defaults'], \n",
|
||||||
" conda_packages=['numpy==1.16.2'], \n",
|
" conda_packages=['numpy==1.16.2'], \n",
|
||||||
" pin_sdk_version=False)\n",
|
" pin_sdk_version=False)\n",
|
||||||
"#cd.add_pip_package('azureml-explain-model')\n",
|
|
||||||
"conda_run_config.environment.python.conda_dependencies = cd\n",
|
"conda_run_config.environment.python.conda_dependencies = cd\n",
|
||||||
"\n",
|
"\n",
|
||||||
"print('run config is ready')"
|
"print('run config is ready')"
|
||||||
|
|||||||
@@ -0,0 +1,92 @@
|
|||||||
|
# Experimental Notebooks for Automated ML
|
||||||
|
Notebooks listed in this folder are leveraging experimental features. Namespaces or function signitures may change in future SDK releases. The notebooks published here will reflect the latest supported APIs. All of these notebooks can run on a client-only installation of the Automated ML SDK.
|
||||||
|
The client only installation doesn't contain any of the machine learning libraries, such as scikit-learn, xgboost, or tensorflow, making it much faster to install and is less likely to conflict with any packages in an existing environment. However, since the ML libraries are not available locally, models cannot be downloaded and loaded directly in the client. To replace the functionality of having models locally, these notebooks also demonstrate the ModelProxy feature which will allow you to submit a predict/forecast to the training environment.
|
||||||
|
|
||||||
|
<a name="localconda"></a>
|
||||||
|
## Setup using a Local Conda environment
|
||||||
|
|
||||||
|
To run these notebook on your own notebook server, use these installation instructions.
|
||||||
|
The instructions below will install everything you need and then start a Jupyter notebook.
|
||||||
|
If you would like to use a lighter-weight version of the client that does not install all of the machine learning libraries locally, you can leverage the [experimental notebooks.](experimental/README.md)
|
||||||
|
|
||||||
|
### 1. Install mini-conda from [here](https://conda.io/miniconda.html), choose 64-bit Python 3.7 or higher.
|
||||||
|
- **Note**: if you already have conda installed, you can keep using it but it should be version 4.4.10 or later (as shown by: conda -V). If you have a previous version installed, you can update it using the command: conda update conda.
|
||||||
|
There's no need to install mini-conda specifically.
|
||||||
|
|
||||||
|
### 2. Downloading the sample notebooks
|
||||||
|
- Download the sample notebooks from [GitHub](https://github.com/Azure/MachineLearningNotebooks) as zip and extract the contents to a local directory. The automated ML sample notebooks are in the "automated-machine-learning" folder.
|
||||||
|
|
||||||
|
### 3. Setup a new conda environment
|
||||||
|
The **automl_setup** script creates a new conda environment, installs the necessary packages, configures the widget and starts a jupyter notebook. It takes the conda environment name as an optional parameter. The default conda environment name is azure_automl. The exact command depends on the operating system. See the specific sections below for Windows, Mac and Linux. It can take about 10 minutes to execute.
|
||||||
|
|
||||||
|
Packages installed by the **automl_setup** script:
|
||||||
|
<ul><li>python</li><li>nb_conda</li><li>matplotlib</li><li>numpy</li><li>cython</li><li>urllib3</li><li>pandas</li><li>azureml-sdk</li><li>azureml-widgets</li><li>pandas-ml</li></ul>
|
||||||
|
|
||||||
|
For more details refer to the [automl_env.yml](./automl_env.yml)
|
||||||
|
## Windows
|
||||||
|
Start an **Anaconda Prompt** window, cd to the **how-to-use-azureml/automated-machine-learning/experimental** folder where the sample notebooks were extracted and then run:
|
||||||
|
```
|
||||||
|
automl_setup
|
||||||
|
```
|
||||||
|
## Mac
|
||||||
|
Install "Command line developer tools" if it is not already installed (you can use the command: `xcode-select --install`).
|
||||||
|
|
||||||
|
Start a Terminal windows, cd to the **how-to-use-azureml/automated-machine-learning/experimental** folder where the sample notebooks were extracted and then run:
|
||||||
|
|
||||||
|
```
|
||||||
|
bash automl_setup_mac.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
## Linux
|
||||||
|
cd to the **how-to-use-azureml/automated-machine-learning/experimental** folder where the sample notebooks were extracted and then run:
|
||||||
|
|
||||||
|
```
|
||||||
|
bash automl_setup_linux.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4. Running configuration.ipynb
|
||||||
|
- Before running any samples you next need to run the configuration notebook. Click on [configuration](../../configuration.ipynb) notebook
|
||||||
|
- Execute the cells in the notebook to Register Machine Learning Services Resource Provider and create a workspace. (*instructions in notebook*)
|
||||||
|
|
||||||
|
### 5. Running Samples
|
||||||
|
- Please make sure you use the Python [conda env:azure_automl] kernel when trying the sample Notebooks.
|
||||||
|
- Follow the instructions in the individual notebooks to explore various features in automated ML.
|
||||||
|
|
||||||
|
### 6. Starting jupyter notebook manually
|
||||||
|
To start your Jupyter notebook manually, use:
|
||||||
|
|
||||||
|
```
|
||||||
|
conda activate azure_automl
|
||||||
|
jupyter notebook
|
||||||
|
```
|
||||||
|
|
||||||
|
or on Mac or Linux:
|
||||||
|
|
||||||
|
```
|
||||||
|
source activate azure_automl
|
||||||
|
jupyter notebook
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
<a name="samples"></a>
|
||||||
|
# Automated ML SDK Sample Notebooks
|
||||||
|
|
||||||
|
- [auto-ml-regression.ipynb](regression/auto-ml-regression.ipynb)
|
||||||
|
- Dataset: Hardware Performance Dataset
|
||||||
|
- Simple example of using automated ML for regression
|
||||||
|
- Uses azure compute for training
|
||||||
|
- Uses ModelProxy for submitting prediction to training environment on azure compute
|
||||||
|
|
||||||
|
<a name="documentation"></a>
|
||||||
|
See [Configure automated machine learning experiments](https://docs.microsoft.com/azure/machine-learning/service/how-to-configure-auto-train) to learn how more about the the settings and features available for automated machine learning experiments.
|
||||||
|
|
||||||
|
<a name="pythoncommand"></a>
|
||||||
|
# Running using python command
|
||||||
|
Jupyter notebook provides a File / Download as / Python (.py) option for saving the notebook as a Python file.
|
||||||
|
You can then run this file using the python command.
|
||||||
|
However, on Windows the file needs to be modified before it can be run.
|
||||||
|
The following condition must be added to the main code in the file:
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
|
||||||
|
The main code of the file must be indented so that it is under this condition.
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
name: azure_automl_experimental
|
||||||
|
dependencies:
|
||||||
|
# The python interpreter version.
|
||||||
|
# Currently Azure ML only supports 3.5.2 and later.
|
||||||
|
- pip<=19.3.1
|
||||||
|
- python>=3.5.2,<3.8
|
||||||
|
- nb_conda
|
||||||
|
- matplotlib==2.1.0
|
||||||
|
- numpy~=1.18.0
|
||||||
|
- cython
|
||||||
|
- urllib3<1.24
|
||||||
|
- scikit-learn==0.22.1
|
||||||
|
- pandas==0.25.1
|
||||||
|
|
||||||
|
- pip:
|
||||||
|
# Required packages for AzureML execution, history, and data preparation.
|
||||||
|
- azureml-defaults
|
||||||
|
- azureml-sdk
|
||||||
|
- azureml-widgets
|
||||||
|
- azureml-explain-model
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
name: azure_automl_experimental
|
||||||
|
dependencies:
|
||||||
|
# The python interpreter version.
|
||||||
|
# Currently Azure ML only supports 3.5.2 and later.
|
||||||
|
- pip<=19.3.1
|
||||||
|
- nomkl
|
||||||
|
- python>=3.5.2,<3.8
|
||||||
|
- nb_conda
|
||||||
|
- matplotlib==2.1.0
|
||||||
|
- numpy~=1.18.0
|
||||||
|
- cython
|
||||||
|
- urllib3<1.24
|
||||||
|
- scikit-learn==0.22.1
|
||||||
|
- pandas==0.25.1
|
||||||
|
|
||||||
|
- pip:
|
||||||
|
# Required packages for AzureML execution, history, and data preparation.
|
||||||
|
- azureml-defaults
|
||||||
|
- azureml-sdk
|
||||||
|
- azureml-widgets
|
||||||
|
- azureml-explain-model
|
||||||
@@ -0,0 +1,63 @@
|
|||||||
|
@echo off
|
||||||
|
set conda_env_name=%1
|
||||||
|
set automl_env_file=%2
|
||||||
|
set options=%3
|
||||||
|
set PIP_NO_WARN_SCRIPT_LOCATION=0
|
||||||
|
|
||||||
|
IF "%conda_env_name%"=="" SET conda_env_name="azure_automl_experimental"
|
||||||
|
IF "%automl_env_file%"=="" SET automl_env_file="automl_env.yml"
|
||||||
|
|
||||||
|
IF NOT EXIST %automl_env_file% GOTO YmlMissing
|
||||||
|
|
||||||
|
IF "%CONDA_EXE%"=="" GOTO CondaMissing
|
||||||
|
|
||||||
|
call conda activate %conda_env_name% 2>nul:
|
||||||
|
|
||||||
|
if not errorlevel 1 (
|
||||||
|
echo Upgrading existing conda environment %conda_env_name%
|
||||||
|
call pip uninstall azureml-train-automl -y -q
|
||||||
|
call conda env update --name %conda_env_name% --file %automl_env_file%
|
||||||
|
if errorlevel 1 goto ErrorExit
|
||||||
|
) else (
|
||||||
|
call conda env create -f %automl_env_file% -n %conda_env_name%
|
||||||
|
)
|
||||||
|
|
||||||
|
call conda activate %conda_env_name% 2>nul:
|
||||||
|
if errorlevel 1 goto ErrorExit
|
||||||
|
|
||||||
|
call python -m ipykernel install --user --name %conda_env_name% --display-name "Python (%conda_env_name%)"
|
||||||
|
|
||||||
|
REM azureml.widgets is now installed as part of the pip install under the conda env.
|
||||||
|
REM Removing the old user install so that the notebooks will use the latest widget.
|
||||||
|
call jupyter nbextension uninstall --user --py azureml.widgets
|
||||||
|
|
||||||
|
echo.
|
||||||
|
echo.
|
||||||
|
echo ***************************************
|
||||||
|
echo * AutoML setup completed successfully *
|
||||||
|
echo ***************************************
|
||||||
|
IF NOT "%options%"=="nolaunch" (
|
||||||
|
echo.
|
||||||
|
echo Starting jupyter notebook - please run the configuration notebook
|
||||||
|
echo.
|
||||||
|
jupyter notebook --log-level=50 --notebook-dir='..\..'
|
||||||
|
)
|
||||||
|
|
||||||
|
goto End
|
||||||
|
|
||||||
|
:CondaMissing
|
||||||
|
echo Please run this script from an Anaconda Prompt window.
|
||||||
|
echo You can start an Anaconda Prompt window by
|
||||||
|
echo typing Anaconda Prompt on the Start menu.
|
||||||
|
echo If you don't see the Anaconda Prompt app, install Miniconda.
|
||||||
|
echo If you are running an older version of Miniconda or Anaconda,
|
||||||
|
echo you can upgrade using the command: conda update conda
|
||||||
|
goto End
|
||||||
|
|
||||||
|
:YmlMissing
|
||||||
|
echo File %automl_env_file% not found.
|
||||||
|
|
||||||
|
:ErrorExit
|
||||||
|
echo Install failed
|
||||||
|
|
||||||
|
:End
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
CONDA_ENV_NAME=$1
|
||||||
|
AUTOML_ENV_FILE=$2
|
||||||
|
OPTIONS=$3
|
||||||
|
PIP_NO_WARN_SCRIPT_LOCATION=0
|
||||||
|
|
||||||
|
if [ "$CONDA_ENV_NAME" == "" ]
|
||||||
|
then
|
||||||
|
CONDA_ENV_NAME="azure_automl_experimental"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$AUTOML_ENV_FILE" == "" ]
|
||||||
|
then
|
||||||
|
AUTOML_ENV_FILE="automl_env.yml"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! -f $AUTOML_ENV_FILE ]; then
|
||||||
|
echo "File $AUTOML_ENV_FILE not found"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if source activate $CONDA_ENV_NAME 2> /dev/null
|
||||||
|
then
|
||||||
|
echo "Upgrading existing conda environment" $CONDA_ENV_NAME
|
||||||
|
pip uninstall azureml-train-automl -y -q
|
||||||
|
conda env update --name $CONDA_ENV_NAME --file $AUTOML_ENV_FILE &&
|
||||||
|
jupyter nbextension uninstall --user --py azureml.widgets
|
||||||
|
else
|
||||||
|
conda env create -f $AUTOML_ENV_FILE -n $CONDA_ENV_NAME &&
|
||||||
|
source activate $CONDA_ENV_NAME &&
|
||||||
|
python -m ipykernel install --user --name $CONDA_ENV_NAME --display-name "Python ($CONDA_ENV_NAME)" &&
|
||||||
|
jupyter nbextension uninstall --user --py azureml.widgets &&
|
||||||
|
echo "" &&
|
||||||
|
echo "" &&
|
||||||
|
echo "***************************************" &&
|
||||||
|
echo "* AutoML setup completed successfully *" &&
|
||||||
|
echo "***************************************" &&
|
||||||
|
if [ "$OPTIONS" != "nolaunch" ]
|
||||||
|
then
|
||||||
|
echo "" &&
|
||||||
|
echo "Starting jupyter notebook - please run the configuration notebook" &&
|
||||||
|
echo "" &&
|
||||||
|
jupyter notebook --log-level=50 --notebook-dir '../..'
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ $? -gt 0 ]
|
||||||
|
then
|
||||||
|
echo "Installation failed"
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
CONDA_ENV_NAME=$1
|
||||||
|
AUTOML_ENV_FILE=$2
|
||||||
|
OPTIONS=$3
|
||||||
|
PIP_NO_WARN_SCRIPT_LOCATION=0
|
||||||
|
|
||||||
|
if [ "$CONDA_ENV_NAME" == "" ]
|
||||||
|
then
|
||||||
|
CONDA_ENV_NAME="azure_automl_experimental"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$AUTOML_ENV_FILE" == "" ]
|
||||||
|
then
|
||||||
|
AUTOML_ENV_FILE="automl_env.yml"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! -f $AUTOML_ENV_FILE ]; then
|
||||||
|
echo "File $AUTOML_ENV_FILE not found"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if source activate $CONDA_ENV_NAME 2> /dev/null
|
||||||
|
then
|
||||||
|
echo "Upgrading existing conda environment" $CONDA_ENV_NAME
|
||||||
|
pip uninstall azureml-train-automl -y -q
|
||||||
|
conda env update --name $CONDA_ENV_NAME --file $AUTOML_ENV_FILE &&
|
||||||
|
jupyter nbextension uninstall --user --py azureml.widgets
|
||||||
|
else
|
||||||
|
conda env create -f $AUTOML_ENV_FILE -n $CONDA_ENV_NAME &&
|
||||||
|
source activate $CONDA_ENV_NAME &&
|
||||||
|
conda install lightgbm -c conda-forge -y &&
|
||||||
|
python -m ipykernel install --user --name $CONDA_ENV_NAME --display-name "Python ($CONDA_ENV_NAME)" &&
|
||||||
|
jupyter nbextension uninstall --user --py azureml.widgets &&
|
||||||
|
echo "" &&
|
||||||
|
echo "" &&
|
||||||
|
echo "***************************************" &&
|
||||||
|
echo "* AutoML setup completed successfully *" &&
|
||||||
|
echo "***************************************" &&
|
||||||
|
if [ "$OPTIONS" != "nolaunch" ]
|
||||||
|
then
|
||||||
|
echo "" &&
|
||||||
|
echo "Starting jupyter notebook - please run the configuration notebook" &&
|
||||||
|
echo "" &&
|
||||||
|
jupyter notebook --log-level=50 --notebook-dir '../..'
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ $? -gt 0 ]
|
||||||
|
then
|
||||||
|
echo "Installation failed"
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@@ -0,0 +1,481 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Copyright (c) Microsoft Corporation. All rights reserved.\n",
|
||||||
|
"\n",
|
||||||
|
"Licensed under the MIT License."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
""
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"# Automated Machine Learning\n",
|
||||||
|
"_**Regression with Aml Compute**_\n",
|
||||||
|
"\n",
|
||||||
|
"## Contents\n",
|
||||||
|
"1. [Introduction](#Introduction)\n",
|
||||||
|
"1. [Setup](#Setup)\n",
|
||||||
|
"1. [Data](#Data)\n",
|
||||||
|
"1. [Train](#Train)\n",
|
||||||
|
"1. [Results](#Results)\n",
|
||||||
|
"1. [Test](#Test)\n",
|
||||||
|
"\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Introduction\n",
|
||||||
|
"In this example we use the Hardware Performance Dataset to showcase how you can use AutoML for a simple regression problem. The Regression goal is to predict the performance of certain combinations of hardware parts.\n",
|
||||||
|
"\n",
|
||||||
|
"If you are using an Azure Machine Learning Compute Instance, you are all set. Otherwise, go through the [configuration](../../../../configuration.ipynb) notebook first if you haven't already to establish your connection to the AzureML Workspace. \n",
|
||||||
|
"\n",
|
||||||
|
"In this notebook you will learn how to:\n",
|
||||||
|
"1. Create an `Experiment` in an existing `Workspace`.\n",
|
||||||
|
"2. Configure AutoML using `AutoMLConfig`.\n",
|
||||||
|
"3. Train the model using remote compute.\n",
|
||||||
|
"4. Explore the results.\n",
|
||||||
|
"5. Test the best fitted model."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Setup\n",
|
||||||
|
"\n",
|
||||||
|
"As part of the setup you have already created an Azure ML `Workspace` object. For Automated ML you will need to create an `Experiment` object, which is a named object in a `Workspace` used to run experiments."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import logging\n",
|
||||||
|
"\n",
|
||||||
|
"from matplotlib import pyplot as plt\n",
|
||||||
|
"import numpy as np\n",
|
||||||
|
"import pandas as pd\n",
|
||||||
|
" \n",
|
||||||
|
"\n",
|
||||||
|
"import azureml.core\n",
|
||||||
|
"from azureml.core.experiment import Experiment\n",
|
||||||
|
"from azureml.core.workspace import Workspace\n",
|
||||||
|
"from azureml.core.dataset import Dataset\n",
|
||||||
|
"from azureml.train.automl import AutoMLConfig"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"This sample notebook may use features that are not available in previous versions of the Azure ML SDK."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"print(\"This notebook was created using version 1.13.0 of the Azure ML SDK\")\n",
|
||||||
|
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"ws = Workspace.from_config()\n",
|
||||||
|
"\n",
|
||||||
|
"# Choose a name for the experiment.\n",
|
||||||
|
"experiment_name = 'automl-regression-model-proxy'\n",
|
||||||
|
"\n",
|
||||||
|
"experiment = Experiment(ws, experiment_name)\n",
|
||||||
|
"\n",
|
||||||
|
"output = {}\n",
|
||||||
|
"output['Subscription ID'] = ws.subscription_id\n",
|
||||||
|
"output['Workspace'] = ws.name\n",
|
||||||
|
"output['Resource Group'] = ws.resource_group\n",
|
||||||
|
"output['Location'] = ws.location\n",
|
||||||
|
"output['Run History Name'] = experiment_name\n",
|
||||||
|
"pd.set_option('display.max_colwidth', -1)\n",
|
||||||
|
"outputDf = pd.DataFrame(data = output, index = [''])\n",
|
||||||
|
"outputDf.T"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Using AmlCompute\n",
|
||||||
|
"You will need to create a [compute target](https://docs.microsoft.com/azure/machine-learning/service/concept-azure-machine-learning-architecture#compute-target) for your AutoML run. In this tutorial, you use `AmlCompute` as your training compute resource."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"from azureml.core.compute import ComputeTarget, AmlCompute\n",
|
||||||
|
"from azureml.core.compute_target import ComputeTargetException\n",
|
||||||
|
"\n",
|
||||||
|
"# Choose a name for your CPU cluster\n",
|
||||||
|
"cpu_cluster_name = \"reg-cluster\"\n",
|
||||||
|
"\n",
|
||||||
|
"# Verify that cluster does not exist already\n",
|
||||||
|
"try:\n",
|
||||||
|
" compute_target = ComputeTarget(workspace=ws, name=cpu_cluster_name)\n",
|
||||||
|
" print('Found existing cluster, use it.')\n",
|
||||||
|
"except ComputeTargetException:\n",
|
||||||
|
" compute_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_D2_V2',\n",
|
||||||
|
" max_nodes=4)\n",
|
||||||
|
" compute_target = ComputeTarget.create(ws, cpu_cluster_name, compute_config)\n",
|
||||||
|
"\n",
|
||||||
|
"compute_target.wait_for_completion(show_output=True)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Data\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Load Data\n",
|
||||||
|
"Load the hardware dataset from a csv file containing both training features and labels. The features are inputs to the model, while the training labels represent the expected output of the model. Next, we'll split the data using random_split and extract the training data for the model. "
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"data = \"https://automlsamplenotebookdata.blob.core.windows.net/automl-sample-notebook-data/machineData.csv\"\n",
|
||||||
|
"dataset = Dataset.Tabular.from_delimited_files(data)\n",
|
||||||
|
"\n",
|
||||||
|
"# Split the dataset into train and test datasets\n",
|
||||||
|
"train_data, test_data = dataset.random_split(percentage=0.8, seed=223)\n",
|
||||||
|
"\n",
|
||||||
|
"label = \"ERP\"\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Train\n",
|
||||||
|
"\n",
|
||||||
|
"Instantiate an `AutoMLConfig` object to specify the settings and data used to run the experiment.\n",
|
||||||
|
"\n",
|
||||||
|
"|Property|Description|\n",
|
||||||
|
"|-|-|\n",
|
||||||
|
"|**task**|classification, regression or forecasting|\n",
|
||||||
|
"|**primary_metric**|This is the metric that you want to optimize. Regression supports the following primary metrics: <br><i>spearman_correlation</i><br><i>normalized_root_mean_squared_error</i><br><i>r2_score</i><br><i>normalized_mean_absolute_error</i>|\n",
|
||||||
|
"|**n_cross_validations**|Number of cross validation splits.|\n",
|
||||||
|
"|**training_data**|(sparse) array-like, shape = [n_samples, n_features]|\n",
|
||||||
|
"|**label_column_name**|(sparse) array-like, shape = [n_samples, ], targets values.|\n",
|
||||||
|
"\n",
|
||||||
|
"**_You can find more information about primary metrics_** [here](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-configure-auto-train#primary-metric)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"tags": [
|
||||||
|
"automlconfig-remarks-sample"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"automl_settings = {\n",
|
||||||
|
" \"n_cross_validations\": 3,\n",
|
||||||
|
" \"primary_metric\": 'r2_score',\n",
|
||||||
|
" \"enable_early_stopping\": True, \n",
|
||||||
|
" \"experiment_timeout_hours\": 0.3, #for real scenarios we reccommend a timeout of at least one hour \n",
|
||||||
|
" \"max_concurrent_iterations\": 4,\n",
|
||||||
|
" \"max_cores_per_iteration\": -1,\n",
|
||||||
|
" \"verbosity\": logging.INFO,\n",
|
||||||
|
"}\n",
|
||||||
|
"\n",
|
||||||
|
"automl_config = AutoMLConfig(task = 'regression',\n",
|
||||||
|
" compute_target = compute_target,\n",
|
||||||
|
" training_data = train_data,\n",
|
||||||
|
" label_column_name = label,\n",
|
||||||
|
" **automl_settings\n",
|
||||||
|
" )"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Call the `submit` method on the experiment object and pass the run configuration. Execution of remote runs is asynchronous. Depending on the data and the number of iterations this can run for a while. Validation errors and current status will be shown when setting `show_output=True` and the execution will be synchronous."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"remote_run = experiment.submit(automl_config, show_output = False)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# If you need to retrieve a run that already started, use the following code\n",
|
||||||
|
"#from azureml.train.automl.run import AutoMLRun\n",
|
||||||
|
"#remote_run = AutoMLRun(experiment = experiment, run_id = '<replace with your run id>')"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"remote_run"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Results"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"#### Widget for Monitoring Runs\n",
|
||||||
|
"\n",
|
||||||
|
"The widget will first report a \"loading\" status while running the first iteration. After completing the first iteration, an auto-updating graph and table will be shown. The widget will refresh once per minute, so you should see the graph update as child runs complete.\n",
|
||||||
|
"\n",
|
||||||
|
"**Note:** The widget displays a link at the bottom. Use this link to open a web interface to explore the individual run details."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"from azureml.widgets import RunDetails\n",
|
||||||
|
"RunDetails(remote_run).show() "
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"remote_run.wait_for_completion()"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Retrieve the Best Child Run\n",
|
||||||
|
"\n",
|
||||||
|
"Below we select the best pipeline from our iterations. The `get_best_child` method returns the best run. Overloads on `get_best_child` allow you to retrieve the best run for *any* logged metric."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"best_run = remote_run.get_best_child()\n",
|
||||||
|
"print(best_run)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"#### Best Child Run Based on Any Other Metric\n",
|
||||||
|
"Show the run and the model that has the smallest `root_mean_squared_error` value (which turned out to be the same as the one with largest `spearman_correlation` value):"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"lookup_metric = \"root_mean_squared_error\"\n",
|
||||||
|
"best_run = remote_run.get_best_child(metric = lookup_metric)\n",
|
||||||
|
"print(best_run)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# preview the first 3 rows of the dataset\n",
|
||||||
|
"\n",
|
||||||
|
"test_data = test_data.to_pandas_dataframe()\n",
|
||||||
|
"y_test = test_data['ERP'].fillna(0)\n",
|
||||||
|
"test_data = test_data.drop('ERP', 1)\n",
|
||||||
|
"test_data = test_data.fillna(0)\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"train_data = train_data.to_pandas_dataframe()\n",
|
||||||
|
"y_train = train_data['ERP'].fillna(0)\n",
|
||||||
|
"train_data = train_data.drop('ERP', 1)\n",
|
||||||
|
"train_data = train_data.fillna(0)\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"#### Creating ModelProxy for submitting prediction runs to the training environment.\n",
|
||||||
|
"We will create a ModelProxy for the best child run, which will allow us to submit a run that does the prediction in the training environment. Unlike the local client, which can have different versions of some libraries, the training environment will have all the compatible libraries for the model already."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"from azureml.train.automl.model_proxy import ModelProxy\n",
|
||||||
|
"best_model_proxy = ModelProxy(best_run)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"y_pred_train = best_model_proxy.predict(train_data).to_pandas_dataframe()\n",
|
||||||
|
"y_residual_train = y_train - y_pred_train\n",
|
||||||
|
"\n",
|
||||||
|
"y_pred_test = best_model_proxy.predict(test_data).to_pandas_dataframe()\n",
|
||||||
|
"y_residual_test = y_test - y_pred_test"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"%matplotlib inline\n",
|
||||||
|
"from sklearn.metrics import mean_squared_error, r2_score\n",
|
||||||
|
"\n",
|
||||||
|
"# Set up a multi-plot chart.\n",
|
||||||
|
"f, (a0, a1) = plt.subplots(1, 2, gridspec_kw = {'width_ratios':[1, 1], 'wspace':0, 'hspace': 0})\n",
|
||||||
|
"f.suptitle('Regression Residual Values', fontsize = 18)\n",
|
||||||
|
"f.set_figheight(6)\n",
|
||||||
|
"f.set_figwidth(16)\n",
|
||||||
|
"\n",
|
||||||
|
"# Plot residual values of training set.\n",
|
||||||
|
"a0.axis([0, 360, -100, 100])\n",
|
||||||
|
"a0.plot(y_residual_train, 'bo', alpha = 0.5)\n",
|
||||||
|
"a0.plot([-10,360],[0,0], 'r-', lw = 3)\n",
|
||||||
|
"a0.text(16,170,'RMSE = {0:.2f}'.format(np.sqrt(mean_squared_error(y_train, y_pred_train))), fontsize = 12)\n",
|
||||||
|
"a0.text(16,140,'R2 score = {0:.2f}'.format(r2_score(y_train, y_pred_train)),fontsize = 12)\n",
|
||||||
|
"a0.set_xlabel('Training samples', fontsize = 12)\n",
|
||||||
|
"a0.set_ylabel('Residual Values', fontsize = 12)\n",
|
||||||
|
"\n",
|
||||||
|
"# Plot residual values of test set.\n",
|
||||||
|
"a1.axis([0, 90, -100, 100])\n",
|
||||||
|
"a1.plot(y_residual_test, 'bo', alpha = 0.5)\n",
|
||||||
|
"a1.plot([-10,360],[0,0], 'r-', lw = 3)\n",
|
||||||
|
"a1.text(5,170,'RMSE = {0:.2f}'.format(np.sqrt(mean_squared_error(y_test, y_pred_test))), fontsize = 12)\n",
|
||||||
|
"a1.text(5,140,'R2 score = {0:.2f}'.format(r2_score(y_test, y_pred_test)),fontsize = 12)\n",
|
||||||
|
"a1.set_xlabel('Test samples', fontsize = 12)\n",
|
||||||
|
"a1.set_yticklabels([])\n",
|
||||||
|
"\n",
|
||||||
|
"plt.show()"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"%matplotlib inline\n",
|
||||||
|
"test_pred = plt.scatter(y_test, y_pred_test, color='')\n",
|
||||||
|
"test_test = plt.scatter(y_test, y_test, color='g')\n",
|
||||||
|
"plt.legend((test_pred, test_test), ('prediction', 'truth'), loc='upper left', fontsize=8)\n",
|
||||||
|
"plt.show()"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"authors": [
|
||||||
|
{
|
||||||
|
"name": "rakellam"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"categories": [
|
||||||
|
"how-to-use-azureml",
|
||||||
|
"automated-machine-learning"
|
||||||
|
],
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3.6",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python36"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.6.2"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 2
|
||||||
|
}
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
name: auto-ml-regression-model-proxy
|
||||||
|
dependencies:
|
||||||
|
- pip:
|
||||||
|
- azureml-sdk
|
||||||
@@ -114,7 +114,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"print(\"This notebook was created using version 1.10.0 of the Azure ML SDK\")\n",
|
"print(\"This notebook was created using version 1.13.0 of the Azure ML SDK\")\n",
|
||||||
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
|
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -217,7 +217,7 @@
|
|||||||
"\n",
|
"\n",
|
||||||
"**Time column** is the time axis along which to predict.\n",
|
"**Time column** is the time axis along which to predict.\n",
|
||||||
"\n",
|
"\n",
|
||||||
"**Grain** is another word for an individual time series in your dataset. Grains are identified by values of the columns listed `grain_column_names`, for example \"store\" and \"item\" if your data has multiple time series of sales, one series for each combination of store and item sold.\n",
|
"**Time series identifier columns** are identified by values of the columns listed `time_series_id_column_names`, for example \"store\" and \"item\" if your data has multiple time series of sales, one series for each combination of store and item sold.\n",
|
||||||
"\n",
|
"\n",
|
||||||
"This dataset has only one time series. Please see the [orange juice notebook](https://github.com/Azure/MachineLearningNotebooks/tree/master/how-to-use-azureml/automated-machine-learning/forecasting-orange-juice-sales) for an example of a multi-time series dataset."
|
"This dataset has only one time series. Please see the [orange juice notebook](https://github.com/Azure/MachineLearningNotebooks/tree/master/how-to-use-azureml/automated-machine-learning/forecasting-orange-juice-sales) for an example of a multi-time series dataset."
|
||||||
]
|
]
|
||||||
@@ -269,7 +269,7 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"target_column_name = 'BeerProduction'\n",
|
"target_column_name = 'BeerProduction'\n",
|
||||||
"time_column_name = 'DATE'\n",
|
"time_column_name = 'DATE'\n",
|
||||||
"grain_column_names = []\n",
|
"time_series_id_column_names = []\n",
|
||||||
"freq = 'M' #Monthly data"
|
"freq = 'M' #Monthly data"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -329,7 +329,7 @@
|
|||||||
},
|
},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"max_horizon = 12"
|
"forecast_horizon = 12"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -364,11 +364,10 @@
|
|||||||
},
|
},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"automl_settings = {\n",
|
"from azureml.automl.core.forecasting_parameters import ForecastingParameters\n",
|
||||||
" 'time_column_name': time_column_name,\n",
|
"forecasting_parameters = ForecastingParameters(\n",
|
||||||
" 'max_horizon': max_horizon,\n",
|
" time_column_name=time_column_name, forecast_horizon=forecast_horizon\n",
|
||||||
" 'enable_dnn' : True,\n",
|
")\n",
|
||||||
"}\n",
|
|
||||||
"\n",
|
"\n",
|
||||||
"automl_config = AutoMLConfig(task='forecasting', \n",
|
"automl_config = AutoMLConfig(task='forecasting', \n",
|
||||||
" primary_metric='normalized_root_mean_squared_error',\n",
|
" primary_metric='normalized_root_mean_squared_error',\n",
|
||||||
@@ -380,7 +379,8 @@
|
|||||||
" compute_target=compute_target,\n",
|
" compute_target=compute_target,\n",
|
||||||
" max_concurrent_iterations=4,\n",
|
" max_concurrent_iterations=4,\n",
|
||||||
" max_cores_per_iteration=-1,\n",
|
" max_cores_per_iteration=-1,\n",
|
||||||
" **automl_settings)"
|
" enable_dnn=True,\n",
|
||||||
|
" forecasting_parameters=forecasting_parameters)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -390,7 +390,7 @@
|
|||||||
"hidePrompt": false
|
"hidePrompt": false
|
||||||
},
|
},
|
||||||
"source": [
|
"source": [
|
||||||
"We will now run the experiment, starting with 10 iterations of model search. The experiment can be continued for more iterations if more accurate results are required."
|
"We will now run the experiment, starting with 10 iterations of model search. The experiment can be continued for more iterations if more accurate results are required. Validation errors and current status will be shown when setting `show_output=True` and the execution will be synchronous."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -581,7 +581,7 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"from helper import run_inference\n",
|
"from helper import run_inference\n",
|
||||||
"\n",
|
"\n",
|
||||||
"test_run = run_inference(test_experiment, compute_target, script_folder, best_dnn_run, test_dataset, valid_dataset, max_horizon,\n",
|
"test_run = run_inference(test_experiment, compute_target, script_folder, best_dnn_run, test_dataset, valid_dataset, forecast_horizon,\n",
|
||||||
" target_column_name, time_column_name, freq)"
|
" target_column_name, time_column_name, freq)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -603,7 +603,7 @@
|
|||||||
"from helper import run_multiple_inferences\n",
|
"from helper import run_multiple_inferences\n",
|
||||||
"\n",
|
"\n",
|
||||||
"summary_df = run_multiple_inferences(summary_df, experiment, test_experiment, compute_target, script_folder, test_dataset, \n",
|
"summary_df = run_multiple_inferences(summary_df, experiment, test_experiment, compute_target, script_folder, test_dataset, \n",
|
||||||
" valid_dataset, max_horizon, target_column_name, time_column_name, freq)"
|
" valid_dataset, forecast_horizon, target_column_name, time_column_name, freq)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -87,7 +87,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"print(\"This notebook was created using version 1.10.0 of the Azure ML SDK\")\n",
|
"print(\"This notebook was created using version 1.13.0 of the Azure ML SDK\")\n",
|
||||||
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
|
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -238,6 +238,22 @@
|
|||||||
"test.to_pandas_dataframe().head(5).reset_index(drop=True)"
|
"test.to_pandas_dataframe().head(5).reset_index(drop=True)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Forecasting Parameters\n",
|
||||||
|
"To define forecasting parameters for your experiment training, you can leverage the ForecastingParameters class. The table below details the forecasting parameter we will be passing into our experiment.\n",
|
||||||
|
"\n",
|
||||||
|
"|Property|Description|\n",
|
||||||
|
"|-|-|\n",
|
||||||
|
"|**time_column_name**|The name of your time column.|\n",
|
||||||
|
"|**forecast_horizon**|The forecast horizon is how many periods forward you would like to forecast. This integer horizon is in units of the timeseries frequency (e.g. daily, weekly).|\n",
|
||||||
|
"|**country_or_region_for_holidays**|The country/region used to generate holiday features. These should be ISO 3166 two-letter country/region codes (i.e. 'US', 'GB').|\n",
|
||||||
|
"|**target_lags**|The target_lags specifies how far back we will construct the lags of the target variable.|\n",
|
||||||
|
"|**drop_column_names**|Name(s) of columns to drop prior to modeling|"
|
||||||
|
]
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
@@ -257,11 +273,7 @@
|
|||||||
"|**compute_target**|The remote compute for training.|\n",
|
"|**compute_target**|The remote compute for training.|\n",
|
||||||
"|**n_cross_validations**|Number of cross validation splits.|\n",
|
"|**n_cross_validations**|Number of cross validation splits.|\n",
|
||||||
"|**enable_early_stopping**|If early stopping is on, training will stop when the primary metric is no longer improving.|\n",
|
"|**enable_early_stopping**|If early stopping is on, training will stop when the primary metric is no longer improving.|\n",
|
||||||
"|**time_column_name**|Name of the datetime column in the input data|\n",
|
"|**forecasting_parameters**|A class that holds all the forecasting related parameters.|\n",
|
||||||
"|**max_horizon**|Maximum desired forecast horizon in units of time-series frequency|\n",
|
|
||||||
"|**country_or_region**|The country/region used to generate holiday features. These should be ISO 3166 two-letter country/region codes (i.e. 'US', 'GB').|\n",
|
|
||||||
"|**target_lags**|The target_lags specifies how far back we will construct the lags of the target variable.|\n",
|
|
||||||
"|**drop_column_names**|Name(s) of columns to drop prior to modeling|\n",
|
|
||||||
"\n",
|
"\n",
|
||||||
"This notebook uses the blocked_models parameter to exclude some models that take a longer time to train on this dataset. You can choose to remove models from the blocked_models list but you may need to increase the experiment_timeout_hours parameter value to get results."
|
"This notebook uses the blocked_models parameter to exclude some models that take a longer time to train on this dataset. You can choose to remove models from the blocked_models list but you may need to increase the experiment_timeout_hours parameter value to get results."
|
||||||
]
|
]
|
||||||
@@ -281,7 +293,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"max_horizon = 14"
|
"forecast_horizon = 14"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -297,13 +309,14 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"time_series_settings = {\n",
|
"from azureml.automl.core.forecasting_parameters import ForecastingParameters\n",
|
||||||
" 'time_column_name': time_column_name,\n",
|
"forecasting_parameters = ForecastingParameters(\n",
|
||||||
" 'max_horizon': max_horizon, \n",
|
" time_column_name=time_column_name,\n",
|
||||||
" 'country_or_region': 'US', # set country_or_region will trigger holiday featurizer\n",
|
" forecast_horizon=forecast_horizon,\n",
|
||||||
" 'target_lags': 'auto', # use heuristic based lag setting \n",
|
" country_or_region_for_holidays='US', # set country_or_region will trigger holiday featurizer\n",
|
||||||
" 'drop_column_names': ['casual', 'registered'] # these columns are a breakdown of the total and therefore a leak\n",
|
" target_lags='auto', # use heuristic based lag setting \n",
|
||||||
"}\n",
|
" drop_column_names=['casual', 'registered'] # these columns are a breakdown of the total and therefore a leak\n",
|
||||||
|
")\n",
|
||||||
"\n",
|
"\n",
|
||||||
"automl_config = AutoMLConfig(task='forecasting', \n",
|
"automl_config = AutoMLConfig(task='forecasting', \n",
|
||||||
" primary_metric='normalized_root_mean_squared_error',\n",
|
" primary_metric='normalized_root_mean_squared_error',\n",
|
||||||
@@ -317,7 +330,7 @@
|
|||||||
" max_concurrent_iterations=4,\n",
|
" max_concurrent_iterations=4,\n",
|
||||||
" max_cores_per_iteration=-1,\n",
|
" max_cores_per_iteration=-1,\n",
|
||||||
" verbosity=logging.INFO,\n",
|
" verbosity=logging.INFO,\n",
|
||||||
" **time_series_settings)"
|
" forecasting_parameters=forecasting_parameters)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -422,7 +435,7 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"We now use the best fitted model from the AutoML Run to make forecasts for the test set. We will do batch scoring on the test dataset which should have the same schema as training dataset.\n",
|
"We now use the best fitted model from the AutoML Run to make forecasts for the test set. We will do batch scoring on the test dataset which should have the same schema as training dataset.\n",
|
||||||
"\n",
|
"\n",
|
||||||
"The scoring will run on a remote compute. In this example, it will reuse the training compute.|"
|
"The scoring will run on a remote compute. In this example, it will reuse the training compute."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -439,7 +452,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"### Retrieving forecasts from the model\n",
|
"### Retrieving forecasts from the model\n",
|
||||||
"To run the forecast on the remote compute we will use two helper scripts: forecasting_script and forecasting_helper. These scripts contain the utility methods which will be used by the remote estimator. We copy these scripts to the project folder to upload them to remote compute."
|
"To run the forecast on the remote compute we will use a helper script: forecasting_script. This script contains the utility methods which will be used by the remote estimator. We copy the script to the project folder to upload it to remote compute."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -453,15 +466,14 @@
|
|||||||
"\n",
|
"\n",
|
||||||
"script_folder = os.path.join(os.getcwd(), 'forecast')\n",
|
"script_folder = os.path.join(os.getcwd(), 'forecast')\n",
|
||||||
"os.makedirs(script_folder, exist_ok=True)\n",
|
"os.makedirs(script_folder, exist_ok=True)\n",
|
||||||
"shutil.copy('forecasting_script.py', script_folder)\n",
|
"shutil.copy('forecasting_script.py', script_folder)"
|
||||||
"shutil.copy('forecasting_helper.py', script_folder)"
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"For brevity we have created the function called run_forecast. It submits the test data to the best model and run the estimation on the selected compute target."
|
"For brevity, we have created a function called run_forecast that submits the test data to the best model determined during the training run and retrieves forecasts. The test set is longer than the forecast horizon specified at train time, so the forecasting script uses a so-called rolling evaluation to generate predictions over the whole test set. A rolling evaluation iterates the forecaster over the test set, using the actuals in the test set to make lag features as needed. "
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -472,8 +484,7 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"from run_forecast import run_rolling_forecast\n",
|
"from run_forecast import run_rolling_forecast\n",
|
||||||
"\n",
|
"\n",
|
||||||
"remote_run = run_rolling_forecast(test_experiment, compute_target, best_run, test, max_horizon,\n",
|
"remote_run = run_rolling_forecast(test_experiment, compute_target, best_run, test, target_column_name)\n",
|
||||||
" target_column_name, time_column_name)\n",
|
|
||||||
"remote_run"
|
"remote_run"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -537,7 +548,7 @@
|
|||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"The MAPE seems high; it is being skewed by an actual with a small absolute value. For a more informative evaluation, we can calculate the metrics by forecast horizon:"
|
"Since we did a rolling evaluation on the test set, we can analyze the predictions by their forecast horizon relative to the rolling origin. The model was initially trained at a forecast horizon of 14, so each prediction from the model is associated with a horizon value from 1 to 14. The horizon values are in a column named, \"horizon_origin,\" in the prediction set. For example, we can calculate some of the error metrics grouped by the horizon:"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -557,7 +568,7 @@
|
|||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"It's also interesting to see the distributions of APE (absolute percentage error) by horizon. On a log scale, the outlying APE in the horizon-3 group is clear."
|
"To drill down more, we can look at the distributions of APE (absolute percentage error) by horizon. From the chart, it is clear that the overall MAPE is being skewed by one particular point where the actual value is of small absolute value."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -567,7 +578,7 @@
|
|||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"df_all_APE = df_all.assign(APE=APE(df_all[target_column_name], df_all['predicted']))\n",
|
"df_all_APE = df_all.assign(APE=APE(df_all[target_column_name], df_all['predicted']))\n",
|
||||||
"APEs = [df_all_APE[df_all['horizon_origin'] == h].APE.values for h in range(1, max_horizon + 1)]\n",
|
"APEs = [df_all_APE[df_all['horizon_origin'] == h].APE.values for h in range(1, forecast_horizon + 1)]\n",
|
||||||
"\n",
|
"\n",
|
||||||
"%matplotlib inline\n",
|
"%matplotlib inline\n",
|
||||||
"plt.boxplot(APEs)\n",
|
"plt.boxplot(APEs)\n",
|
||||||
@@ -631,5 +642,5 @@
|
|||||||
"version": 3
|
"version": 3
|
||||||
},
|
},
|
||||||
"nbformat": 4,
|
"nbformat": 4,
|
||||||
"nbformat_minor": 2
|
"nbformat_minor": 4
|
||||||
}
|
}
|
||||||
@@ -1,99 +0,0 @@
|
|||||||
import pandas as pd
|
|
||||||
import numpy as np
|
|
||||||
from pandas.tseries.frequencies import to_offset
|
|
||||||
|
|
||||||
|
|
||||||
def align_outputs(y_predicted, X_trans, X_test, y_test, target_column_name,
|
|
||||||
predicted_column_name='predicted',
|
|
||||||
horizon_colname='horizon_origin'):
|
|
||||||
"""
|
|
||||||
Demonstrates how to get the output aligned to the inputs
|
|
||||||
using pandas indexes. Helps understand what happened if
|
|
||||||
the output's shape differs from the input shape, or if
|
|
||||||
the data got re-sorted by time and grain during forecasting.
|
|
||||||
|
|
||||||
Typical causes of misalignment are:
|
|
||||||
* we predicted some periods that were missing in actuals -> drop from eval
|
|
||||||
* model was asked to predict past max_horizon -> increase max horizon
|
|
||||||
* data at start of X_test was needed for lags -> provide previous periods
|
|
||||||
"""
|
|
||||||
|
|
||||||
if (horizon_colname in X_trans):
|
|
||||||
df_fcst = pd.DataFrame({predicted_column_name: y_predicted,
|
|
||||||
horizon_colname: X_trans[horizon_colname]})
|
|
||||||
else:
|
|
||||||
df_fcst = pd.DataFrame({predicted_column_name: y_predicted})
|
|
||||||
|
|
||||||
# y and X outputs are aligned by forecast() function contract
|
|
||||||
df_fcst.index = X_trans.index
|
|
||||||
|
|
||||||
# align original X_test to y_test
|
|
||||||
X_test_full = X_test.copy()
|
|
||||||
X_test_full[target_column_name] = y_test
|
|
||||||
|
|
||||||
# X_test_full's index does not include origin, so reset for merge
|
|
||||||
df_fcst.reset_index(inplace=True)
|
|
||||||
X_test_full = X_test_full.reset_index().drop(columns='index')
|
|
||||||
together = df_fcst.merge(X_test_full, how='right')
|
|
||||||
|
|
||||||
# drop rows where prediction or actuals are nan
|
|
||||||
# happens because of missing actuals
|
|
||||||
# or at edges of time due to lags/rolling windows
|
|
||||||
clean = together[together[[target_column_name,
|
|
||||||
predicted_column_name]].notnull().all(axis=1)]
|
|
||||||
return(clean)
|
|
||||||
|
|
||||||
|
|
||||||
def do_rolling_forecast(fitted_model, X_test, y_test, target_column_name,
|
|
||||||
time_column_name, max_horizon, freq='D'):
|
|
||||||
"""
|
|
||||||
Produce forecasts on a rolling origin over the given test set.
|
|
||||||
|
|
||||||
Each iteration makes a forecast for the next 'max_horizon' periods
|
|
||||||
with respect to the current origin, then advances the origin by the
|
|
||||||
horizon time duration. The prediction context for each forecast is set so
|
|
||||||
that the forecaster uses the actual target values prior to the current
|
|
||||||
origin time for constructing lag features.
|
|
||||||
|
|
||||||
This function returns a concatenated DataFrame of rolling forecasts.
|
|
||||||
"""
|
|
||||||
df_list = []
|
|
||||||
origin_time = X_test[time_column_name].min()
|
|
||||||
while origin_time <= X_test[time_column_name].max():
|
|
||||||
# Set the horizon time - end date of the forecast
|
|
||||||
horizon_time = origin_time + max_horizon * to_offset(freq)
|
|
||||||
|
|
||||||
# Extract test data from an expanding window up-to the horizon
|
|
||||||
expand_wind = (X_test[time_column_name] < horizon_time)
|
|
||||||
X_test_expand = X_test[expand_wind]
|
|
||||||
y_query_expand = np.zeros(len(X_test_expand)).astype(np.float)
|
|
||||||
y_query_expand.fill(np.NaN)
|
|
||||||
|
|
||||||
if origin_time != X_test[time_column_name].min():
|
|
||||||
# Set the context by including actuals up-to the origin time
|
|
||||||
test_context_expand_wind = (X_test[time_column_name] < origin_time)
|
|
||||||
context_expand_wind = (
|
|
||||||
X_test_expand[time_column_name] < origin_time)
|
|
||||||
y_query_expand[context_expand_wind] = y_test[
|
|
||||||
test_context_expand_wind]
|
|
||||||
|
|
||||||
# Make a forecast out to the maximum horizon
|
|
||||||
y_fcst, X_trans = fitted_model.forecast(X_test_expand, y_query_expand)
|
|
||||||
|
|
||||||
# Align forecast with test set for dates within the
|
|
||||||
# current rolling window
|
|
||||||
trans_tindex = X_trans.index.get_level_values(time_column_name)
|
|
||||||
trans_roll_wind = (trans_tindex >= origin_time) & (
|
|
||||||
trans_tindex < horizon_time)
|
|
||||||
test_roll_wind = expand_wind & (
|
|
||||||
X_test[time_column_name] >= origin_time)
|
|
||||||
df_list.append(align_outputs(y_fcst[trans_roll_wind],
|
|
||||||
X_trans[trans_roll_wind],
|
|
||||||
X_test[test_roll_wind],
|
|
||||||
y_test[test_roll_wind],
|
|
||||||
target_column_name))
|
|
||||||
|
|
||||||
# Advance the origin time
|
|
||||||
origin_time = horizon_time
|
|
||||||
|
|
||||||
return pd.concat(df_list, ignore_index=True)
|
|
||||||
@@ -1,37 +1,21 @@
|
|||||||
import argparse
|
import argparse
|
||||||
import azureml.train.automl
|
import azureml.train.automl
|
||||||
from azureml.automl.runtime.shared import forecasting_models
|
|
||||||
from azureml.core import Run
|
from azureml.core import Run
|
||||||
from sklearn.externals import joblib
|
from sklearn.externals import joblib
|
||||||
import forecasting_helper
|
|
||||||
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser()
|
parser = argparse.ArgumentParser()
|
||||||
parser.add_argument(
|
|
||||||
'--max_horizon', type=int, dest='max_horizon',
|
|
||||||
default=10, help='Max Horizon for forecasting')
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'--target_column_name', type=str, dest='target_column_name',
|
'--target_column_name', type=str, dest='target_column_name',
|
||||||
help='Target Column Name')
|
help='Target Column Name')
|
||||||
parser.add_argument(
|
|
||||||
'--time_column_name', type=str, dest='time_column_name',
|
|
||||||
help='Time Column Name')
|
|
||||||
parser.add_argument(
|
|
||||||
'--frequency', type=str, dest='freq',
|
|
||||||
help='Frequency of prediction')
|
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
max_horizon = args.max_horizon
|
|
||||||
target_column_name = args.target_column_name
|
target_column_name = args.target_column_name
|
||||||
time_column_name = args.time_column_name
|
|
||||||
freq = args.freq
|
|
||||||
|
|
||||||
run = Run.get_context()
|
run = Run.get_context()
|
||||||
# get input dataset by name
|
# get input dataset by name
|
||||||
test_dataset = run.input_datasets['test_data']
|
test_dataset = run.input_datasets['test_data']
|
||||||
|
|
||||||
grain_column_names = []
|
|
||||||
|
|
||||||
df = test_dataset.to_pandas_dataframe().reset_index(drop=True)
|
df = test_dataset.to_pandas_dataframe().reset_index(drop=True)
|
||||||
|
|
||||||
X_test_df = test_dataset.drop_columns(columns=[target_column_name]).to_pandas_dataframe().reset_index(drop=True)
|
X_test_df = test_dataset.drop_columns(columns=[target_column_name]).to_pandas_dataframe().reset_index(drop=True)
|
||||||
@@ -39,14 +23,12 @@ y_test_df = test_dataset.with_timestamp_columns(None).keep_columns(columns=[targ
|
|||||||
|
|
||||||
fitted_model = joblib.load('model.pkl')
|
fitted_model = joblib.load('model.pkl')
|
||||||
|
|
||||||
df_all = forecasting_helper.do_rolling_forecast(
|
y_pred, X_trans = fitted_model.rolling_evaluation(X_test_df, y_test_df.values)
|
||||||
fitted_model,
|
|
||||||
X_test_df,
|
# Add predictions, actuals, and horizon relative to rolling origin to the test feature data
|
||||||
y_test_df.values.T[0],
|
assign_dict = {'horizon_origin': X_trans['horizon_origin'].values, 'predicted': y_pred,
|
||||||
target_column_name,
|
target_column_name: y_test_df[target_column_name].values}
|
||||||
time_column_name,
|
df_all = X_test_df.assign(**assign_dict)
|
||||||
max_horizon,
|
|
||||||
freq)
|
|
||||||
|
|
||||||
file_name = 'outputs/predictions.csv'
|
file_name = 'outputs/predictions.csv'
|
||||||
export_csv = df_all.to_csv(file_name, header=True)
|
export_csv = df_all.to_csv(file_name, header=True)
|
||||||
|
|||||||
@@ -5,8 +5,7 @@ from azureml.core.run import Run
|
|||||||
|
|
||||||
|
|
||||||
def run_rolling_forecast(test_experiment, compute_target, train_run, test_dataset,
|
def run_rolling_forecast(test_experiment, compute_target, train_run, test_dataset,
|
||||||
max_horizon, target_column_name, time_column_name,
|
target_column_name, inference_folder='./forecast'):
|
||||||
freq='D', inference_folder='./forecast'):
|
|
||||||
condafile = inference_folder + '/condafile.yml'
|
condafile = inference_folder + '/condafile.yml'
|
||||||
train_run.download_file('outputs/model.pkl',
|
train_run.download_file('outputs/model.pkl',
|
||||||
inference_folder + '/model.pkl')
|
inference_folder + '/model.pkl')
|
||||||
@@ -20,10 +19,7 @@ def run_rolling_forecast(test_experiment, compute_target, train_run, test_datase
|
|||||||
est = Estimator(source_directory=inference_folder,
|
est = Estimator(source_directory=inference_folder,
|
||||||
entry_script='forecasting_script.py',
|
entry_script='forecasting_script.py',
|
||||||
script_params={
|
script_params={
|
||||||
'--max_horizon': max_horizon,
|
'--target_column_name': target_column_name
|
||||||
'--target_column_name': target_column_name,
|
|
||||||
'--time_column_name': time_column_name,
|
|
||||||
'--frequency': freq
|
|
||||||
},
|
},
|
||||||
inputs=[test_dataset.as_named_input('test_data')],
|
inputs=[test_dataset.as_named_input('test_data')],
|
||||||
compute_target=compute_target,
|
compute_target=compute_target,
|
||||||
|
|||||||
@@ -97,7 +97,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"print(\"This notebook was created using version 1.10.0 of the Azure ML SDK\")\n",
|
"print(\"This notebook was created using version 1.13.0 of the Azure ML SDK\")\n",
|
||||||
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
|
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -288,7 +288,20 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"max_horizon = 48"
|
"forecast_horizon = 48"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Forecasting Parameters\n",
|
||||||
|
"To define forecasting parameters for your experiment training, you can leverage the ForecastingParameters class. The table below details the forecasting parameter we will be passing into our experiment.\n",
|
||||||
|
"\n",
|
||||||
|
"|Property|Description|\n",
|
||||||
|
"|-|-|\n",
|
||||||
|
"|**time_column_name**|The name of your time column.|\n",
|
||||||
|
"|**forecast_horizon**|The forecast horizon is how many periods forward you would like to forecast. This integer horizon is in units of the timeseries frequency (e.g. daily, weekly).|"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -297,7 +310,7 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"## Train\n",
|
"## Train\n",
|
||||||
"\n",
|
"\n",
|
||||||
"Instantiate an AutoMLConfig object. This config defines the settings and data used to run the experiment. We can provide extra configurations within 'automl_settings', for this forecasting task we add the name of the time column and the maximum forecast horizon.\n",
|
"Instantiate an AutoMLConfig object. This config defines the settings and data used to run the experiment. We can provide extra configurations within 'automl_settings', for this forecasting task we add the forecasting parameters to hold all the additional forecasting parameters.\n",
|
||||||
"\n",
|
"\n",
|
||||||
"|Property|Description|\n",
|
"|Property|Description|\n",
|
||||||
"|-|-|\n",
|
"|-|-|\n",
|
||||||
@@ -310,8 +323,7 @@
|
|||||||
"|**compute_target**|The remote compute for training.|\n",
|
"|**compute_target**|The remote compute for training.|\n",
|
||||||
"|**n_cross_validations**|Number of cross validation splits. Rolling Origin Validation is used to split time-series in a temporally consistent way.|\n",
|
"|**n_cross_validations**|Number of cross validation splits. Rolling Origin Validation is used to split time-series in a temporally consistent way.|\n",
|
||||||
"|**enable_early_stopping**|Flag to enble early termination if the score is not improving in the short term.|\n",
|
"|**enable_early_stopping**|Flag to enble early termination if the score is not improving in the short term.|\n",
|
||||||
"|**time_column_name**|The name of your time column.|\n",
|
"|**forecasting_parameters**|A class holds all the forecasting related parameters.|\n"
|
||||||
"|**max_horizon**|The number of periods out you would like to predict past your training data. Periods are inferred from your data.|\n"
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -327,10 +339,10 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"automl_settings = {\n",
|
"from azureml.automl.core.forecasting_parameters import ForecastingParameters\n",
|
||||||
" 'time_column_name': time_column_name,\n",
|
"forecasting_parameters = ForecastingParameters(\n",
|
||||||
" 'max_horizon': max_horizon,\n",
|
" time_column_name=time_column_name, forecast_horizon=forecast_horizon\n",
|
||||||
"}\n",
|
")\n",
|
||||||
"\n",
|
"\n",
|
||||||
"automl_config = AutoMLConfig(task='forecasting', \n",
|
"automl_config = AutoMLConfig(task='forecasting', \n",
|
||||||
" primary_metric='normalized_root_mean_squared_error',\n",
|
" primary_metric='normalized_root_mean_squared_error',\n",
|
||||||
@@ -342,7 +354,7 @@
|
|||||||
" enable_early_stopping=True,\n",
|
" enable_early_stopping=True,\n",
|
||||||
" n_cross_validations=3, \n",
|
" n_cross_validations=3, \n",
|
||||||
" verbosity=logging.INFO,\n",
|
" verbosity=logging.INFO,\n",
|
||||||
" **automl_settings)"
|
" forecasting_parameters=forecasting_parameters)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -550,7 +562,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"## Advanced Training <a id=\"advanced_training\"></a>\n",
|
"## Advanced Training <a id=\"advanced_training\"></a>\n",
|
||||||
"We did not use lags in the previous model specification. In effect, the prediction was the result of a simple regression on date, grain and any additional features. This is often a very good prediction as common time series patterns like seasonality and trends can be captured in this manner. Such simple regression is horizon-less: it doesn't matter how far into the future we are predicting, because we are not using past data. In the previous example, the horizon was only used to split the data for cross-validation."
|
"We did not use lags in the previous model specification. In effect, the prediction was the result of a simple regression on date, time series identifier columns and any additional features. This is often a very good prediction as common time series patterns like seasonality and trends can be captured in this manner. Such simple regression is horizon-less: it doesn't matter how far into the future we are predicting, because we are not using past data. In the previous example, the horizon was only used to split the data for cross-validation."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -558,7 +570,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"### Using lags and rolling window features\n",
|
"### Using lags and rolling window features\n",
|
||||||
"Now we will configure the target lags, that is the previous values of the target variables, meaning the prediction is no longer horizon-less. We therefore must still specify the `max_horizon` that the model will learn to forecast. The `target_lags` keyword specifies how far back we will construct the lags of the target variable, and the `target_rolling_window_size` specifies the size of the rolling window over which we will generate the `max`, `min` and `sum` features.\n",
|
"Now we will configure the target lags, that is the previous values of the target variables, meaning the prediction is no longer horizon-less. We therefore must still specify the `forecast_horizon` that the model will learn to forecast. The `target_lags` keyword specifies how far back we will construct the lags of the target variable, and the `target_rolling_window_size` specifies the size of the rolling window over which we will generate the `max`, `min` and `sum` features.\n",
|
||||||
"\n",
|
"\n",
|
||||||
"This notebook uses the blocked_models parameter to exclude some models that take a longer time to train on this dataset. You can choose to remove models from the blocked_models list but you may need to increase the iteration_timeout_minutes parameter value to get results."
|
"This notebook uses the blocked_models parameter to exclude some models that take a longer time to train on this dataset. You can choose to remove models from the blocked_models list but you may need to increase the iteration_timeout_minutes parameter value to get results."
|
||||||
]
|
]
|
||||||
@@ -569,12 +581,10 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"automl_advanced_settings = {\n",
|
"advanced_forecasting_parameters = ForecastingParameters(\n",
|
||||||
" 'time_column_name': time_column_name,\n",
|
" time_column_name=time_column_name, forecast_horizon=forecast_horizon,\n",
|
||||||
" 'max_horizon': max_horizon,\n",
|
" target_lags=12, target_rolling_window_size=4\n",
|
||||||
" 'target_lags': 12,\n",
|
")\n",
|
||||||
" 'target_rolling_window_size': 4,\n",
|
|
||||||
"}\n",
|
|
||||||
"\n",
|
"\n",
|
||||||
"automl_config = AutoMLConfig(task='forecasting', \n",
|
"automl_config = AutoMLConfig(task='forecasting', \n",
|
||||||
" primary_metric='normalized_root_mean_squared_error',\n",
|
" primary_metric='normalized_root_mean_squared_error',\n",
|
||||||
@@ -586,7 +596,7 @@
|
|||||||
" enable_early_stopping = True,\n",
|
" enable_early_stopping = True,\n",
|
||||||
" n_cross_validations=3, \n",
|
" n_cross_validations=3, \n",
|
||||||
" verbosity=logging.INFO,\n",
|
" verbosity=logging.INFO,\n",
|
||||||
" **automl_advanced_settings)"
|
" forecasting_parameters=advanced_forecasting_parameters)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -635,7 +645,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"## Advanced Results<a id=\"advanced_results\"></a>\n",
|
"## Advanced Results<a id=\"advanced_results\"></a>\n",
|
||||||
"We did not use lags in the previous model specification. In effect, the prediction was the result of a simple regression on date, grain and any additional features. This is often a very good prediction as common time series patterns like seasonality and trends can be captured in this manner. Such simple regression is horizon-less: it doesn't matter how far into the future we are predicting, because we are not using past data. In the previous example, the horizon was only used to split the data for cross-validation."
|
"We did not use lags in the previous model specification. In effect, the prediction was the result of a simple regression on date, time series identifier columns and any additional features. This is often a very good prediction as common time series patterns like seasonality and trends can be captured in this manner. Such simple regression is horizon-less: it doesn't matter how far into the future we are predicting, because we are not using past data. In the previous example, the horizon was only used to split the data for cross-validation."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -94,7 +94,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"print(\"This notebook was created using version 1.10.0 of the Azure ML SDK\")\n",
|
"print(\"This notebook was created using version 1.13.0 of the Azure ML SDK\")\n",
|
||||||
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
|
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -142,15 +142,15 @@
|
|||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"TIME_COLUMN_NAME = 'date'\n",
|
"TIME_COLUMN_NAME = 'date'\n",
|
||||||
"GRAIN_COLUMN_NAME = 'grain'\n",
|
"TIME_SERIES_ID_COLUMN_NAME = 'time_series_id'\n",
|
||||||
"TARGET_COLUMN_NAME = 'y'\n",
|
"TARGET_COLUMN_NAME = 'y'\n",
|
||||||
"\n",
|
"\n",
|
||||||
"def get_timeseries(train_len: int,\n",
|
"def get_timeseries(train_len: int,\n",
|
||||||
" test_len: int,\n",
|
" test_len: int,\n",
|
||||||
" time_column_name: str,\n",
|
" time_column_name: str,\n",
|
||||||
" target_column_name: str,\n",
|
" target_column_name: str,\n",
|
||||||
" grain_column_name: str,\n",
|
" time_series_id_column_name: str,\n",
|
||||||
" grains: int = 1,\n",
|
" time_series_number: int = 1,\n",
|
||||||
" freq: str = 'H'):\n",
|
" freq: str = 'H'):\n",
|
||||||
" \"\"\"\n",
|
" \"\"\"\n",
|
||||||
" Return the time series of designed length.\n",
|
" Return the time series of designed length.\n",
|
||||||
@@ -161,9 +161,8 @@
|
|||||||
" :type test_len: int\n",
|
" :type test_len: int\n",
|
||||||
" :param time_column_name: The desired name of a time column.\n",
|
" :param time_column_name: The desired name of a time column.\n",
|
||||||
" :type time_column_name: str\n",
|
" :type time_column_name: str\n",
|
||||||
" :param\n",
|
" :param time_series_number: The number of time series in the data set.\n",
|
||||||
" :param grains: The number of grains.\n",
|
" :type time_series_number: int\n",
|
||||||
" :type grains: int\n",
|
|
||||||
" :param freq: The frequency string representing pandas offset.\n",
|
" :param freq: The frequency string representing pandas offset.\n",
|
||||||
" see https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html\n",
|
" see https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html\n",
|
||||||
" :type freq: str\n",
|
" :type freq: str\n",
|
||||||
@@ -174,14 +173,14 @@
|
|||||||
" data_train = [] # type: List[pd.DataFrame]\n",
|
" data_train = [] # type: List[pd.DataFrame]\n",
|
||||||
" data_test = [] # type: List[pd.DataFrame]\n",
|
" data_test = [] # type: List[pd.DataFrame]\n",
|
||||||
" data_length = train_len + test_len\n",
|
" data_length = train_len + test_len\n",
|
||||||
" for i in range(grains):\n",
|
" for i in range(time_series_number):\n",
|
||||||
" X = pd.DataFrame({\n",
|
" X = pd.DataFrame({\n",
|
||||||
" time_column_name: pd.date_range(start='2000-01-01',\n",
|
" time_column_name: pd.date_range(start='2000-01-01',\n",
|
||||||
" periods=data_length,\n",
|
" periods=data_length,\n",
|
||||||
" freq=freq),\n",
|
" freq=freq),\n",
|
||||||
" target_column_name: np.arange(data_length).astype(float) + np.random.rand(data_length) + i*5,\n",
|
" target_column_name: np.arange(data_length).astype(float) + np.random.rand(data_length) + i*5,\n",
|
||||||
" 'ext_predictor': np.asarray(range(42, 42 + data_length)),\n",
|
" 'ext_predictor': np.asarray(range(42, 42 + data_length)),\n",
|
||||||
" grain_column_name: np.repeat('g{}'.format(i), data_length)\n",
|
" time_series_id_column_name: np.repeat('ts{}'.format(i), data_length)\n",
|
||||||
" })\n",
|
" })\n",
|
||||||
" data_train.append(X[:train_len])\n",
|
" data_train.append(X[:train_len])\n",
|
||||||
" data_test.append(X[train_len:])\n",
|
" data_test.append(X[train_len:])\n",
|
||||||
@@ -197,8 +196,8 @@
|
|||||||
" test_len=n_test_periods,\n",
|
" test_len=n_test_periods,\n",
|
||||||
" time_column_name=TIME_COLUMN_NAME,\n",
|
" time_column_name=TIME_COLUMN_NAME,\n",
|
||||||
" target_column_name=TARGET_COLUMN_NAME,\n",
|
" target_column_name=TARGET_COLUMN_NAME,\n",
|
||||||
" grain_column_name=GRAIN_COLUMN_NAME,\n",
|
" time_series_id_column_name=TIME_SERIES_ID_COLUMN_NAME,\n",
|
||||||
" grains=2)"
|
" time_series_number=2)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -228,7 +227,7 @@
|
|||||||
"whole_data = X_train.copy()\n",
|
"whole_data = X_train.copy()\n",
|
||||||
"target_label = 'y'\n",
|
"target_label = 'y'\n",
|
||||||
"whole_data[target_label] = y_train\n",
|
"whole_data[target_label] = y_train\n",
|
||||||
"for g in whole_data.groupby('grain'): \n",
|
"for g in whole_data.groupby('time_series_id'): \n",
|
||||||
" plt.plot(g[1]['date'].values, g[1]['y'].values, label=g[0])\n",
|
" plt.plot(g[1]['date'].values, g[1]['y'].values, label=g[0])\n",
|
||||||
"plt.legend()\n",
|
"plt.legend()\n",
|
||||||
"plt.show()"
|
"plt.show()"
|
||||||
@@ -297,7 +296,7 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"## Create the configuration and train a forecaster <a id=\"train\"></a>\n",
|
"## Create the configuration and train a forecaster <a id=\"train\"></a>\n",
|
||||||
"First generate the configuration, in which we:\n",
|
"First generate the configuration, in which we:\n",
|
||||||
"* Set metadata columns: target, time column and grain column names.\n",
|
"* Set metadata columns: target, time column and time-series id column names.\n",
|
||||||
"* Validate our data using cross validation with rolling window method.\n",
|
"* Validate our data using cross validation with rolling window method.\n",
|
||||||
"* Set normalized root mean squared error as a metric to select the best model.\n",
|
"* Set normalized root mean squared error as a metric to select the best model.\n",
|
||||||
"* Set early termination to True, so the iterations through the models will stop when no improvements in accuracy score will be made.\n",
|
"* Set early termination to True, so the iterations through the models will stop when no improvements in accuracy score will be made.\n",
|
||||||
@@ -312,21 +311,22 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
|
"from azureml.automl.core.forecasting_parameters import ForecastingParameters\n",
|
||||||
"lags = [1,2,3]\n",
|
"lags = [1,2,3]\n",
|
||||||
"max_horizon = n_test_periods\n",
|
"forecast_horizon = n_test_periods\n",
|
||||||
"time_series_settings = { \n",
|
"forecasting_parameters = ForecastingParameters(\n",
|
||||||
" 'time_column_name': TIME_COLUMN_NAME,\n",
|
" time_column_name=TIME_COLUMN_NAME,\n",
|
||||||
" 'grain_column_names': [ GRAIN_COLUMN_NAME ],\n",
|
" forecast_horizon=forecast_horizon,\n",
|
||||||
" 'max_horizon': max_horizon,\n",
|
" time_series_id_column_names=[ TIME_SERIES_ID_COLUMN_NAME ],\n",
|
||||||
" 'target_lags': lags\n",
|
" target_lags=lags\n",
|
||||||
"}"
|
")"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"Run the model selection and training process."
|
"Run the model selection and training process. Validation errors and current status will be shown when setting `show_output=True` and the execution will be synchronous."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -352,7 +352,7 @@
|
|||||||
" max_concurrent_iterations=4,\n",
|
" max_concurrent_iterations=4,\n",
|
||||||
" max_cores_per_iteration=-1,\n",
|
" max_cores_per_iteration=-1,\n",
|
||||||
" label_column_name=target_label,\n",
|
" label_column_name=target_label,\n",
|
||||||
" **time_series_settings)\n",
|
" forecasting_parameters=forecasting_parameters)\n",
|
||||||
"\n",
|
"\n",
|
||||||
"remote_run = experiment.submit(automl_config, show_output=False)"
|
"remote_run = experiment.submit(automl_config, show_output=False)"
|
||||||
]
|
]
|
||||||
@@ -482,7 +482,7 @@
|
|||||||
"# use forecast_quantiles function, not the forecast() one\n",
|
"# use forecast_quantiles function, not the forecast() one\n",
|
||||||
"y_pred_quantiles = fitted_model.forecast_quantiles(X_test)\n",
|
"y_pred_quantiles = fitted_model.forecast_quantiles(X_test)\n",
|
||||||
"\n",
|
"\n",
|
||||||
"# quantile forecasts returned in a Dataframe along with the time and grain columns \n",
|
"# quantile forecasts returned in a Dataframe along with the time and time series id columns \n",
|
||||||
"y_pred_quantiles"
|
"y_pred_quantiles"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -492,7 +492,7 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"#### Destination-date forecast: \"just do something\"\n",
|
"#### Destination-date forecast: \"just do something\"\n",
|
||||||
"\n",
|
"\n",
|
||||||
"In some scenarios, the X_test is not known. The forecast is likely to be weak, because it is missing contemporaneous predictors, which we will need to impute. If you still wish to predict forward under the assumption that the last known values will be carried forward, you can forecast out to \"destination date\". The destination date still needs to fit within the maximum horizon from training."
|
"In some scenarios, the X_test is not known. The forecast is likely to be weak, because it is missing contemporaneous predictors, which we will need to impute. If you still wish to predict forward under the assumption that the last known values will be carried forward, you can forecast out to \"destination date\". The destination date still needs to fit within the forecast horizon from training."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -519,7 +519,7 @@
|
|||||||
"\n",
|
"\n",
|
||||||
"\n",
|
"\n",
|
||||||
"\n",
|
"\n",
|
||||||
"The notion of forecast origin comes into play: the forecast origin is **the last period for which we have seen the target value**. This applies per grain, so each grain can have a different forecast origin. \n",
|
"The notion of forecast origin comes into play: the forecast origin is **the last period for which we have seen the target value**. This applies per time-series, so each time-series can have a different forecast origin. \n",
|
||||||
"\n",
|
"\n",
|
||||||
"The part of data before the forecast origin is the **prediction context**. To provide the context values the model needs when it looks back, we pass definite values in `y_test` (aligned with corresponding times in `X_test`)."
|
"The part of data before the forecast origin is the **prediction context**. To provide the context values the model needs when it looks back, we pass definite values in `y_test` (aligned with corresponding times in `X_test`)."
|
||||||
]
|
]
|
||||||
@@ -536,13 +536,13 @@
|
|||||||
" test_len=4,\n",
|
" test_len=4,\n",
|
||||||
" time_column_name=TIME_COLUMN_NAME,\n",
|
" time_column_name=TIME_COLUMN_NAME,\n",
|
||||||
" target_column_name=TARGET_COLUMN_NAME,\n",
|
" target_column_name=TARGET_COLUMN_NAME,\n",
|
||||||
" grain_column_name=GRAIN_COLUMN_NAME,\n",
|
" time_series_id_column_name=TIME_SERIES_ID_COLUMN_NAME,\n",
|
||||||
" grains=2)\n",
|
" time_series_number=2)\n",
|
||||||
"\n",
|
"\n",
|
||||||
"# end of the data we trained on\n",
|
"# end of the data we trained on\n",
|
||||||
"print(X_train.groupby(GRAIN_COLUMN_NAME)[TIME_COLUMN_NAME].max())\n",
|
"print(X_train.groupby(TIME_SERIES_ID_COLUMN_NAME)[TIME_COLUMN_NAME].max())\n",
|
||||||
"# start of the data we want to predict on\n",
|
"# start of the data we want to predict on\n",
|
||||||
"print(X_away.groupby(GRAIN_COLUMN_NAME)[TIME_COLUMN_NAME].min())"
|
"print(X_away.groupby(TIME_SERIES_ID_COLUMN_NAME)[TIME_COLUMN_NAME].min())"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -569,7 +569,7 @@
|
|||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"How should we read that eror message? The forecast origin is at the last time the model saw an actual value of `y` (the target). That was at the end of the training data! The model is attempting to forecast from the end of training data. But the requested forecast periods are past the maximum horizon. We need to provide a define `y` value to establish the forecast origin.\n",
|
"How should we read that eror message? The forecast origin is at the last time the model saw an actual value of `y` (the target). That was at the end of the training data! The model is attempting to forecast from the end of training data. But the requested forecast periods are past the forecast horizon. We need to provide a define `y` value to establish the forecast origin.\n",
|
||||||
"\n",
|
"\n",
|
||||||
"We will use this helper function to take the required amount of context from the data preceding the testing data. It's definition is intentionally simplified to keep the idea in the clear."
|
"We will use this helper function to take the required amount of context from the data preceding the testing data. It's definition is intentionally simplified to keep the idea in the clear."
|
||||||
]
|
]
|
||||||
@@ -584,7 +584,7 @@
|
|||||||
"\n",
|
"\n",
|
||||||
" \"\"\"\n",
|
" \"\"\"\n",
|
||||||
" This function will take the full dataset, and create the query\n",
|
" This function will take the full dataset, and create the query\n",
|
||||||
" to predict all values of the grain from the `forecast_origin`\n",
|
" to predict all values of the time series from the `forecast_origin`\n",
|
||||||
" forward for the next `horizon` horizons. Context from previous\n",
|
" forward for the next `horizon` horizons. Context from previous\n",
|
||||||
" `lookback` periods will be included.\n",
|
" `lookback` periods will be included.\n",
|
||||||
"\n",
|
"\n",
|
||||||
@@ -654,8 +654,8 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"print(X_context.groupby(GRAIN_COLUMN_NAME)[TIME_COLUMN_NAME].agg(['min','max','count']))\n",
|
"print(X_context.groupby(TIME_SERIES_ID_COLUMN_NAME)[TIME_COLUMN_NAME].agg(['min','max','count']))\n",
|
||||||
"print(X_away.groupby(GRAIN_COLUMN_NAME)[TIME_COLUMN_NAME].agg(['min','max','count']))\n",
|
"print(X_away.groupby(TIME_SERIES_ID_COLUMN_NAME)[TIME_COLUMN_NAME].agg(['min','max','count']))\n",
|
||||||
"X_context.tail(5)"
|
"X_context.tail(5)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -685,7 +685,7 @@
|
|||||||
"n_lookback_periods = max(lags)\n",
|
"n_lookback_periods = max(lags)\n",
|
||||||
"lookback = pd.DateOffset(hours=n_lookback_periods)\n",
|
"lookback = pd.DateOffset(hours=n_lookback_periods)\n",
|
||||||
"\n",
|
"\n",
|
||||||
"horizon = pd.DateOffset(hours=max_horizon)\n",
|
"horizon = pd.DateOffset(hours=forecast_horizon)\n",
|
||||||
"\n",
|
"\n",
|
||||||
"# now make the forecast query from context (refer to figure)\n",
|
"# now make the forecast query from context (refer to figure)\n",
|
||||||
"X_pred, y_pred = make_forecasting_query(fulldata, TIME_COLUMN_NAME, TARGET_COLUMN_NAME,\n",
|
"X_pred, y_pred = make_forecasting_query(fulldata, TIME_COLUMN_NAME, TARGET_COLUMN_NAME,\n",
|
||||||
@@ -701,7 +701,7 @@
|
|||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"Note that the forecast origin is at 17:00 for both grains, and periods from 18:00 are to be forecast."
|
"Note that the forecast origin is at 17:00 for both time-series, and periods from 18:00 are to be forecast."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -716,7 +716,7 @@
|
|||||||
"# show the forecast aligned\n",
|
"# show the forecast aligned\n",
|
||||||
"X_show = xy_away.reset_index()\n",
|
"X_show = xy_away.reset_index()\n",
|
||||||
"# without the generated features\n",
|
"# without the generated features\n",
|
||||||
"X_show[['date', 'grain', 'ext_predictor', '_automl_target_col']]\n",
|
"X_show[['date', 'time_series_id', 'ext_predictor', '_automl_target_col']]\n",
|
||||||
"# prediction is in _automl_target_col"
|
"# prediction is in _automl_target_col"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -724,14 +724,14 @@
|
|||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"## Forecasting farther than the maximum horizon <a id=\"recursive forecasting\"></a>\n",
|
"## Forecasting farther than the forecast horizon <a id=\"recursive forecasting\"></a>\n",
|
||||||
"When the forecast destination, or the latest date in the prediction data frame, is farther into the future than the specified maximum horizon, the `forecast()` function will still make point predictions out to the later date using a recursive operation mode. Internally, the method recursively applies the regular forecaster to generate context so that we can forecast further into the future. \n",
|
"When the forecast destination, or the latest date in the prediction data frame, is farther into the future than the specified forecast horizon, the `forecast()` function will still make point predictions out to the later date using a recursive operation mode. Internally, the method recursively applies the regular forecaster to generate context so that we can forecast further into the future. \n",
|
||||||
"\n",
|
"\n",
|
||||||
"To illustrate the use-case and operation of recursive forecasting, we'll consider an example with a single time-series where the forecasting period directly follows the training period and is twice as long as the maximum horizon given at training time.\n",
|
"To illustrate the use-case and operation of recursive forecasting, we'll consider an example with a single time-series where the forecasting period directly follows the training period and is twice as long as the forecasting horizon given at training time.\n",
|
||||||
"\n",
|
"\n",
|
||||||
"\n",
|
"\n",
|
||||||
"\n",
|
"\n",
|
||||||
"Internally, we apply the forecaster in an iterative manner and finish the forecast task in two interations. In the first iteration, we apply the forecaster and get the prediction for the first max-horizon periods (y_pred1). In the second iteraction, y_pred1 is used as the context to produce the prediction for the next max-horizon periods (y_pred2). The combination of (y_pred1 and y_pred2) gives the results for the total forecast periods. \n",
|
"Internally, we apply the forecaster in an iterative manner and finish the forecast task in two interations. In the first iteration, we apply the forecaster and get the prediction for the first forecast-horizon periods (y_pred1). In the second iteraction, y_pred1 is used as the context to produce the prediction for the next forecast-horizon periods (y_pred2). The combination of (y_pred1 and y_pred2) gives the results for the total forecast periods. \n",
|
||||||
"\n",
|
"\n",
|
||||||
"A caveat: forecast accuracy will likely be worse the farther we predict into the future since errors are compounded with recursive application of the forecaster.\n",
|
"A caveat: forecast accuracy will likely be worse the farther we predict into the future since errors are compounded with recursive application of the forecaster.\n",
|
||||||
"\n",
|
"\n",
|
||||||
@@ -745,16 +745,17 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"# generate the same kind of test data we trained on, but with a single grain/time-series and test period twice as long as the max_horizon\n",
|
"# generate the same kind of test data we trained on, but with a single time-series and test period twice as long\n",
|
||||||
|
"# as the forecast_horizon.\n",
|
||||||
"_, _, X_test_long, y_test_long = get_timeseries(train_len=n_train_periods,\n",
|
"_, _, X_test_long, y_test_long = get_timeseries(train_len=n_train_periods,\n",
|
||||||
" test_len=max_horizon*2,\n",
|
" test_len=forecast_horizon*2,\n",
|
||||||
" time_column_name=TIME_COLUMN_NAME,\n",
|
" time_column_name=TIME_COLUMN_NAME,\n",
|
||||||
" target_column_name=TARGET_COLUMN_NAME,\n",
|
" target_column_name=TARGET_COLUMN_NAME,\n",
|
||||||
" grain_column_name=GRAIN_COLUMN_NAME,\n",
|
" time_series_id_column_name=TIME_SERIES_ID_COLUMN_NAME,\n",
|
||||||
" grains=1)\n",
|
" time_series_number=1)\n",
|
||||||
"\n",
|
"\n",
|
||||||
"print(X_test_long.groupby(GRAIN_COLUMN_NAME)[TIME_COLUMN_NAME].min())\n",
|
"print(X_test_long.groupby(TIME_SERIES_ID_COLUMN_NAME)[TIME_COLUMN_NAME].min())\n",
|
||||||
"print(X_test_long.groupby(GRAIN_COLUMN_NAME)[TIME_COLUMN_NAME].max())"
|
"print(X_test_long.groupby(TIME_SERIES_ID_COLUMN_NAME)[TIME_COLUMN_NAME].max())"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -775,8 +776,8 @@
|
|||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"# What forecast() function does in this case is equivalent to iterating it twice over the test set as the following. \n",
|
"# What forecast() function does in this case is equivalent to iterating it twice over the test set as the following. \n",
|
||||||
"y_pred1, _ = fitted_model.forecast(X_test_long[:max_horizon])\n",
|
"y_pred1, _ = fitted_model.forecast(X_test_long[:forecast_horizon])\n",
|
||||||
"y_pred_all, _ = fitted_model.forecast(X_test_long, np.concatenate((y_pred1, np.full(max_horizon, np.nan))))\n",
|
"y_pred_all, _ = fitted_model.forecast(X_test_long, np.concatenate((y_pred1, np.full(forecast_horizon, np.nan))))\n",
|
||||||
"np.array_equal(y_pred_all, y_pred_long)"
|
"np.array_equal(y_pred_all, y_pred_long)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -785,7 +786,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"#### Confidence interval and distributional forecasts\n",
|
"#### Confidence interval and distributional forecasts\n",
|
||||||
"AutoML cannot currently estimate forecast errors beyond the maximum horizon set during training, so the `forecast_quantiles()` function will return missing values for quantiles not equal to 0.5 beyond the maximum horizon. "
|
"AutoML cannot currently estimate forecast errors beyond the forecast horizon set during training, so the `forecast_quantiles()` function will return missing values for quantiles not equal to 0.5 beyond the forecast horizon. "
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -801,7 +802,7 @@
|
|||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"Similarly with the simple senarios illustrated above, forecasting farther than the max horizon in other senarios like 'multiple grain', 'Destination-date forecast', and 'forecast away from the training data' are also automatically handled by the `forecast()` function. "
|
"Similarly with the simple senarios illustrated above, forecasting farther than the forecast horizon in other senarios like 'multiple time-series', 'Destination-date forecast', and 'forecast away from the training data' are also automatically handled by the `forecast()` function. "
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
|
|||||||
Binary file not shown.
|
Before Width: | Height: | Size: 24 KiB After Width: | Height: | Size: 69 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 24 KiB After Width: | Height: | Size: 65 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 26 KiB After Width: | Height: | Size: 28 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 30 KiB After Width: | Height: | Size: 61 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 21 KiB After Width: | Height: | Size: 25 KiB |
@@ -82,7 +82,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"print(\"This notebook was created using version 1.10.0 of the Azure ML SDK\")\n",
|
"print(\"This notebook was created using version 1.13.0 of the Azure ML SDK\")\n",
|
||||||
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
|
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -178,7 +178,7 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"Each row in the DataFrame holds a quantity of weekly sales for an OJ brand at a single store. The data also includes the sales price, a flag indicating if the OJ brand was advertised in the store that week, and some customer demographic information based on the store location. For historical reasons, the data also include the logarithm of the sales quantity. The Dominick's grocery data is commonly used to illustrate econometric modeling techniques where logarithms of quantities are generally preferred. \n",
|
"Each row in the DataFrame holds a quantity of weekly sales for an OJ brand at a single store. The data also includes the sales price, a flag indicating if the OJ brand was advertised in the store that week, and some customer demographic information based on the store location. For historical reasons, the data also include the logarithm of the sales quantity. The Dominick's grocery data is commonly used to illustrate econometric modeling techniques where logarithms of quantities are generally preferred. \n",
|
||||||
"\n",
|
"\n",
|
||||||
"The task is now to build a time-series model for the _Quantity_ column. It is important to note that this dataset is comprised of many individual time-series - one for each unique combination of _Store_ and _Brand_. To distinguish the individual time-series, we thus define the **grain** - the columns whose values determine the boundaries between time-series: "
|
"The task is now to build a time-series model for the _Quantity_ column. It is important to note that this dataset is comprised of many individual time-series - one for each unique combination of _Store_ and _Brand_. To distinguish the individual time-series, we define the **time_series_id_column_names** - the columns whose values determine the boundaries between time-series: "
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -187,8 +187,8 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"grain_column_names = ['Store', 'Brand']\n",
|
"time_series_id_column_names = ['Store', 'Brand']\n",
|
||||||
"nseries = data.groupby(grain_column_names).ngroups\n",
|
"nseries = data.groupby(time_series_id_column_names).ngroups\n",
|
||||||
"print('Data contains {0} individual time-series.'.format(nseries))"
|
"print('Data contains {0} individual time-series.'.format(nseries))"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -207,7 +207,7 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"use_stores = [2, 5, 8]\n",
|
"use_stores = [2, 5, 8]\n",
|
||||||
"data_subset = data[data.Store.isin(use_stores)]\n",
|
"data_subset = data[data.Store.isin(use_stores)]\n",
|
||||||
"nseries = data_subset.groupby(grain_column_names).ngroups\n",
|
"nseries = data_subset.groupby(time_series_id_column_names).ngroups\n",
|
||||||
"print('Data subset contains {0} individual time-series.'.format(nseries))"
|
"print('Data subset contains {0} individual time-series.'.format(nseries))"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -216,7 +216,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"### Data Splitting\n",
|
"### Data Splitting\n",
|
||||||
"We now split the data into a training and a testing set for later forecast evaluation. The test set will contain the final 20 weeks of observed sales for each time-series. The splits should be stratified by series, so we use a group-by statement on the grain columns."
|
"We now split the data into a training and a testing set for later forecast evaluation. The test set will contain the final 20 weeks of observed sales for each time-series. The splits should be stratified by series, so we use a group-by statement on the time series identifier columns."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -227,15 +227,15 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"n_test_periods = 20\n",
|
"n_test_periods = 20\n",
|
||||||
"\n",
|
"\n",
|
||||||
"def split_last_n_by_grain(df, n):\n",
|
"def split_last_n_by_series_id(df, n):\n",
|
||||||
" \"\"\"Group df by grain and split on last n rows for each group.\"\"\"\n",
|
" \"\"\"Group df by series identifiers and split on last n rows for each group.\"\"\"\n",
|
||||||
" df_grouped = (df.sort_values(time_column_name) # Sort by ascending time\n",
|
" df_grouped = (df.sort_values(time_column_name) # Sort by ascending time\n",
|
||||||
" .groupby(grain_column_names, group_keys=False))\n",
|
" .groupby(time_series_id_column_names, group_keys=False))\n",
|
||||||
" df_head = df_grouped.apply(lambda dfg: dfg.iloc[:-n])\n",
|
" df_head = df_grouped.apply(lambda dfg: dfg.iloc[:-n])\n",
|
||||||
" df_tail = df_grouped.apply(lambda dfg: dfg.iloc[-n:])\n",
|
" df_tail = df_grouped.apply(lambda dfg: dfg.iloc[-n:])\n",
|
||||||
" return df_head, df_tail\n",
|
" return df_head, df_tail\n",
|
||||||
"\n",
|
"\n",
|
||||||
"train, test = split_last_n_by_grain(data_subset, n_test_periods)"
|
"train, test = split_last_n_by_series_id(data_subset, n_test_periods)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -301,11 +301,11 @@
|
|||||||
"For forecasting tasks, AutoML uses pre-processing and estimation steps that are specific to time-series. AutoML will undertake the following pre-processing steps:\n",
|
"For forecasting tasks, AutoML uses pre-processing and estimation steps that are specific to time-series. AutoML will undertake the following pre-processing steps:\n",
|
||||||
"* Detect time-series sample frequency (e.g. hourly, daily, weekly) and create new records for absent time points to make the series regular. A regular time series has a well-defined frequency and has a value at every sample point in a contiguous time span \n",
|
"* Detect time-series sample frequency (e.g. hourly, daily, weekly) and create new records for absent time points to make the series regular. A regular time series has a well-defined frequency and has a value at every sample point in a contiguous time span \n",
|
||||||
"* Impute missing values in the target (via forward-fill) and feature columns (using median column values) \n",
|
"* Impute missing values in the target (via forward-fill) and feature columns (using median column values) \n",
|
||||||
"* Create grain-based features to enable fixed effects across different series\n",
|
"* Create features based on time series identifiers to enable fixed effects across different series\n",
|
||||||
"* Create time-based features to assist in learning seasonal patterns\n",
|
"* Create time-based features to assist in learning seasonal patterns\n",
|
||||||
"* Encode categorical variables to numeric quantities\n",
|
"* Encode categorical variables to numeric quantities\n",
|
||||||
"\n",
|
"\n",
|
||||||
"In this notebook, AutoML will train a single, regression-type model across **all** time-series in a given training set. This allows the model to generalize across related series. If you're looking for training multiple models for different time-series, please check out the forecasting grouping notebook. \n",
|
"In this notebook, AutoML will train a single, regression-type model across **all** time-series in a given training set. This allows the model to generalize across related series. If you're looking for training multiple models for different time-series, please see the many-models notebook.\n",
|
||||||
"\n",
|
"\n",
|
||||||
"You are almost ready to start an AutoML training job. First, we need to separate the target column from the rest of the DataFrame: "
|
"You are almost ready to start an AutoML training job. First, we need to separate the target column from the rest of the DataFrame: "
|
||||||
]
|
]
|
||||||
@@ -327,7 +327,7 @@
|
|||||||
"\n",
|
"\n",
|
||||||
"The featurization customization in forecasting is an advanced feature in AutoML which allows our customers to change the default forecasting featurization behaviors and column types through `FeaturizationConfig`. The supported scenarios include,\n",
|
"The featurization customization in forecasting is an advanced feature in AutoML which allows our customers to change the default forecasting featurization behaviors and column types through `FeaturizationConfig`. The supported scenarios include,\n",
|
||||||
"1. Column purposes update: Override feature type for the specified column. Currently supports DateTime, Categorical and Numeric. This customization can be used in the scenario that the type of the column cannot correctly reflect its purpose. Some numerical columns, for instance, can be treated as Categorical columns which need to be converted to categorical while some can be treated as epoch timestamp which need to be converted to datetime. To tell our SDK to correctly preprocess these columns, a configuration need to be add with the columns and their desired types.\n",
|
"1. Column purposes update: Override feature type for the specified column. Currently supports DateTime, Categorical and Numeric. This customization can be used in the scenario that the type of the column cannot correctly reflect its purpose. Some numerical columns, for instance, can be treated as Categorical columns which need to be converted to categorical while some can be treated as epoch timestamp which need to be converted to datetime. To tell our SDK to correctly preprocess these columns, a configuration need to be add with the columns and their desired types.\n",
|
||||||
"2. Transformer parameters update: Currently supports parameter change for Imputer only. User can customize imputation methods, the supported methods are constant for target data and mean, median, most frequent and constant for training data. This customization can be used for the scenario that our customers know which imputation methods fit best to the input data. For instance, some datasets use NaN to represent 0 which the correct behavior should impute all the missing value with 0. To achieve this behavior, these columns need to be configured as constant imputation with `fill_value` 0.\n",
|
"2. Transformer parameters update: Currently supports parameter change for Imputer only. User can customize imputation methods. The supported imputing methods for target column are constant and ffill (forward fill). The supported imputing methods for feature columns are mean, median, most frequent, constant and ffill (forward fill). This customization can be used for the scenario that our customers know which imputation methods fit best to the input data. For instance, some datasets use NaN to represent 0 which the correct behavior should impute all the missing value with 0. To achieve this behavior, these columns need to be configured as constant imputation with `fill_value` 0.\n",
|
||||||
"3. Drop columns: Columns to drop from being featurized. These usually are the columns which are leaky or the columns contain no useful data.\n",
|
"3. Drop columns: Columns to drop from being featurized. These usually are the columns which are leaky or the columns contain no useful data.\n",
|
||||||
"\n",
|
"\n",
|
||||||
"This step requires an Enterprise workspace to gain access to this feature. To learn more about creating an Enterprise workspace or upgrading to an Enterprise workspace from the Azure portal, please visit our [Workspace page.](https://docs.microsoft.com/azure/machine-learning/service/concept-workspace#upgrade)"
|
"This step requires an Enterprise workspace to gain access to this feature. To learn more about creating an Enterprise workspace or upgrading to an Enterprise workspace from the Azure portal, please visit our [Workspace page.](https://docs.microsoft.com/azure/machine-learning/service/concept-workspace#upgrade)"
|
||||||
@@ -350,7 +350,24 @@
|
|||||||
"# Fill missing values in the target column, Quantity, with zeros.\n",
|
"# Fill missing values in the target column, Quantity, with zeros.\n",
|
||||||
"featurization_config.add_transformer_params('Imputer', ['Quantity'], {\"strategy\": \"constant\", \"fill_value\": 0})\n",
|
"featurization_config.add_transformer_params('Imputer', ['Quantity'], {\"strategy\": \"constant\", \"fill_value\": 0})\n",
|
||||||
"# Fill missing values in the INCOME column with median value.\n",
|
"# Fill missing values in the INCOME column with median value.\n",
|
||||||
"featurization_config.add_transformer_params('Imputer', ['INCOME'], {\"strategy\": \"median\"})"
|
"featurization_config.add_transformer_params('Imputer', ['INCOME'], {\"strategy\": \"median\"})\n",
|
||||||
|
"# Fill missing values in the Price column with forward fill (last value carried forward).\n",
|
||||||
|
"featurization_config.add_transformer_params('Imputer', ['Price'], {\"strategy\": \"ffill\"})"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Forecasting Parameters\n",
|
||||||
|
"To define forecasting parameters for your experiment training, you can leverage the ForecastingParameters class. The table below details the forecasting parameter we will be passing into our experiment.\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"|Property|Description|\n",
|
||||||
|
"|-|-|\n",
|
||||||
|
"|**time_column_name**|The name of your time column.|\n",
|
||||||
|
"|**forecast_horizon**|The forecast horizon is how many periods forward you would like to forecast. This integer horizon is in units of the timeseries frequency (e.g. daily, weekly).|\n",
|
||||||
|
"|**time_series_id_column_names**|The column names used to uniquely identify the time series in data that has multiple rows with the same timestamp. If the time series identifiers are not defined, the data set is assumed to be one time series.|"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -361,9 +378,9 @@
|
|||||||
"\n",
|
"\n",
|
||||||
"The [AutoMLConfig](https://docs.microsoft.com/en-us/python/api/azureml-train-automl-client/azureml.train.automl.automlconfig.automlconfig?view=azure-ml-py) object defines the settings and data for an AutoML training job. Here, we set necessary inputs like the task type, the number of AutoML iterations to try, the training data, and cross-validation parameters.\n",
|
"The [AutoMLConfig](https://docs.microsoft.com/en-us/python/api/azureml-train-automl-client/azureml.train.automl.automlconfig.automlconfig?view=azure-ml-py) object defines the settings and data for an AutoML training job. Here, we set necessary inputs like the task type, the number of AutoML iterations to try, the training data, and cross-validation parameters.\n",
|
||||||
"\n",
|
"\n",
|
||||||
"For forecasting tasks, there are some additional parameters that can be set: the name of the column holding the date/time, the grain column names, and the maximum forecast horizon. A time column is required for forecasting, while the grain is optional. If grain columns are not given, AutoML assumes that the whole dataset is a single time-series. We also pass a list of columns to drop prior to modeling. The _logQuantity_ column is completely correlated with the target quantity, so it must be removed to prevent a target leak.\n",
|
"For forecasting tasks, there are some additional parameters that can be set in the `ForecastingParameters` class: the name of the column holding the date/time, the timeseries id column names, and the maximum forecast horizon. A time column is required for forecasting, while the time_series_id is optional. If time_series_id columns are not given, AutoML assumes that the whole dataset is a single time-series. We also pass a list of columns to drop prior to modeling. The _logQuantity_ column is completely correlated with the target quantity, so it must be removed to prevent a target leak.\n",
|
||||||
"\n",
|
"\n",
|
||||||
"The forecast horizon is given in units of the time-series frequency; for instance, the OJ series frequency is weekly, so a horizon of 20 means that a trained model will estimate sales up to 20 weeks beyond the latest date in the training data for each series. In this example, we set the maximum horizon to the number of samples per series in the test set (n_test_periods). Generally, the value of this parameter will be dictated by business needs. For example, a demand planning application that estimates the next month of sales should set the horizon according to suitable planning time-scales. Please see the [energy_demand notebook](https://github.com/Azure/MachineLearningNotebooks/tree/master/how-to-use-azureml/automated-machine-learning/forecasting-energy-demand) for more discussion of forecast horizon.\n",
|
"The forecast horizon is given in units of the time-series frequency; for instance, the OJ series frequency is weekly, so a horizon of 20 means that a trained model will estimate sales up to 20 weeks beyond the latest date in the training data for each series. In this example, we set the forecast horizon to the number of samples per series in the test set (n_test_periods). Generally, the value of this parameter will be dictated by business needs. For example, a demand planning application that estimates the next month of sales should set the horizon according to suitable planning time-scales. Please see the [energy_demand notebook](https://github.com/Azure/MachineLearningNotebooks/tree/master/how-to-use-azureml/automated-machine-learning/forecasting-energy-demand) for more discussion of forecast horizon.\n",
|
||||||
"\n",
|
"\n",
|
||||||
"We note here that AutoML can sweep over two types of time-series models:\n",
|
"We note here that AutoML can sweep over two types of time-series models:\n",
|
||||||
"* Models that are trained for each series such as ARIMA and Facebook's Prophet. Note that these models are only available for [Enterprise Edition Workspaces](https://docs.microsoft.com/en-us/azure/machine-learning/how-to-manage-workspace#upgrade).\n",
|
"* Models that are trained for each series such as ARIMA and Facebook's Prophet. Note that these models are only available for [Enterprise Edition Workspaces](https://docs.microsoft.com/en-us/azure/machine-learning/how-to-manage-workspace#upgrade).\n",
|
||||||
@@ -389,11 +406,8 @@
|
|||||||
"|**enable_voting_ensemble**|Allow AutoML to create a Voting ensemble of the best performing models|\n",
|
"|**enable_voting_ensemble**|Allow AutoML to create a Voting ensemble of the best performing models|\n",
|
||||||
"|**enable_stack_ensemble**|Allow AutoML to create a Stack ensemble of the best performing models|\n",
|
"|**enable_stack_ensemble**|Allow AutoML to create a Stack ensemble of the best performing models|\n",
|
||||||
"|**debug_log**|Log file path for writing debugging information|\n",
|
"|**debug_log**|Log file path for writing debugging information|\n",
|
||||||
"|**time_column_name**|Name of the datetime column in the input data|\n",
|
|
||||||
"|**grain_column_names**|Name(s) of the columns defining individual series in the input data|\n",
|
|
||||||
"|**max_horizon**|Maximum desired forecast horizon in units of time-series frequency|\n",
|
|
||||||
"|**featurization**| 'auto' / 'off' / FeaturizationConfig Indicator for whether featurization step should be done automatically or not, or whether customized featurization should be used. Setting this enables AutoML to perform featurization on the input to handle *missing data*, and to perform some common *feature extraction*.|\n",
|
"|**featurization**| 'auto' / 'off' / FeaturizationConfig Indicator for whether featurization step should be done automatically or not, or whether customized featurization should be used. Setting this enables AutoML to perform featurization on the input to handle *missing data*, and to perform some common *feature extraction*.|\n",
|
||||||
"|**max_cores_per_iteration**|Maximum number of cores to utilize per iteration. A value of -1 indicates all available cores should be used.|"
|
"|**max_cores_per_iteration**|Maximum number of cores to utilize per iteration. A value of -1 indicates all available cores should be used"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -402,11 +416,12 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"time_series_settings = {\n",
|
"from azureml.automl.core.forecasting_parameters import ForecastingParameters\n",
|
||||||
" 'time_column_name': time_column_name,\n",
|
"forecasting_parameters = ForecastingParameters(\n",
|
||||||
" 'grain_column_names': grain_column_names,\n",
|
" time_column_name=time_column_name,\n",
|
||||||
" 'max_horizon': n_test_periods\n",
|
" forecast_horizon=n_test_periods,\n",
|
||||||
"}\n",
|
" time_series_id_column_names=time_series_id_column_names\n",
|
||||||
|
")\n",
|
||||||
"\n",
|
"\n",
|
||||||
"automl_config = AutoMLConfig(task='forecasting',\n",
|
"automl_config = AutoMLConfig(task='forecasting',\n",
|
||||||
" debug_log='automl_oj_sales_errors.log',\n",
|
" debug_log='automl_oj_sales_errors.log',\n",
|
||||||
@@ -420,7 +435,7 @@
|
|||||||
" n_cross_validations=3,\n",
|
" n_cross_validations=3,\n",
|
||||||
" verbosity=logging.INFO,\n",
|
" verbosity=logging.INFO,\n",
|
||||||
" max_cores_per_iteration=-1,\n",
|
" max_cores_per_iteration=-1,\n",
|
||||||
" **time_series_settings)"
|
" forecasting_parameters=forecasting_parameters)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -428,7 +443,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"You can now submit a new training run. Depending on the data and number of iterations this operation may take several minutes.\n",
|
"You can now submit a new training run. Depending on the data and number of iterations this operation may take several minutes.\n",
|
||||||
"Information from each iteration will be printed to the console."
|
"Information from each iteration will be printed to the console. Validation errors and current status will be shown when setting `show_output=True` and the execution will be synchronous."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -537,9 +552,8 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"# The featurized data, aligned to y, will also be returned.\n",
|
"# forecast returns the predictions and the featurized data, aligned to X_test.\n",
|
||||||
"# This contains the assumptions that were made in the forecast\n",
|
"# This contains the assumptions that were made in the forecast\n",
|
||||||
"# and helps align the forecast to the original data\n",
|
|
||||||
"y_predictions, X_trans = fitted_model.forecast(X_test)"
|
"y_predictions, X_trans = fitted_model.forecast(X_test)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -560,7 +574,7 @@
|
|||||||
"\n",
|
"\n",
|
||||||
"To evaluate the accuracy of the forecast, we'll compare against the actual sales quantities for some select metrics, included the mean absolute percentage error (MAPE). \n",
|
"To evaluate the accuracy of the forecast, we'll compare against the actual sales quantities for some select metrics, included the mean absolute percentage error (MAPE). \n",
|
||||||
"\n",
|
"\n",
|
||||||
"It is a good practice to always align the output explicitly to the input, as the count and order of the rows may have changed during transformations that span multiple rows."
|
"We'll add predictions and actuals into a single dataframe for convenience in calculating the metrics."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -569,9 +583,8 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"from forecasting_helper import align_outputs\n",
|
"assign_dict = {'predicted': y_predictions, target_column_name: y_test}\n",
|
||||||
"\n",
|
"df_all = X_test.assign(**assign_dict)"
|
||||||
"df_all = align_outputs(y_predictions, X_trans, X_test, y_test, target_column_name)"
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -794,5 +807,5 @@
|
|||||||
"task": "Forecasting"
|
"task": "Forecasting"
|
||||||
},
|
},
|
||||||
"nbformat": 4,
|
"nbformat": 4,
|
||||||
"nbformat_minor": 2
|
"nbformat_minor": 4
|
||||||
}
|
}
|
||||||
@@ -1,98 +0,0 @@
|
|||||||
import pandas as pd
|
|
||||||
import numpy as np
|
|
||||||
from pandas.tseries.frequencies import to_offset
|
|
||||||
|
|
||||||
|
|
||||||
def align_outputs(y_predicted, X_trans, X_test, y_test, target_column_name,
|
|
||||||
predicted_column_name='predicted',
|
|
||||||
horizon_colname='horizon_origin'):
|
|
||||||
"""
|
|
||||||
Demonstrates how to get the output aligned to the inputs
|
|
||||||
using pandas indexes. Helps understand what happened if
|
|
||||||
the output's shape differs from the input shape, or if
|
|
||||||
the data got re-sorted by time and grain during forecasting.
|
|
||||||
|
|
||||||
Typical causes of misalignment are:
|
|
||||||
* we predicted some periods that were missing in actuals -> drop from eval
|
|
||||||
* model was asked to predict past max_horizon -> increase max horizon
|
|
||||||
* data at start of X_test was needed for lags -> provide previous periods
|
|
||||||
"""
|
|
||||||
|
|
||||||
if (horizon_colname in X_trans):
|
|
||||||
df_fcst = pd.DataFrame({predicted_column_name: y_predicted,
|
|
||||||
horizon_colname: X_trans[horizon_colname]})
|
|
||||||
else:
|
|
||||||
df_fcst = pd.DataFrame({predicted_column_name: y_predicted})
|
|
||||||
|
|
||||||
# y and X outputs are aligned by forecast() function contract
|
|
||||||
df_fcst.index = X_trans.index
|
|
||||||
|
|
||||||
# align original X_test to y_test
|
|
||||||
X_test_full = X_test.copy()
|
|
||||||
X_test_full[target_column_name] = y_test
|
|
||||||
|
|
||||||
# X_test_full's index does not include origin, so reset for merge
|
|
||||||
df_fcst.reset_index(inplace=True)
|
|
||||||
X_test_full = X_test_full.reset_index().drop(columns='index')
|
|
||||||
together = df_fcst.merge(X_test_full, how='right')
|
|
||||||
|
|
||||||
# drop rows where prediction or actuals are nan
|
|
||||||
# happens because of missing actuals
|
|
||||||
# or at edges of time due to lags/rolling windows
|
|
||||||
clean = together[together[[target_column_name,
|
|
||||||
predicted_column_name]].notnull().all(axis=1)]
|
|
||||||
return(clean)
|
|
||||||
|
|
||||||
|
|
||||||
def do_rolling_forecast(fitted_model, X_test, y_test, target_column_name, time_column_name, max_horizon, freq='D'):
|
|
||||||
"""
|
|
||||||
Produce forecasts on a rolling origin over the given test set.
|
|
||||||
|
|
||||||
Each iteration makes a forecast for the next 'max_horizon' periods
|
|
||||||
with respect to the current origin, then advances the origin by the
|
|
||||||
horizon time duration. The prediction context for each forecast is set so
|
|
||||||
that the forecaster uses the actual target values prior to the current
|
|
||||||
origin time for constructing lag features.
|
|
||||||
|
|
||||||
This function returns a concatenated DataFrame of rolling forecasts.
|
|
||||||
"""
|
|
||||||
df_list = []
|
|
||||||
origin_time = X_test[time_column_name].min()
|
|
||||||
while origin_time <= X_test[time_column_name].max():
|
|
||||||
# Set the horizon time - end date of the forecast
|
|
||||||
horizon_time = origin_time + max_horizon * to_offset(freq)
|
|
||||||
|
|
||||||
# Extract test data from an expanding window up-to the horizon
|
|
||||||
expand_wind = (X_test[time_column_name] < horizon_time)
|
|
||||||
X_test_expand = X_test[expand_wind]
|
|
||||||
y_query_expand = np.zeros(len(X_test_expand)).astype(np.float)
|
|
||||||
y_query_expand.fill(np.NaN)
|
|
||||||
|
|
||||||
if origin_time != X_test[time_column_name].min():
|
|
||||||
# Set the context by including actuals up-to the origin time
|
|
||||||
test_context_expand_wind = (X_test[time_column_name] < origin_time)
|
|
||||||
context_expand_wind = (
|
|
||||||
X_test_expand[time_column_name] < origin_time)
|
|
||||||
y_query_expand[context_expand_wind] = y_test[
|
|
||||||
test_context_expand_wind]
|
|
||||||
|
|
||||||
# Make a forecast out to the maximum horizon
|
|
||||||
y_fcst, X_trans = fitted_model.forecast(X_test_expand, y_query_expand)
|
|
||||||
|
|
||||||
# Align forecast with test set for dates within the
|
|
||||||
# current rolling window
|
|
||||||
trans_tindex = X_trans.index.get_level_values(time_column_name)
|
|
||||||
trans_roll_wind = (trans_tindex >= origin_time) & (
|
|
||||||
trans_tindex < horizon_time)
|
|
||||||
test_roll_wind = expand_wind & (
|
|
||||||
X_test[time_column_name] >= origin_time)
|
|
||||||
df_list.append(align_outputs(y_fcst[trans_roll_wind],
|
|
||||||
X_trans[trans_roll_wind],
|
|
||||||
X_test[test_roll_wind],
|
|
||||||
y_test[test_roll_wind],
|
|
||||||
target_column_name))
|
|
||||||
|
|
||||||
# Advance the origin time
|
|
||||||
origin_time = horizon_time
|
|
||||||
|
|
||||||
return pd.concat(df_list, ignore_index=True)
|
|
||||||
@@ -1,22 +0,0 @@
|
|||||||
import pandas as pd
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
|
|
||||||
def APE(actual, pred):
|
|
||||||
"""
|
|
||||||
Calculate absolute percentage error.
|
|
||||||
Returns a vector of APE values with same length as actual/pred.
|
|
||||||
"""
|
|
||||||
return 100 * np.abs((actual - pred) / actual)
|
|
||||||
|
|
||||||
|
|
||||||
def MAPE(actual, pred):
|
|
||||||
"""
|
|
||||||
Calculate mean absolute percentage error.
|
|
||||||
Remove NA and values where actual is close to zero
|
|
||||||
"""
|
|
||||||
not_na = ~(np.isnan(actual) | np.isnan(pred))
|
|
||||||
not_zero = ~np.isclose(actual, 0.0)
|
|
||||||
actual_safe = actual[not_na & not_zero]
|
|
||||||
pred_safe = pred[not_na & not_zero]
|
|
||||||
return np.mean(APE(actual_safe, pred_safe))
|
|
||||||
@@ -80,7 +80,7 @@
|
|||||||
"from azureml.core.workspace import Workspace\n",
|
"from azureml.core.workspace import Workspace\n",
|
||||||
"from azureml.core.dataset import Dataset\n",
|
"from azureml.core.dataset import Dataset\n",
|
||||||
"from azureml.train.automl import AutoMLConfig\n",
|
"from azureml.train.automl import AutoMLConfig\n",
|
||||||
"from azureml.explain.model._internal.explanation_client import ExplanationClient"
|
"from azureml.interpret._internal.explanation_client import ExplanationClient"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -96,7 +96,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"print(\"This notebook was created using version 1.10.0 of the Azure ML SDK\")\n",
|
"print(\"This notebook was created using version 1.13.0 of the Azure ML SDK\")\n",
|
||||||
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
|
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -354,7 +354,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"## Explanation\n",
|
"## Explanation\n",
|
||||||
"In this section, we will show how to compute model explanations and visualize the explanations using azureml-explain-model package. We will also show how to run the automl model and the explainer model through deploying an AKS web service.\n",
|
"In this section, we will show how to compute model explanations and visualize the explanations using azureml-interpret package. We will also show how to run the automl model and the explainer model through deploying an AKS web service.\n",
|
||||||
"\n",
|
"\n",
|
||||||
"Besides retrieving an existing model explanation for an AutoML model, you can also explain your AutoML model with different test data. The following steps will allow you to compute and visualize engineered feature importance based on your test data.\n",
|
"Besides retrieving an existing model explanation for an AutoML model, you can also explain your AutoML model with different test data. The following steps will allow you to compute and visualize engineered feature importance based on your test data.\n",
|
||||||
"\n",
|
"\n",
|
||||||
@@ -434,7 +434,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"#### Initialize the Mimic Explainer for feature importance\n",
|
"#### Initialize the Mimic Explainer for feature importance\n",
|
||||||
"For explaining the AutoML models, use the MimicWrapper from azureml.explain.model package. The MimicWrapper can be initialized with fields in automl_explainer_setup_obj, your workspace and a surrogate model to explain the AutoML model (fitted_model here). The MimicWrapper also takes the automl_run object where engineered explanations will be uploaded."
|
"For explaining the AutoML models, use the MimicWrapper from azureml-interpret package. The MimicWrapper can be initialized with fields in automl_explainer_setup_obj, your workspace and a surrogate model to explain the AutoML model (fitted_model here). The MimicWrapper also takes the automl_run object where engineered explanations will be uploaded."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -443,7 +443,8 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"from azureml.explain.model.mimic_wrapper import MimicWrapper\n",
|
"from interpret.ext.glassbox import LGBMExplainableModel\n",
|
||||||
|
"from azureml.interpret.mimic_wrapper import MimicWrapper\n",
|
||||||
"explainer = MimicWrapper(ws, automl_explainer_setup_obj.automl_estimator,\n",
|
"explainer = MimicWrapper(ws, automl_explainer_setup_obj.automl_estimator,\n",
|
||||||
" explainable_model=automl_explainer_setup_obj.surrogate_model, \n",
|
" explainable_model=automl_explainer_setup_obj.surrogate_model, \n",
|
||||||
" init_dataset=automl_explainer_setup_obj.X_transform, run=automl_run,\n",
|
" init_dataset=automl_explainer_setup_obj.X_transform, run=automl_run,\n",
|
||||||
@@ -486,7 +487,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"from azureml.explain.model.scoring.scoring_explainer import TreeScoringExplainer\n",
|
"from azureml.interpret.scoring.scoring_explainer import TreeScoringExplainer\n",
|
||||||
"import joblib\n",
|
"import joblib\n",
|
||||||
"\n",
|
"\n",
|
||||||
"# Initialize the ScoringExplainer\n",
|
"# Initialize the ScoringExplainer\n",
|
||||||
@@ -507,7 +508,7 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"### Deploying the scoring and explainer models to a web service to Azure Kubernetes Service (AKS)\n",
|
"### Deploying the scoring and explainer models to a web service to Azure Kubernetes Service (AKS)\n",
|
||||||
"\n",
|
"\n",
|
||||||
"We use the TreeScoringExplainer from azureml.explain.model package to create the scoring explainer which will be used to compute the raw and engineered feature importances at the inference time. In the cell below, we register the AutoML model and the scoring explainer with the Model Management Service."
|
"We use the TreeScoringExplainer from azureml.interpret package to create the scoring explainer which will be used to compute the raw and engineered feature importances at the inference time. In the cell below, we register the AutoML model and the scoring explainer with the Model Management Service."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -529,7 +530,7 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"#### Create the conda dependencies for setting up the service\n",
|
"#### Create the conda dependencies for setting up the service\n",
|
||||||
"\n",
|
"\n",
|
||||||
"We need to create the conda dependencies comprising of the azureml-explain-model, azureml-train-automl and azureml-defaults packages."
|
"We need to download the conda dependencies using the automl_run object."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -561,16 +562,10 @@
|
|||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"%%writefile score.py\n",
|
"%%writefile score.py\n",
|
||||||
"import numpy as np\n",
|
|
||||||
"import pandas as pd\n",
|
|
||||||
"import os\n",
|
|
||||||
"import pickle\n",
|
|
||||||
"import azureml.train.automl\n",
|
|
||||||
"import azureml.explain.model\n",
|
|
||||||
"from azureml.train.automl.runtime.automl_explain_utilities import AutoMLExplainerSetupClass, \\\n",
|
|
||||||
" automl_setup_model_explanations\n",
|
|
||||||
"import joblib\n",
|
"import joblib\n",
|
||||||
|
"import pandas as pd\n",
|
||||||
"from azureml.core.model import Model\n",
|
"from azureml.core.model import Model\n",
|
||||||
|
"from azureml.train.automl.runtime.automl_explain_utilities import automl_setup_model_explanations\n",
|
||||||
"\n",
|
"\n",
|
||||||
"\n",
|
"\n",
|
||||||
"def init():\n",
|
"def init():\n",
|
||||||
|
|||||||
@@ -98,7 +98,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"print(\"This notebook was created using version 1.10.0 of the Azure ML SDK\")\n",
|
"print(\"This notebook was created using version 1.13.0 of the Azure ML SDK\")\n",
|
||||||
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
|
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -625,7 +625,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"from azureml.explain.model._internal.explanation_client import ExplanationClient\n",
|
"from azureml.interpret._internal.explanation_client import ExplanationClient\n",
|
||||||
"client = ExplanationClient.from_run(automl_run)\n",
|
"client = ExplanationClient.from_run(automl_run)\n",
|
||||||
"engineered_explanations = client.download_model_explanation(raw=False, comment='engineered explanations')\n",
|
"engineered_explanations = client.download_model_explanation(raw=False, comment='engineered explanations')\n",
|
||||||
"print(engineered_explanations.get_feature_importance_dict())\n",
|
"print(engineered_explanations.get_feature_importance_dict())\n",
|
||||||
@@ -659,7 +659,7 @@
|
|||||||
"In this section we will show how you can operationalize an AutoML model and the explainer which was used to compute the explanations in the previous section.\n",
|
"In this section we will show how you can operationalize an AutoML model and the explainer which was used to compute the explanations in the previous section.\n",
|
||||||
"\n",
|
"\n",
|
||||||
"### Register the AutoML model and the scoring explainer\n",
|
"### Register the AutoML model and the scoring explainer\n",
|
||||||
"We use the *TreeScoringExplainer* from *azureml.explain.model* package to create the scoring explainer which will be used to compute the raw and engineered feature importances at the inference time. \n",
|
"We use the *TreeScoringExplainer* from *azureml-interpret* package to create the scoring explainer which will be used to compute the raw and engineered feature importances at the inference time. \n",
|
||||||
"In the cell below, we register the AutoML model and the scoring explainer with the Model Management Service."
|
"In the cell below, we register the AutoML model and the scoring explainer with the Model Management Service."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -681,7 +681,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"### Create the conda dependencies for setting up the service\n",
|
"### Create the conda dependencies for setting up the service\n",
|
||||||
"We need to create the conda dependencies comprising of the *azureml-explain-model*, *azureml-train-automl* and *azureml-defaults* packages. "
|
"We need to create the conda dependencies comprising of the *azureml* packages using the training environment from the *automl_run*."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -1,14 +1,7 @@
|
|||||||
import json
|
|
||||||
import numpy as np
|
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import os
|
|
||||||
import pickle
|
|
||||||
import azureml.train.automl
|
|
||||||
import azureml.explain.model
|
|
||||||
from azureml.train.automl.runtime.automl_explain_utilities import AutoMLExplainerSetupClass, \
|
|
||||||
automl_setup_model_explanations
|
|
||||||
import joblib
|
import joblib
|
||||||
from azureml.core.model import Model
|
from azureml.core.model import Model
|
||||||
|
from azureml.train.automl.runtime.automl_explain_utilities import automl_setup_model_explanations
|
||||||
|
|
||||||
|
|
||||||
def init():
|
def init():
|
||||||
|
|||||||
@@ -1,17 +1,18 @@
|
|||||||
# Copyright (c) Microsoft. All rights reserved.
|
# Copyright (c) Microsoft. All rights reserved.
|
||||||
# Licensed under the MIT license.
|
# Licensed under the MIT license.
|
||||||
import os
|
import os
|
||||||
|
import joblib
|
||||||
|
|
||||||
from azureml.core.run import Run
|
from interpret.ext.glassbox import LGBMExplainableModel
|
||||||
|
from automl.client.core.common.constants import MODEL_PATH
|
||||||
from azureml.core.experiment import Experiment
|
from azureml.core.experiment import Experiment
|
||||||
from azureml.core.dataset import Dataset
|
from azureml.core.dataset import Dataset
|
||||||
from azureml.train.automl.runtime.automl_explain_utilities import AutoMLExplainerSetupClass, \
|
from azureml.core.run import Run
|
||||||
automl_setup_model_explanations, automl_check_model_if_explainable
|
from azureml.interpret.mimic_wrapper import MimicWrapper
|
||||||
from azureml.explain.model.mimic.models.lightgbm_model import LGBMExplainableModel
|
from azureml.interpret.scoring.scoring_explainer import TreeScoringExplainer
|
||||||
from azureml.explain.model.mimic_wrapper import MimicWrapper
|
from azureml.train.automl.runtime.automl_explain_utilities import automl_setup_model_explanations, \
|
||||||
from azureml.automl.core.shared.constants import MODEL_PATH
|
automl_check_model_if_explainable
|
||||||
from azureml.explain.model.scoring.scoring_explainer import TreeScoringExplainer
|
|
||||||
import joblib
|
|
||||||
|
|
||||||
OUTPUT_DIR = './outputs/'
|
OUTPUT_DIR = './outputs/'
|
||||||
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
||||||
|
|||||||
@@ -92,7 +92,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"print(\"This notebook was created using version 1.10.0 of the Azure ML SDK\")\n",
|
"print(\"This notebook was created using version 1.13.0 of the Azure ML SDK\")\n",
|
||||||
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
|
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -233,7 +233,7 @@
|
|||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"Call the `submit` method on the experiment object and pass the run configuration. Execution of remote runs is asynchronous. Depending on the data and the number of iterations this can run for a while."
|
"Call the `submit` method on the experiment object and pass the run configuration. Execution of remote runs is asynchronous. Depending on the data and the number of iterations this can run for a while. Validation errors and current status will be shown when setting `show_output=True` and the execution will be synchronous."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
56
how-to-use-azureml/azure-databricks/automl/README.md
Normal file
56
how-to-use-azureml/azure-databricks/automl/README.md
Normal file
@@ -0,0 +1,56 @@
|
|||||||
|
# Adding an init script to an Azure Databricks cluster
|
||||||
|
|
||||||
|
The [azureml-cluster-init.sh](./azureml-cluster-init.sh) script configures the environment to
|
||||||
|
1. Install the latest AutoML library
|
||||||
|
|
||||||
|
To create the Azure Databricks cluster-scoped init script
|
||||||
|
|
||||||
|
1. Create the base directory you want to store the init script in if it does not exist.
|
||||||
|
```
|
||||||
|
dbutils.fs.mkdirs("dbfs:/databricks/init/")
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Create the script azureml-cluster-init.sh
|
||||||
|
```
|
||||||
|
dbutils.fs.put("/databricks/init/azureml-cluster-init.sh","""
|
||||||
|
#!/bin/bash
|
||||||
|
set -ex
|
||||||
|
/databricks/python/bin/pip install -r https://aka.ms/automl_linux_requirements.txt
|
||||||
|
""", True)
|
||||||
|
```
|
||||||
|
|
||||||
|
3. Check that the script exists.
|
||||||
|
```
|
||||||
|
display(dbutils.fs.ls("dbfs:/databricks/init/azureml-cluster-init.sh"))
|
||||||
|
```
|
||||||
|
|
||||||
|
1. Configure the cluster to run the script.
|
||||||
|
* Using the cluster configuration page
|
||||||
|
1. On the cluster configuration page, click the Advanced Options toggle.
|
||||||
|
1. At the bottom of the page, click the Init Scripts tab.
|
||||||
|
1. In the Destination drop-down, select a destination type. Example: 'DBFS'
|
||||||
|
1. Specify a path to the init script.
|
||||||
|
```
|
||||||
|
dbfs:/databricks/init/azureml-cluster-init.sh
|
||||||
|
```
|
||||||
|
1. Click Add
|
||||||
|
|
||||||
|
* Using the API.
|
||||||
|
```
|
||||||
|
curl -n -X POST -H 'Content-Type: application/json' -d '{
|
||||||
|
"cluster_id": "<cluster_id>",
|
||||||
|
"num_workers": <num_workers>,
|
||||||
|
"spark_version": "<spark_version>",
|
||||||
|
"node_type_id": "<node_type_id>",
|
||||||
|
"cluster_log_conf": {
|
||||||
|
"dbfs" : {
|
||||||
|
"destination": "dbfs:/cluster-logs"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"init_scripts": [ {
|
||||||
|
"dbfs": {
|
||||||
|
"destination": "dbfs:/databricks/init/azureml-cluster-init.sh"
|
||||||
|
}
|
||||||
|
} ]
|
||||||
|
}' https://<databricks-instance>/api/2.0/clusters/edit
|
||||||
|
```
|
||||||
@@ -13,32 +13,46 @@
|
|||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"# Automated ML on Azure Databricks\n",
|
"## AutoML Installation\n",
|
||||||
"\n",
|
"\n",
|
||||||
"In this example we use the scikit-learn's <a href=\"http://scikit-learn.org/stable/datasets/index.html#optical-recognition-of-handwritten-digits-dataset\" target=\"_blank\">digit dataset</a> to showcase how you can use AutoML for a simple classification problem.\n",
|
"**For Databricks non ML runtime 7.1(scala 2.21, spark 3.0.0) and up, Install AML sdk by running the following command in the first cell of the notebook.**\n",
|
||||||
"\n",
|
"\n",
|
||||||
"In this notebook you will learn how to:\n",
|
"%pip install -r https://aka.ms/automl_linux_requirements.txt\n",
|
||||||
"1. Create Azure Machine Learning Workspace object and initialize your notebook directory to easily reload this object from a configuration file.\n",
|
|
||||||
"2. Create an `Experiment` in an existing `Workspace`.\n",
|
|
||||||
"3. Configure Automated ML using `AutoMLConfig`.\n",
|
|
||||||
"4. Train the model using Azure Databricks.\n",
|
|
||||||
"5. Explore the results.\n",
|
|
||||||
"6. Viewing the engineered names for featurized data and featurization summary for all raw features.\n",
|
|
||||||
"7. Test the best fitted model.\n",
|
|
||||||
"\n",
|
"\n",
|
||||||
"Before running this notebook, please follow the <a href=\"https://github.com/Azure/MachineLearningNotebooks/tree/master/how-to-use-azureml/azure-databricks\" target=\"_blank\">readme for using Automated ML on Azure Databricks</a> for installing necessary libraries to your cluster."
|
"**For Databricks non ML runtime 7.0 and lower, Install AML sdk using init script as shown in [readme](readme.md) before running this notebook.**\n"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"We support installing AML SDK with Automated ML as library from GUI. When attaching a library follow <a href=\"https://docs.databricks.com/user-guide/libraries.html\" target=\"_blank\">this link</a> and add the below string as your PyPi package. You can select the option to attach the library to all clusters or just one cluster.\n",
|
"# AutoML : Classification with Local Compute on Azure DataBricks\n",
|
||||||
"\n",
|
"\n",
|
||||||
"**azureml-sdk with automated ml**\n",
|
"In this example we use the scikit-learn's to showcase how you can use AutoML for a simple classification problem.\n",
|
||||||
"* Source: Upload Python Egg or PyPi\n",
|
"\n",
|
||||||
"* PyPi Name: `azureml-sdk[automl]`\n",
|
"In this notebook you will learn how to:\n",
|
||||||
"* Select Install Library"
|
"1. Create Azure Machine Learning Workspace object and initialize your notebook directory to easily reload this object from a configuration file.\n",
|
||||||
|
"2. Create an `Experiment` in an existing `Workspace`.\n",
|
||||||
|
"3. Configure AutoML using `AutoMLConfig`.\n",
|
||||||
|
"4. Train the model using AzureDataBricks.\n",
|
||||||
|
"5. Explore the results.\n",
|
||||||
|
"6. Test the best fitted model.\n",
|
||||||
|
"\n",
|
||||||
|
"Prerequisites:\n",
|
||||||
|
"Before running this notebook, please follow the readme for installing necessary libraries to your cluster."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Register Machine Learning Services Resource Provider\n",
|
||||||
|
"Microsoft.MachineLearningServices only needs to be registed once in the subscription. To register it:\n",
|
||||||
|
"Start the Azure portal.\n",
|
||||||
|
"Select your All services and then Subscription.\n",
|
||||||
|
"Select the subscription that you want to use.\n",
|
||||||
|
"Click on Resource providers\n",
|
||||||
|
"Click the Register link next to Microsoft.MachineLearningServices"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -145,31 +159,8 @@
|
|||||||
" resource_group = resource_group)\n",
|
" resource_group = resource_group)\n",
|
||||||
"\n",
|
"\n",
|
||||||
"# Persist the subscription id, resource group name, and workspace name in aml_config/config.json.\n",
|
"# Persist the subscription id, resource group name, and workspace name in aml_config/config.json.\n",
|
||||||
"ws.write_config()"
|
"ws.write_config()\n",
|
||||||
]
|
"write_config(path=\"/databricks/driver/aml_config/\",file_name=<alias_conf.cfg>)"
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Create a Folder to Host Sample Projects\n",
|
|
||||||
"Finally, create a folder where all the sample projects will be hosted."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"import os\n",
|
|
||||||
"\n",
|
|
||||||
"sample_projects_folder = './sample_projects'\n",
|
|
||||||
"\n",
|
|
||||||
"if not os.path.isdir(sample_projects_folder):\n",
|
|
||||||
" os.mkdir(sample_projects_folder)\n",
|
|
||||||
" \n",
|
|
||||||
"print('Sample projects will be created in {}.'.format(sample_projects_folder))"
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -178,7 +169,7 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"## Create an Experiment\n",
|
"## Create an Experiment\n",
|
||||||
"\n",
|
"\n",
|
||||||
"As part of the setup you have already created an Azure ML `Workspace` object. For Automated ML you will need to create an `Experiment` object, which is a named object in a `Workspace` used to run experiments."
|
"As part of the setup you have already created an Azure ML `Workspace` object. For AutoML you will need to create an `Experiment` object, which is a named object in a `Workspace` used to run experiments."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -191,6 +182,7 @@
|
|||||||
"import os\n",
|
"import os\n",
|
||||||
"import random\n",
|
"import random\n",
|
||||||
"import time\n",
|
"import time\n",
|
||||||
|
"import json\n",
|
||||||
"\n",
|
"\n",
|
||||||
"from matplotlib import pyplot as plt\n",
|
"from matplotlib import pyplot as plt\n",
|
||||||
"from matplotlib.pyplot import imshow\n",
|
"from matplotlib.pyplot import imshow\n",
|
||||||
@@ -212,7 +204,6 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"# Choose a name for the experiment and specify the project folder.\n",
|
"# Choose a name for the experiment and specify the project folder.\n",
|
||||||
"experiment_name = 'automl-local-classification'\n",
|
"experiment_name = 'automl-local-classification'\n",
|
||||||
"project_folder = './sample_projects/automl-local-classification'\n",
|
|
||||||
"\n",
|
"\n",
|
||||||
"experiment = Experiment(ws, experiment_name)\n",
|
"experiment = Experiment(ws, experiment_name)\n",
|
||||||
"\n",
|
"\n",
|
||||||
@@ -222,94 +213,11 @@
|
|||||||
"output['Workspace Name'] = ws.name\n",
|
"output['Workspace Name'] = ws.name\n",
|
||||||
"output['Resource Group'] = ws.resource_group\n",
|
"output['Resource Group'] = ws.resource_group\n",
|
||||||
"output['Location'] = ws.location\n",
|
"output['Location'] = ws.location\n",
|
||||||
"output['Project Directory'] = project_folder\n",
|
|
||||||
"output['Experiment Name'] = experiment.name\n",
|
"output['Experiment Name'] = experiment.name\n",
|
||||||
"pd.set_option('display.max_colwidth', -1)\n",
|
"pd.set_option('display.max_colwidth', -1)\n",
|
||||||
"pd.DataFrame(data = output, index = ['']).T"
|
"pd.DataFrame(data = output, index = ['']).T"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Diagnostics\n",
|
|
||||||
"\n",
|
|
||||||
"Opt-in diagnostics for better experience, quality, and security of future releases."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"from azureml.telemetry import set_diagnostics_collection\n",
|
|
||||||
"set_diagnostics_collection(send_diagnostics = True)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Registering Datastore"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"Datastore is the way to save connection information to a storage service (e.g. Azure Blob, Azure Data Lake, Azure SQL) information to your workspace so you can access them without exposing credentials in your code. The first thing you will need to do is register a datastore, you can refer to our [python SDK documentation](https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.core.datastore.datastore?view=azure-ml-py) on how to register datastores. __Note: for best security practices, please do not check in code that contains registering datastores with secrets into your source control__\n",
|
|
||||||
"\n",
|
|
||||||
"The code below registers a datastore pointing to a publicly readable blob container."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"from azureml.core import Datastore\n",
|
|
||||||
"\n",
|
|
||||||
"datastore_name = 'demo_training'\n",
|
|
||||||
"container_name = 'digits' \n",
|
|
||||||
"account_name = 'automlpublicdatasets'\n",
|
|
||||||
"Datastore.register_azure_blob_container(\n",
|
|
||||||
" workspace = ws, \n",
|
|
||||||
" datastore_name = datastore_name, \n",
|
|
||||||
" container_name = container_name, \n",
|
|
||||||
" account_name = account_name,\n",
|
|
||||||
" overwrite = True\n",
|
|
||||||
")"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"Below is an example on how to register a private blob container\n",
|
|
||||||
"```python\n",
|
|
||||||
"datastore = Datastore.register_azure_blob_container(\n",
|
|
||||||
" workspace = ws, \n",
|
|
||||||
" datastore_name = 'example_datastore', \n",
|
|
||||||
" container_name = 'example-container', \n",
|
|
||||||
" account_name = 'storageaccount',\n",
|
|
||||||
" account_key = 'accountkey'\n",
|
|
||||||
")\n",
|
|
||||||
"```\n",
|
|
||||||
"The example below shows how to register an Azure Data Lake store. Please make sure you have granted the necessary permissions for the service principal to access the data lake.\n",
|
|
||||||
"```python\n",
|
|
||||||
"datastore = Datastore.register_azure_data_lake(\n",
|
|
||||||
" workspace = ws,\n",
|
|
||||||
" datastore_name = 'example_datastore',\n",
|
|
||||||
" store_name = 'adlsstore',\n",
|
|
||||||
" tenant_id = 'tenant-id-of-service-principal',\n",
|
|
||||||
" client_id = 'client-id-of-service-principal',\n",
|
|
||||||
" client_secret = 'client-secret-of-service-principal'\n",
|
|
||||||
")\n",
|
|
||||||
"```"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
@@ -323,9 +231,7 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"Automated ML takes a `TabularDataset` as input.\n",
|
"Automated ML takes a `TabularDataset` as input.\n",
|
||||||
"\n",
|
"\n",
|
||||||
"You are free to use the data preparation libraries/tools of your choice to do the require preparation and once you are done, you can write it to a datastore and create a TabularDataset from it.\n",
|
"You are free to use the data preparation libraries/tools of your choice to do the require preparation and once you are done, you can write it to a datastore and create a TabularDataset from it."
|
||||||
"\n",
|
|
||||||
"You will get the datastore you registered previously and pass it to Dataset for reading. The data comes from the digits dataset: `sklearn.datasets.load_digits()`. `DataPath` points to a specific location within a datastore. "
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -334,13 +240,12 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
|
"# The data referenced here was a 1MB simple random sample of the Chicago Crime data into a local temporary directory.\n",
|
||||||
"from azureml.core.dataset import Dataset\n",
|
"from azureml.core.dataset import Dataset\n",
|
||||||
"from azureml.data.datapath import DataPath\n",
|
|
||||||
"\n",
|
"\n",
|
||||||
"datastore = Datastore.get(workspace = ws, datastore_name = datastore_name)\n",
|
"example_data = 'https://dprepdata.blob.core.windows.net/demo/crime0-random.csv'\n",
|
||||||
"\n",
|
"dataset = Dataset.Tabular.from_delimited_files(example_data)\n",
|
||||||
"X_train = Dataset.Tabular.from_delimited_files(datastore.path('X.csv'))\n",
|
"dataset.take(5).to_pandas_dataframe()"
|
||||||
"y_train = Dataset.Tabular.from_delimited_files(datastore.path('y.csv'))"
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -357,16 +262,8 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"X_train.take(5).to_pandas_dataframe()"
|
"training_data = dataset.drop_columns(columns=['FBI Code'])\n",
|
||||||
]
|
"label = 'Primary Type'"
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"y_train.take(5).to_pandas_dataframe()"
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -384,14 +281,11 @@
|
|||||||
"|**primary_metric**|This is the metric that you want to optimize. Regression supports the following primary metrics: <br><i>spearman_correlation</i><br><i>normalized_root_mean_squared_error</i><br><i>r2_score</i><br><i>normalized_mean_absolute_error</i>|\n",
|
"|**primary_metric**|This is the metric that you want to optimize. Regression supports the following primary metrics: <br><i>spearman_correlation</i><br><i>normalized_root_mean_squared_error</i><br><i>r2_score</i><br><i>normalized_mean_absolute_error</i>|\n",
|
||||||
"|**iteration_timeout_minutes**|Time limit in minutes for each iteration.|\n",
|
"|**iteration_timeout_minutes**|Time limit in minutes for each iteration.|\n",
|
||||||
"|**iterations**|Number of iterations. In each iteration AutoML trains a specific pipeline with the data.|\n",
|
"|**iterations**|Number of iterations. In each iteration AutoML trains a specific pipeline with the data.|\n",
|
||||||
"|**n_cross_validations**|Number of cross validation splits.|\n",
|
|
||||||
"|**spark_context**|Spark Context object. for Databricks, use spark_context=sc|\n",
|
"|**spark_context**|Spark Context object. for Databricks, use spark_context=sc|\n",
|
||||||
"|**max_concurrent_iterations**|Maximum number of iterations to execute in parallel. This should be <= number of worker nodes in your Azure Databricks cluster.|\n",
|
"|**max_concurrent_iterations**|Maximum number of iterations to execute in parallel. This should be <= number of worker nodes in your Azure Databricks cluster.|\n",
|
||||||
"|**X**|(sparse) array-like, shape = [n_samples, n_features]|\n",
|
"|**n_cross_validations**|Number of cross validation splits.|\n",
|
||||||
"|**y**|(sparse) array-like, shape = [n_samples, ], [n_samples, n_classes]<br>Multi-class targets. An indicator matrix turns on multilabel classification. This should be an array of integers.|\n",
|
"|**training_data**|Input dataset, containing both features and label column.|\n",
|
||||||
"|**path**|Relative path to the project folder. AutoML stores configuration files for the experiment under this folder. You can specify a new empty folder.|\n",
|
"|**label_column_name**|The name of the label column.|"
|
||||||
"|**preprocess**|set this to True to enable pre-processing of data eg. string to numeric using one-hot encoding|\n",
|
|
||||||
"|**exit_score**|Target score for experiment. It is associated with the metric. eg. exit_score=0.995 will exit experiment after that|"
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -404,15 +298,13 @@
|
|||||||
" debug_log = 'automl_errors.log',\n",
|
" debug_log = 'automl_errors.log',\n",
|
||||||
" primary_metric = 'AUC_weighted',\n",
|
" primary_metric = 'AUC_weighted',\n",
|
||||||
" iteration_timeout_minutes = 10,\n",
|
" iteration_timeout_minutes = 10,\n",
|
||||||
" iterations = 3,\n",
|
" iterations = 5,\n",
|
||||||
" preprocess = True,\n",
|
|
||||||
" n_cross_validations = 10,\n",
|
" n_cross_validations = 10,\n",
|
||||||
" max_concurrent_iterations = 2, #change it based on number of worker nodes\n",
|
" max_concurrent_iterations = 2, #change it based on number of worker nodes\n",
|
||||||
" verbosity = logging.INFO,\n",
|
" verbosity = logging.INFO,\n",
|
||||||
" spark_context=sc, #databricks/spark related\n",
|
" spark_context=sc, #databricks/spark related\n",
|
||||||
" X = X_train, \n",
|
" training_data=training_data,\n",
|
||||||
" y = y_train,\n",
|
" label_column_name=label)"
|
||||||
" path = project_folder)"
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -433,26 +325,6 @@
|
|||||||
"local_run = experiment.submit(automl_config, show_output = True)"
|
"local_run = experiment.submit(automl_config, show_output = True)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Continue experiment"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"local_run.continue_experiment(iterations=2,\n",
|
|
||||||
" X=X_train, \n",
|
|
||||||
" y=y_train,\n",
|
|
||||||
" spark_context=sc,\n",
|
|
||||||
" show_output=True)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
@@ -475,14 +347,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"displayHTML(\"<a href={} target='_blank'>Your experiment in Azure Portal: {}</a>\".format(local_run.get_portal_url(), local_run.id))"
|
"displayHTML(\"<a href={} target='_blank'>Azure Portal: {}</a>\".format(local_run.get_portal_url(), local_run.id))"
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"The following will show the child runs and waits for the parent run to complete."
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -503,6 +368,7 @@
|
|||||||
"metricslist = {}\n",
|
"metricslist = {}\n",
|
||||||
"for run in children:\n",
|
"for run in children:\n",
|
||||||
" properties = run.get_properties()\n",
|
" properties = run.get_properties()\n",
|
||||||
|
" #print(properties)\n",
|
||||||
" metrics = {k: v for k, v in run.get_metrics().items() if isinstance(v, float)} \n",
|
" metrics = {k: v for k, v in run.get_metrics().items() if isinstance(v, float)} \n",
|
||||||
" metricslist[int(properties['iteration'])] = metrics\n",
|
" metricslist[int(properties['iteration'])] = metrics\n",
|
||||||
"\n",
|
"\n",
|
||||||
@@ -514,9 +380,11 @@
|
|||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"### Retrieve the Best Model after the above run is complete \n",
|
"## Deploy\n",
|
||||||
"\n",
|
"\n",
|
||||||
"Below we select the best pipeline from our iterations. The `get_output` method returns the best run and the fitted model. The Model includes the pipeline and any pre-processing. Overloads on `get_output` allow you to retrieve the best run and fitted model for *any* logged metric or for a particular *iteration*."
|
"### Retrieve the Best Model\n",
|
||||||
|
"\n",
|
||||||
|
"Below we select the best pipeline from our iterations. The `get_output` method on `automl_classifier` returns the best run and the fitted model for the last invocation. Overloads on `get_output` allow you to retrieve the best run and fitted model for *any* logged metric or for a particular *iteration*."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -525,71 +393,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"best_run, fitted_model = local_run.get_output()\n",
|
"best_run, fitted_model = local_run.get_output()"
|
||||||
"print(best_run)\n",
|
|
||||||
"print(fitted_model)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"#### Best Model Based on Any Other Metric after the above run is complete based on the child run\n",
|
|
||||||
"Show the run and the model that has the smallest `log_loss` value:"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"lookup_metric = \"log_loss\"\n",
|
|
||||||
"best_run, fitted_model = local_run.get_output(metric = lookup_metric)\n",
|
|
||||||
"print(best_run)\n",
|
|
||||||
"print(fitted_model)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"#### View the engineered names for featurized data\n",
|
|
||||||
"Below we display the engineered feature names generated for the featurized data using the preprocessing featurization."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"fitted_model.named_steps['datatransformer'].get_engineered_feature_names()"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"#### View the featurization summary\n",
|
|
||||||
"Below we display the featurization that was performed on different raw features in the user data. For each raw feature in the user data, the following information is displayed:-\n",
|
|
||||||
"- Raw feature name\n",
|
|
||||||
"- Number of engineered features formed out of this raw feature\n",
|
|
||||||
"- Type detected\n",
|
|
||||||
"- If feature was dropped\n",
|
|
||||||
"- List of feature transformations for the raw feature"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# Get the featurization summary as a list of JSON\n",
|
|
||||||
"featurization_summary = fitted_model.named_steps['datatransformer'].get_featurization_summary()\n",
|
|
||||||
"# View the featurization summary as a pandas dataframe\n",
|
|
||||||
"pd.DataFrame.from_records(featurization_summary)"
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -607,11 +411,13 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"blob_location = \"https://{}.blob.core.windows.net/{}\".format(account_name, container_name)\n",
|
"dataset_test = Dataset.Tabular.from_delimited_files(path='https://dprepdata.blob.core.windows.net/demo/crime0-test.csv')\n",
|
||||||
"X_test = pd.read_csv(\"{}./X_valid.csv\".format(blob_location), header=0)\n",
|
"\n",
|
||||||
"y_test = pd.read_csv(\"{}/y_valid.csv\".format(blob_location), header=0)\n",
|
"df_test = dataset_test.to_pandas_dataframe()\n",
|
||||||
"images = pd.read_csv(\"{}/images.csv\".format(blob_location), header=None)\n",
|
"df_test = df_test[pd.notnull(df_test['Primary Type'])]\n",
|
||||||
"images = np.reshape(images.values, (100,8,8))"
|
"\n",
|
||||||
|
"y_test = df_test[['Primary Type']]\n",
|
||||||
|
"X_test = df_test.drop(['Primary Type', 'FBI Code'], axis=1)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -628,35 +434,9 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"# Randomly select digits and test.\n",
|
"fitted_model.predict(X_test)"
|
||||||
"for index in np.random.choice(len(y_test), 2, replace = False):\n",
|
|
||||||
" print(index)\n",
|
|
||||||
" predicted = fitted_model.predict(X_test[index:index + 1])[0]\n",
|
|
||||||
" label = y_test.values[index]\n",
|
|
||||||
" title = \"Label value = %d Predicted value = %d \" % (label, predicted)\n",
|
|
||||||
" fig = plt.figure(3, figsize = (5,5))\n",
|
|
||||||
" ax1 = fig.add_axes((0,0,.8,.8))\n",
|
|
||||||
" ax1.set_title(title)\n",
|
|
||||||
" plt.imshow(images[index], cmap = plt.cm.gray_r, interpolation = 'nearest')\n",
|
|
||||||
" display(fig)"
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"When deploying an automated ML trained model, please specify _pippackages=['azureml-sdk[automl]']_ in your CondaDependencies.\n",
|
|
||||||
"\n",
|
|
||||||
"Please refer to only the **Deploy** section in this notebook - <a href=\"https://github.com/Azure/MachineLearningNotebooks/blob/master/how-to-use-azureml/automated-machine-learning/classification-with-deployment\" target=\"_blank\">Deployment of Automated ML trained model</a>"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": []
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
@@ -689,10 +469,10 @@
|
|||||||
"name": "python",
|
"name": "python",
|
||||||
"nbconvert_exporter": "python",
|
"nbconvert_exporter": "python",
|
||||||
"pygments_lexer": "ipython3",
|
"pygments_lexer": "ipython3",
|
||||||
"version": "3.6.5"
|
"version": "3.6.8"
|
||||||
},
|
},
|
||||||
"name": "auto-ml-classification-local-adb",
|
"name": "auto-ml-classification-local-adb",
|
||||||
"notebookId": 587284549713154
|
"notebookId": 1275190406842063
|
||||||
},
|
},
|
||||||
"nbformat": 4,
|
"nbformat": 4,
|
||||||
"nbformat_minor": 1
|
"nbformat_minor": 1
|
||||||
|
|||||||
@@ -13,12 +13,13 @@
|
|||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"We support installing AML SDK as library from GUI. When attaching a library follow this https://docs.databricks.com/user-guide/libraries.html and add the below string as your PyPi package. You can select the option to attach the library to all clusters or just one cluster.\n",
|
"## AutoML Installation\n",
|
||||||
"\n",
|
"\n",
|
||||||
"**install azureml-sdk with Automated ML**\n",
|
"**For Databricks non ML runtime 7.1(scala 2.21, spark 3.0.0) and up, Install AML sdk by running the following command in the first cell of the notebook.**\n",
|
||||||
"* Source: Upload Python Egg or PyPi\n",
|
"\n",
|
||||||
"* PyPi Name: `azureml-sdk[automl]`\n",
|
"%pip install -r https://aka.ms/automl_linux_requirements.txt\n",
|
||||||
"* Select Install Library"
|
"\n",
|
||||||
|
"**For Databricks non ML runtime 7.0 and lower, Install AML sdk using init script as shown in [readme](readme.md) before running this notebook.**"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -27,7 +28,7 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"# AutoML : Classification with Local Compute on Azure DataBricks with deployment to ACI\n",
|
"# AutoML : Classification with Local Compute on Azure DataBricks with deployment to ACI\n",
|
||||||
"\n",
|
"\n",
|
||||||
"In this example we use the scikit-learn's [digit dataset](http://scikit-learn.org/stable/datasets/index.html#optical-recognition-of-handwritten-digits-dataset) to showcase how you can use AutoML for a simple classification problem.\n",
|
"In this example we use the scikit-learn's to showcase how you can use AutoML for a simple classification problem.\n",
|
||||||
"\n",
|
"\n",
|
||||||
"In this notebook you will learn how to:\n",
|
"In this notebook you will learn how to:\n",
|
||||||
"1. Create Azure Machine Learning Workspace object and initialize your notebook directory to easily reload this object from a configuration file.\n",
|
"1. Create Azure Machine Learning Workspace object and initialize your notebook directory to easily reload this object from a configuration file.\n",
|
||||||
@@ -164,30 +165,6 @@
|
|||||||
"write_config(path=\"/databricks/driver/aml_config/\",file_name=<alias_conf.cfg>)"
|
"write_config(path=\"/databricks/driver/aml_config/\",file_name=<alias_conf.cfg>)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Create a Folder to Host Sample Projects\n",
|
|
||||||
"Finally, create a folder where all the sample projects will be hosted."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"import os\n",
|
|
||||||
"\n",
|
|
||||||
"sample_projects_folder = './sample_projects'\n",
|
|
||||||
"\n",
|
|
||||||
"if not os.path.isdir(sample_projects_folder):\n",
|
|
||||||
" os.mkdir(sample_projects_folder)\n",
|
|
||||||
" \n",
|
|
||||||
"print('Sample projects will be created in {}.'.format(sample_projects_folder))"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
@@ -229,7 +206,6 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"# Choose a name for the experiment and specify the project folder.\n",
|
"# Choose a name for the experiment and specify the project folder.\n",
|
||||||
"experiment_name = 'automl-local-classification'\n",
|
"experiment_name = 'automl-local-classification'\n",
|
||||||
"project_folder = './sample_projects/automl-local-classification'\n",
|
|
||||||
"\n",
|
"\n",
|
||||||
"experiment = Experiment(ws, experiment_name)\n",
|
"experiment = Experiment(ws, experiment_name)\n",
|
||||||
"\n",
|
"\n",
|
||||||
@@ -239,94 +215,11 @@
|
|||||||
"output['Workspace Name'] = ws.name\n",
|
"output['Workspace Name'] = ws.name\n",
|
||||||
"output['Resource Group'] = ws.resource_group\n",
|
"output['Resource Group'] = ws.resource_group\n",
|
||||||
"output['Location'] = ws.location\n",
|
"output['Location'] = ws.location\n",
|
||||||
"output['Project Directory'] = project_folder\n",
|
|
||||||
"output['Experiment Name'] = experiment.name\n",
|
"output['Experiment Name'] = experiment.name\n",
|
||||||
"pd.set_option('display.max_colwidth', -1)\n",
|
"pd.set_option('display.max_colwidth', -1)\n",
|
||||||
"pd.DataFrame(data = output, index = ['']).T"
|
"pd.DataFrame(data = output, index = ['']).T"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Diagnostics\n",
|
|
||||||
"\n",
|
|
||||||
"Opt-in diagnostics for better experience, quality, and security of future releases."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"from azureml.telemetry import set_diagnostics_collection\n",
|
|
||||||
"set_diagnostics_collection(send_diagnostics = True)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Registering Datastore"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"Datastore is the way to save connection information to a storage service (e.g. Azure Blob, Azure Data Lake, Azure SQL) information to your workspace so you can access them without exposing credentials in your code. The first thing you will need to do is register a datastore, you can refer to our [python SDK documentation](https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.core.datastore.datastore?view=azure-ml-py) on how to register datastores. __Note: for best security practices, please do not check in code that contains registering datastores with secrets into your source control__\n",
|
|
||||||
"\n",
|
|
||||||
"The code below registers a datastore pointing to a publicly readable blob container."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"from azureml.core import Datastore\n",
|
|
||||||
"\n",
|
|
||||||
"datastore_name = 'demo_training'\n",
|
|
||||||
"container_name = 'digits' \n",
|
|
||||||
"account_name = 'automlpublicdatasets'\n",
|
|
||||||
"Datastore.register_azure_blob_container(\n",
|
|
||||||
" workspace = ws, \n",
|
|
||||||
" datastore_name = datastore_name, \n",
|
|
||||||
" container_name = container_name, \n",
|
|
||||||
" account_name = account_name,\n",
|
|
||||||
" overwrite = True\n",
|
|
||||||
")"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"Below is an example on how to register a private blob container\n",
|
|
||||||
"```python\n",
|
|
||||||
"datastore = Datastore.register_azure_blob_container(\n",
|
|
||||||
" workspace = ws, \n",
|
|
||||||
" datastore_name = 'example_datastore', \n",
|
|
||||||
" container_name = 'example-container', \n",
|
|
||||||
" account_name = 'storageaccount',\n",
|
|
||||||
" account_key = 'accountkey'\n",
|
|
||||||
")\n",
|
|
||||||
"```\n",
|
|
||||||
"The example below shows how to register an Azure Data Lake store. Please make sure you have granted the necessary permissions for the service principal to access the data lake.\n",
|
|
||||||
"```python\n",
|
|
||||||
"datastore = Datastore.register_azure_data_lake(\n",
|
|
||||||
" workspace = ws,\n",
|
|
||||||
" datastore_name = 'example_datastore',\n",
|
|
||||||
" store_name = 'adlsstore',\n",
|
|
||||||
" tenant_id = 'tenant-id-of-service-principal',\n",
|
|
||||||
" client_id = 'client-id-of-service-principal',\n",
|
|
||||||
" client_secret = 'client-secret-of-service-principal'\n",
|
|
||||||
")\n",
|
|
||||||
"```"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
@@ -340,9 +233,7 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"Automated ML takes a `TabularDataset` as input.\n",
|
"Automated ML takes a `TabularDataset` as input.\n",
|
||||||
"\n",
|
"\n",
|
||||||
"You are free to use the data preparation libraries/tools of your choice to do the require preparation and once you are done, you can write it to a datastore and create a TabularDataset from it.\n",
|
"You are free to use the data preparation libraries/tools of your choice to do the require preparation and once you are done, you can write it to a datastore and create a TabularDataset from it."
|
||||||
"\n",
|
|
||||||
"You will get the datastore you registered previously and pass it to Dataset for reading. The data comes from the digits dataset: `sklearn.datasets.load_digits()`. `DataPath` points to a specific location within a datastore. "
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -351,13 +242,12 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
|
"# The data referenced here was a 1MB simple random sample of the Chicago Crime data into a local temporary directory.\n",
|
||||||
"from azureml.core.dataset import Dataset\n",
|
"from azureml.core.dataset import Dataset\n",
|
||||||
"from azureml.data.datapath import DataPath\n",
|
|
||||||
"\n",
|
"\n",
|
||||||
"datastore = Datastore.get(workspace = ws, datastore_name = datastore_name)\n",
|
"example_data = 'https://dprepdata.blob.core.windows.net/demo/crime0-random.csv'\n",
|
||||||
"\n",
|
"dataset = Dataset.Tabular.from_delimited_files(example_data)\n",
|
||||||
"X_train = Dataset.Tabular.from_delimited_files(datastore.path('X.csv'))\n",
|
"dataset.take(5).to_pandas_dataframe()"
|
||||||
"y_train = Dataset.Tabular.from_delimited_files(datastore.path('y.csv'))"
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -374,16 +264,8 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"X_train.take(5).to_pandas_dataframe()"
|
"training_data = dataset.drop_columns(columns=['FBI Code'])\n",
|
||||||
]
|
"label = 'Primary Type'"
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"y_train.take(5).to_pandas_dataframe()"
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -401,14 +283,11 @@
|
|||||||
"|**primary_metric**|This is the metric that you want to optimize. Regression supports the following primary metrics: <br><i>spearman_correlation</i><br><i>normalized_root_mean_squared_error</i><br><i>r2_score</i><br><i>normalized_mean_absolute_error</i>|\n",
|
"|**primary_metric**|This is the metric that you want to optimize. Regression supports the following primary metrics: <br><i>spearman_correlation</i><br><i>normalized_root_mean_squared_error</i><br><i>r2_score</i><br><i>normalized_mean_absolute_error</i>|\n",
|
||||||
"|**iteration_timeout_minutes**|Time limit in minutes for each iteration.|\n",
|
"|**iteration_timeout_minutes**|Time limit in minutes for each iteration.|\n",
|
||||||
"|**iterations**|Number of iterations. In each iteration AutoML trains a specific pipeline with the data.|\n",
|
"|**iterations**|Number of iterations. In each iteration AutoML trains a specific pipeline with the data.|\n",
|
||||||
"|**n_cross_validations**|Number of cross validation splits.|\n",
|
|
||||||
"|**spark_context**|Spark Context object. for Databricks, use spark_context=sc|\n",
|
"|**spark_context**|Spark Context object. for Databricks, use spark_context=sc|\n",
|
||||||
"|**max_concurrent_iterations**|Maximum number of iterations to execute in parallel. This should be <= number of worker nodes in your Azure Databricks cluster.|\n",
|
"|**max_concurrent_iterations**|Maximum number of iterations to execute in parallel. This should be <= number of worker nodes in your Azure Databricks cluster.|\n",
|
||||||
"|**X**|(sparse) array-like, shape = [n_samples, n_features]|\n",
|
"|**n_cross_validations**|Number of cross validation splits.|\n",
|
||||||
"|**y**|(sparse) array-like, shape = [n_samples, ], [n_samples, n_classes]<br>Multi-class targets. An indicator matrix turns on multilabel classification. This should be an array of integers.|\n",
|
"|**training_data**|Input dataset, containing both features and label column.|\n",
|
||||||
"|**path**|Relative path to the project folder. AutoML stores configuration files for the experiment under this folder. You can specify a new empty folder.|\n",
|
"|**label_column_name**|The name of the label column.|"
|
||||||
"|**preprocess**|set this to True to enable pre-processing of data eg. string to numeric using one-hot encoding|\n",
|
|
||||||
"|**exit_score**|Target score for experiment. It is associated with the metric. eg. exit_score=0.995 will exit experiment after that|"
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -422,14 +301,12 @@
|
|||||||
" primary_metric = 'AUC_weighted',\n",
|
" primary_metric = 'AUC_weighted',\n",
|
||||||
" iteration_timeout_minutes = 10,\n",
|
" iteration_timeout_minutes = 10,\n",
|
||||||
" iterations = 5,\n",
|
" iterations = 5,\n",
|
||||||
" preprocess = True,\n",
|
|
||||||
" n_cross_validations = 10,\n",
|
" n_cross_validations = 10,\n",
|
||||||
" max_concurrent_iterations = 2, #change it based on number of worker nodes\n",
|
" max_concurrent_iterations = 2, #change it based on number of worker nodes\n",
|
||||||
" verbosity = logging.INFO,\n",
|
" verbosity = logging.INFO,\n",
|
||||||
" spark_context=sc, #databricks/spark related\n",
|
" spark_context=sc, #databricks/spark related\n",
|
||||||
" X = X_train, \n",
|
" training_data=training_data,\n",
|
||||||
" y = y_train,\n",
|
" label_column_name=label)"
|
||||||
" path = project_folder)"
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -475,13 +352,6 @@
|
|||||||
"displayHTML(\"<a href={} target='_blank'>Azure Portal: {}</a>\".format(local_run.get_portal_url(), local_run.id))"
|
"displayHTML(\"<a href={} target='_blank'>Azure Portal: {}</a>\".format(local_run.get_portal_url(), local_run.id))"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"The following will show the child runs and waits for the parent run to complete."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
@@ -651,11 +521,13 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"blob_location = \"https://{}.blob.core.windows.net/{}\".format(account_name, container_name)\n",
|
"dataset_test = Dataset.Tabular.from_delimited_files(path='https://dprepdata.blob.core.windows.net/demo/crime0-test.csv')\n",
|
||||||
"X_test = pd.read_csv(\"{}./X_valid.csv\".format(blob_location), header=0)\n",
|
"\n",
|
||||||
"y_test = pd.read_csv(\"{}/y_valid.csv\".format(blob_location), header=0)\n",
|
"df_test = dataset_test.to_pandas_dataframe()\n",
|
||||||
"images = pd.read_csv(\"{}/images.csv\".format(blob_location), header=None)\n",
|
"df_test = df_test[pd.notnull(df_test['Primary Type'])]\n",
|
||||||
"images = np.reshape(images.values, (100,8,8))"
|
"\n",
|
||||||
|
"y_test = df_test[['Primary Type']]\n",
|
||||||
|
"X_test = df_test.drop(['Primary Type', 'FBI Code'], axis=1)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -672,20 +544,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"import json\n",
|
"fitted_model.predict(X_test)"
|
||||||
"# Randomly select digits and test.\n",
|
|
||||||
"for index in np.random.choice(len(y_test), 2, replace = False):\n",
|
|
||||||
" print(index)\n",
|
|
||||||
" test_sample = json.dumps({'data':X_test[index:index + 1].values.tolist()})\n",
|
|
||||||
" predicted = aci_service.run(input_data = test_sample)\n",
|
|
||||||
" label = y_test.values[index]\n",
|
|
||||||
" predictedDict = json.loads(predicted)\n",
|
|
||||||
" title = \"Label value = %d Predicted value = %s \" % ( label,predictedDict['result'][0]) \n",
|
|
||||||
" fig = plt.figure(3, figsize = (5,5))\n",
|
|
||||||
" ax1 = fig.add_axes((0,0,.8,.8))\n",
|
|
||||||
" ax1.set_title(title)\n",
|
|
||||||
" plt.imshow(images[index], cmap = plt.cm.gray_r, interpolation = 'nearest')\n",
|
|
||||||
" display(fig)"
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -703,7 +562,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"myservice.delete()"
|
"aci_service.delete()"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -741,7 +600,7 @@
|
|||||||
"version": "3.6.8"
|
"version": "3.6.8"
|
||||||
},
|
},
|
||||||
"name": "auto-ml-classification-local-adb",
|
"name": "auto-ml-classification-local-adb",
|
||||||
"notebookId": 2733885892129020
|
"notebookId": 3772036807853791
|
||||||
},
|
},
|
||||||
"nbformat": 4,
|
"nbformat": 4,
|
||||||
"nbformat_minor": 1
|
"nbformat_minor": 1
|
||||||
|
|||||||
@@ -1,5 +0,0 @@
|
|||||||
name: register-model-deploy-local-advanced
|
|
||||||
dependencies:
|
|
||||||
- pip:
|
|
||||||
- azureml-sdk
|
|
||||||
- scikit-learn
|
|
||||||
@@ -1,5 +0,0 @@
|
|||||||
name: register-model-deploy-local
|
|
||||||
dependencies:
|
|
||||||
- pip:
|
|
||||||
- azureml-sdk
|
|
||||||
- scikit-learn
|
|
||||||
@@ -172,7 +172,7 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"from azureml.core.conda_dependencies import CondaDependencies\n",
|
"from azureml.core.conda_dependencies import CondaDependencies\n",
|
||||||
"\n",
|
"\n",
|
||||||
"myenv = CondaDependencies.create(conda_packages=['numpy','scikit-learn'],\n",
|
"myenv = CondaDependencies.create(conda_packages=['numpy','scikit-learn==0.20.3'],\n",
|
||||||
" pip_packages=['azureml-defaults'])\n",
|
" pip_packages=['azureml-defaults'])\n",
|
||||||
"\n",
|
"\n",
|
||||||
"with open(\"myenv.yml\",\"w\") as f:\n",
|
"with open(\"myenv.yml\",\"w\") as f:\n",
|
||||||
@@ -465,7 +465,7 @@
|
|||||||
"metadata": {
|
"metadata": {
|
||||||
"authors": [
|
"authors": [
|
||||||
{
|
{
|
||||||
"name": "shipatel"
|
"name": "gopalv"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"kernelspec": {
|
"kernelspec": {
|
||||||
|
|||||||
@@ -334,14 +334,27 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"# Use the default configuration (can also provide parameters to customize)\n",
|
"from azureml.core.compute import ComputeTarget\n",
|
||||||
"prov_config = AksCompute.provisioning_configuration()\n",
|
"from azureml.core.compute_target import ComputeTargetException\n",
|
||||||
"\n",
|
"\n",
|
||||||
|
"# Choose a name for your AKS cluster\n",
|
||||||
"aks_name = 'my-aks-9' \n",
|
"aks_name = 'my-aks-9' \n",
|
||||||
"# Create the cluster\n",
|
"\n",
|
||||||
"aks_target = ComputeTarget.create(workspace = ws, \n",
|
"# Verify that cluster does not exist already\n",
|
||||||
" name = aks_name, \n",
|
"try:\n",
|
||||||
" provisioning_configuration = prov_config)"
|
" aks_target = ComputeTarget(workspace=ws, name=aks_name)\n",
|
||||||
|
" print('Found existing cluster, use it.')\n",
|
||||||
|
"except ComputeTargetException:\n",
|
||||||
|
" # Use the default configuration (can also provide parameters to customize)\n",
|
||||||
|
" prov_config = AksCompute.provisioning_configuration()\n",
|
||||||
|
"\n",
|
||||||
|
" # Create the cluster\n",
|
||||||
|
" aks_target = ComputeTarget.create(workspace = ws, \n",
|
||||||
|
" name = aks_name, \n",
|
||||||
|
" provisioning_configuration = prov_config)\n",
|
||||||
|
"\n",
|
||||||
|
"if aks_target.get_status() != \"Succeeded\":\n",
|
||||||
|
" aks_target.wait_for_completion(show_output=True)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -262,8 +262,7 @@
|
|||||||
"run_config.environment.docker.enabled = True\n",
|
"run_config.environment.docker.enabled = True\n",
|
||||||
"\n",
|
"\n",
|
||||||
"azureml_pip_packages = [\n",
|
"azureml_pip_packages = [\n",
|
||||||
" 'azureml-defaults', 'azureml-contrib-interpret', 'azureml-core', 'azureml-telemetry',\n",
|
" 'azureml-defaults', 'azureml-contrib-interpret', 'azureml-telemetry', 'azureml-interpret'\n",
|
||||||
" 'azureml-interpret', 'azureml-dataprep'\n",
|
|
||||||
"]\n",
|
"]\n",
|
||||||
"\n",
|
"\n",
|
||||||
"# Note: this is to pin the scikit-learn and pandas versions to be same as notebook.\n",
|
"# Note: this is to pin the scikit-learn and pandas versions to be same as notebook.\n",
|
||||||
@@ -288,8 +287,8 @@
|
|||||||
"# the submitted job is run in. Note the remote environment(s) needs to be similar to the local\n",
|
"# the submitted job is run in. Note the remote environment(s) needs to be similar to the local\n",
|
||||||
"# environment, otherwise if a model is trained or deployed in a different environment this can\n",
|
"# environment, otherwise if a model is trained or deployed in a different environment this can\n",
|
||||||
"# cause errors. Please take extra care when specifying your dependencies in a production environment.\n",
|
"# cause errors. Please take extra care when specifying your dependencies in a production environment.\n",
|
||||||
"run_config.environment.python.conda_dependencies = CondaDependencies.create(conda_packages=[sklearn_dep, pandas_dep],\n",
|
"azureml_pip_packages.extend([sklearn_dep, pandas_dep])\n",
|
||||||
" pip_packages=azureml_pip_packages)\n",
|
"run_config.environment.python.conda_dependencies = CondaDependencies.create(pip_packages=azureml_pip_packages)\n",
|
||||||
"\n",
|
"\n",
|
||||||
"from azureml.core import Run\n",
|
"from azureml.core import Run\n",
|
||||||
"from azureml.core import ScriptRunConfig\n",
|
"from azureml.core import ScriptRunConfig\n",
|
||||||
@@ -401,8 +400,7 @@
|
|||||||
"run_config.environment.docker.enabled = True\n",
|
"run_config.environment.docker.enabled = True\n",
|
||||||
"\n",
|
"\n",
|
||||||
"azureml_pip_packages = [\n",
|
"azureml_pip_packages = [\n",
|
||||||
" 'azureml-defaults', 'azureml-contrib-interpret', 'azureml-core', 'azureml-telemetry',\n",
|
" 'azureml-defaults', 'azureml-contrib-interpret', 'azureml-telemetry', 'azureml-interpret'\n",
|
||||||
" 'azureml-interpret', 'azureml-dataprep'\n",
|
|
||||||
"]\n",
|
"]\n",
|
||||||
"\n",
|
"\n",
|
||||||
"\n",
|
"\n",
|
||||||
@@ -429,8 +427,8 @@
|
|||||||
"# the submitted job is run in. Note the remote environment(s) needs to be similar to the local\n",
|
"# the submitted job is run in. Note the remote environment(s) needs to be similar to the local\n",
|
||||||
"# environment, otherwise if a model is trained or deployed in a different environment this can\n",
|
"# environment, otherwise if a model is trained or deployed in a different environment this can\n",
|
||||||
"# cause errors. Please take extra care when specifying your dependencies in a production environment.\n",
|
"# cause errors. Please take extra care when specifying your dependencies in a production environment.\n",
|
||||||
"run_config.environment.python.conda_dependencies = CondaDependencies.create(conda_packages=[sklearn_dep, pandas_dep],\n",
|
"azureml_pip_packages.extend([sklearn_dep, pandas_dep])\n",
|
||||||
" pip_packages=azureml_pip_packages)\n",
|
"run_config.environment.python.conda_dependencies = CondaDependencies.create(pip_packages=azureml_pip_packages)\n",
|
||||||
"\n",
|
"\n",
|
||||||
"from azureml.core import Run\n",
|
"from azureml.core import Run\n",
|
||||||
"from azureml.core import ScriptRunConfig\n",
|
"from azureml.core import ScriptRunConfig\n",
|
||||||
|
|||||||
@@ -7,5 +7,5 @@ dependencies:
|
|||||||
- matplotlib
|
- matplotlib
|
||||||
- azureml-contrib-interpret
|
- azureml-contrib-interpret
|
||||||
- sklearn-pandas
|
- sklearn-pandas
|
||||||
- azureml-dataprep
|
- azureml-dataset-runtime
|
||||||
- ipywidgets
|
- ipywidgets
|
||||||
|
|||||||
@@ -350,8 +350,7 @@
|
|||||||
"# the submitted job is run in. Note the remote environment(s) needs to be similar to the local\n",
|
"# the submitted job is run in. Note the remote environment(s) needs to be similar to the local\n",
|
||||||
"# environment, otherwise if a model is trained or deployed in a different environment this can\n",
|
"# environment, otherwise if a model is trained or deployed in a different environment this can\n",
|
||||||
"# cause errors. Please take extra care when specifying your dependencies in a production environment.\n",
|
"# cause errors. Please take extra care when specifying your dependencies in a production environment.\n",
|
||||||
"myenv = CondaDependencies.create(conda_packages=[sklearn_dep, pandas_dep],\n",
|
"myenv = CondaDependencies.create(pip_packages=['sklearn-pandas', 'pyyaml', sklearn_dep, pandas_dep] + azureml_pip_packages,\n",
|
||||||
" pip_packages=['sklearn-pandas', 'pyyaml'] + azureml_pip_packages,\n",
|
|
||||||
" pin_sdk_version=False)\n",
|
" pin_sdk_version=False)\n",
|
||||||
"\n",
|
"\n",
|
||||||
"with open(\"myenv.yml\",\"w\") as f:\n",
|
"with open(\"myenv.yml\",\"w\") as f:\n",
|
||||||
|
|||||||
@@ -267,8 +267,7 @@
|
|||||||
"run_config.environment.python.user_managed_dependencies = False\n",
|
"run_config.environment.python.user_managed_dependencies = False\n",
|
||||||
"\n",
|
"\n",
|
||||||
"azureml_pip_packages = [\n",
|
"azureml_pip_packages = [\n",
|
||||||
" 'azureml-defaults', 'azureml-contrib-interpret', 'azureml-core', 'azureml-telemetry',\n",
|
" 'azureml-defaults', 'azureml-contrib-interpret', 'azureml-telemetry', 'azureml-interpret'\n",
|
||||||
" 'azureml-interpret', 'azureml-dataprep'\n",
|
|
||||||
"]\n",
|
"]\n",
|
||||||
" \n",
|
" \n",
|
||||||
"\n",
|
"\n",
|
||||||
@@ -295,8 +294,8 @@
|
|||||||
"# the submitted job is run in. Note the remote environment(s) needs to be similar to the local\n",
|
"# the submitted job is run in. Note the remote environment(s) needs to be similar to the local\n",
|
||||||
"# environment, otherwise if a model is trained or deployed in a different environment this can\n",
|
"# environment, otherwise if a model is trained or deployed in a different environment this can\n",
|
||||||
"# cause errors. Please take extra care when specifying your dependencies in a production environment.\n",
|
"# cause errors. Please take extra care when specifying your dependencies in a production environment.\n",
|
||||||
"run_config.environment.python.conda_dependencies = CondaDependencies.create(conda_packages=[sklearn_dep, pandas_dep],\n",
|
"azureml_pip_packages.extend(['sklearn-pandas', 'pyyaml', sklearn_dep, pandas_dep])\n",
|
||||||
" pip_packages=['sklearn_pandas', 'pyyaml'] + azureml_pip_packages,\n",
|
"run_config.environment.python.conda_dependencies = CondaDependencies.create(pip_packages=azureml_pip_packages,\n",
|
||||||
" pin_sdk_version=False)\n",
|
" pin_sdk_version=False)\n",
|
||||||
"# Now submit a run on AmlCompute\n",
|
"# Now submit a run on AmlCompute\n",
|
||||||
"from azureml.core.script_run_config import ScriptRunConfig\n",
|
"from azureml.core.script_run_config import ScriptRunConfig\n",
|
||||||
@@ -423,13 +422,6 @@
|
|||||||
"Deploy Model and ScoringExplainer"
|
"Deploy Model and ScoringExplainer"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": []
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
"execution_count": null,
|
"execution_count": null,
|
||||||
@@ -467,8 +459,8 @@
|
|||||||
"# the submitted job is run in. Note the remote environment(s) needs to be similar to the local\n",
|
"# the submitted job is run in. Note the remote environment(s) needs to be similar to the local\n",
|
||||||
"# environment, otherwise if a model is trained or deployed in a different environment this can\n",
|
"# environment, otherwise if a model is trained or deployed in a different environment this can\n",
|
||||||
"# cause errors. Please take extra care when specifying your dependencies in a production environment.\n",
|
"# cause errors. Please take extra care when specifying your dependencies in a production environment.\n",
|
||||||
"myenv = CondaDependencies.create(conda_packages=[sklearn_dep, pandas_dep],\n",
|
"azureml_pip_packages.extend(['sklearn-pandas', 'pyyaml', sklearn_dep, pandas_dep])\n",
|
||||||
" pip_packages=['sklearn-pandas', 'pyyaml'] + azureml_pip_packages,\n",
|
"myenv = CondaDependencies.create(pip_packages=azureml_pip_packages,\n",
|
||||||
" pin_sdk_version=False)\n",
|
" pin_sdk_version=False)\n",
|
||||||
"\n",
|
"\n",
|
||||||
"with open(\"myenv.yml\",\"w\") as f:\n",
|
"with open(\"myenv.yml\",\"w\") as f:\n",
|
||||||
@@ -563,13 +555,6 @@
|
|||||||
"1. [Inferencing time: deploy a locally-trained model and explainer](./train-explain-model-locally-and-deploy.ipynb)\n",
|
"1. [Inferencing time: deploy a locally-trained model and explainer](./train-explain-model-locally-and-deploy.ipynb)\n",
|
||||||
"1. [Inferencing time: deploy a locally-trained keras model and explainer](./train-explain-model-keras-locally-and-deploy.ipynb)"
|
"1. [Inferencing time: deploy a locally-trained keras model and explainer](./train-explain-model-keras-locally-and-deploy.ipynb)"
|
||||||
]
|
]
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": []
|
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"metadata": {
|
"metadata": {
|
||||||
|
|||||||
@@ -7,6 +7,6 @@ dependencies:
|
|||||||
- matplotlib
|
- matplotlib
|
||||||
- azureml-contrib-interpret
|
- azureml-contrib-interpret
|
||||||
- sklearn-pandas
|
- sklearn-pandas
|
||||||
- azureml-dataprep
|
- azureml-dataset-runtime
|
||||||
- azureml-core
|
- azureml-core
|
||||||
- ipywidgets
|
- ipywidgets
|
||||||
|
|||||||
@@ -342,7 +342,7 @@
|
|||||||
"## Running a few steps in parallel\n",
|
"## Running a few steps in parallel\n",
|
||||||
"Here we are looking at a simple scenario where we are running a few steps (all involving PythonScriptStep) in parallel. Running nodes in **parallel** is the default behavior for steps in a pipeline.\n",
|
"Here we are looking at a simple scenario where we are running a few steps (all involving PythonScriptStep) in parallel. Running nodes in **parallel** is the default behavior for steps in a pipeline.\n",
|
||||||
"\n",
|
"\n",
|
||||||
"We already have one step defined earlier. Let's define few more steps."
|
"We already have one step defined earlier. Let's define few more steps. For step3, we are using customized conda-dependency, and job might fail when \"azureml-defaults\" (or other meta package) is not in pip-package list. We need to be aware if we are not using any of the meta packages (azureml-sdk, azureml-defaults, azureml-core), and we recommend installing \"azureml-defaults\"."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -637,7 +637,7 @@
|
|||||||
"name": "python",
|
"name": "python",
|
||||||
"nbconvert_exporter": "python",
|
"nbconvert_exporter": "python",
|
||||||
"pygments_lexer": "ipython3",
|
"pygments_lexer": "ipython3",
|
||||||
"version": "3.6.7"
|
"version": "3.6.2"
|
||||||
},
|
},
|
||||||
"order_index": 1,
|
"order_index": 1,
|
||||||
"tags": [
|
"tags": [
|
||||||
|
|||||||
@@ -544,7 +544,7 @@
|
|||||||
"metadata": {
|
"metadata": {
|
||||||
"authors": [
|
"authors": [
|
||||||
{
|
{
|
||||||
"name": "sanpil"
|
"name": "nagaur"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"category": "tutorial",
|
"category": "tutorial",
|
||||||
|
|||||||
@@ -40,7 +40,7 @@
|
|||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"import azureml.core\n",
|
"import azureml.core\n",
|
||||||
"from azureml.core import Workspace, Datastore, Experiment\n",
|
"from azureml.core import Workspace, Datastore, Experiment, Dataset\n",
|
||||||
"from azureml.core.compute import AmlCompute\n",
|
"from azureml.core.compute import AmlCompute\n",
|
||||||
"from azureml.core.compute import ComputeTarget\n",
|
"from azureml.core.compute import ComputeTarget\n",
|
||||||
"\n",
|
"\n",
|
||||||
@@ -109,7 +109,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"## Building Pipeline Steps with Inputs and Outputs\n",
|
"## Building Pipeline Steps with Inputs and Outputs\n",
|
||||||
"As mentioned earlier, a step in the pipeline can take data as input. This data can be a data source that lives in one of the accessible data locations, or intermediate data produced by a previous step in the pipeline."
|
"A step in the pipeline can take [dataset](https://docs.microsoft.com/python/api/azureml-core/azureml.data.filedataset?view=azure-ml-py) as input. This dataset can be a data source that lives in one of the accessible data locations, or intermediate data produced by a previous step in the pipeline."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -118,13 +118,20 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"# Reference the data uploaded to blob storage using DataReference\n",
|
"# Uploading data to the datastore\n",
|
||||||
|
"data_path = def_blob_store.upload_files([\"./20news.pkl\"], target_path=\"20newsgroups\", overwrite=True)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# Reference the data uploaded to blob storage using file dataset\n",
|
||||||
"# Assign the datasource to blob_input_data variable\n",
|
"# Assign the datasource to blob_input_data variable\n",
|
||||||
"blob_input_data = DataReference(\n",
|
"blob_input_data = Dataset.File.from_files(data_path).as_named_input(\"test_data\")\n",
|
||||||
" datastore=def_blob_store,\n",
|
"print(\"Dataset created\")"
|
||||||
" data_reference_name=\"test_data\",\n",
|
|
||||||
" path_on_datastore=\"20newsgroups/20news.pkl\")\n",
|
|
||||||
"print(\"DataReference object created\")"
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -142,8 +149,8 @@
|
|||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"#### Define a Step that consumes a datasource and produces intermediate data.\n",
|
"#### Define a Step that consumes a dataset and produces intermediate data.\n",
|
||||||
"In this step, we define a step that consumes a datasource and produces intermediate data.\n",
|
"In this step, we define a step that consumes a dataset and produces intermediate data.\n",
|
||||||
"\n",
|
"\n",
|
||||||
"**Open `train.py` in the local machine and examine the arguments, inputs, and outputs for the script. That will give you a good sense of why the script argument names used below are important.** \n",
|
"**Open `train.py` in the local machine and examine the arguments, inputs, and outputs for the script. That will give you a good sense of why the script argument names used below are important.** \n",
|
||||||
"\n",
|
"\n",
|
||||||
|
|||||||
@@ -1,5 +1,13 @@
|
|||||||
{
|
{
|
||||||
"cells": [
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Copyright (c) Microsoft Corporation. All rights reserved. \n",
|
||||||
|
"Licensed under the MIT License."
|
||||||
|
]
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
@@ -1062,7 +1070,7 @@
|
|||||||
"metadata": {
|
"metadata": {
|
||||||
"authors": [
|
"authors": [
|
||||||
{
|
{
|
||||||
"name": "sanpil"
|
"name": "anshirga"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"kernelspec": {
|
"kernelspec": {
|
||||||
|
|||||||
@@ -29,8 +29,8 @@ print("Argument 2(output final transformed taxi data): %s" % args.output_transfo
|
|||||||
# use the drop_columns() function to delete the original fields as the newly generated features are preferred.
|
# use the drop_columns() function to delete the original fields as the newly generated features are preferred.
|
||||||
# Rename the rest of the fields to use meaningful descriptions.
|
# Rename the rest of the fields to use meaningful descriptions.
|
||||||
|
|
||||||
normalized_df = normalized_df.astype({"pickup_date": 'datetime64', "dropoff_date": 'datetime64',
|
normalized_df = normalized_df.astype({"pickup_date": 'datetime64[ns]', "dropoff_date": 'datetime64[ns]',
|
||||||
"pickup_time": 'datetime64', "dropoff_time": 'datetime64',
|
"pickup_time": 'datetime64[us]', "dropoff_time": 'datetime64[us]',
|
||||||
"distance": 'float64', "cost": 'float64'})
|
"distance": 'float64', "cost": 'float64'})
|
||||||
|
|
||||||
normalized_df["pickup_weekday"] = normalized_df["pickup_date"].dt.dayofweek
|
normalized_df["pickup_weekday"] = normalized_df["pickup_date"].dt.dayofweek
|
||||||
|
|||||||
@@ -6,10 +6,15 @@ import numpy as np
|
|||||||
from azureml.core.model import Model
|
from azureml.core.model import Model
|
||||||
from sklearn.linear_model import LogisticRegression
|
from sklearn.linear_model import LogisticRegression
|
||||||
|
|
||||||
|
from azureml_user.parallel_run import EntryScript
|
||||||
|
|
||||||
|
|
||||||
def init():
|
def init():
|
||||||
global iris_model
|
global iris_model
|
||||||
|
|
||||||
|
logger = EntryScript().logger
|
||||||
|
logger.info("init() is called.")
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(description="Iris model serving")
|
parser = argparse.ArgumentParser(description="Iris model serving")
|
||||||
parser.add_argument('--model_name', dest="model_name", required=True)
|
parser.add_argument('--model_name', dest="model_name", required=True)
|
||||||
args, unknown_args = parser.parse_known_args()
|
args, unknown_args = parser.parse_known_args()
|
||||||
@@ -20,6 +25,9 @@ def init():
|
|||||||
|
|
||||||
|
|
||||||
def run(input_data):
|
def run(input_data):
|
||||||
|
logger = EntryScript().logger
|
||||||
|
logger.info("run() is called with: {}.".format(input_data))
|
||||||
|
|
||||||
# make inference
|
# make inference
|
||||||
num_rows, num_cols = input_data.shape
|
num_rows, num_cols = input_data.shape
|
||||||
pred = iris_model.predict(input_data).reshape((num_rows, 1))
|
pred = iris_model.predict(input_data).reshape((num_rows, 1))
|
||||||
|
|||||||
@@ -51,7 +51,23 @@
|
|||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
"execution_count": null,
|
"execution_count": null,
|
||||||
"metadata": {},
|
"metadata": {
|
||||||
|
"tags": []
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# Check core SDK version number\n",
|
||||||
|
"import azureml.core\n",
|
||||||
|
"\n",
|
||||||
|
"print(\"SDK version:\", azureml.core.VERSION)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"tags": []
|
||||||
|
},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"from azureml.core import Workspace\n",
|
"from azureml.core import Workspace\n",
|
||||||
@@ -138,7 +154,7 @@
|
|||||||
"\n",
|
"\n",
|
||||||
"mnist_data = Datastore.register_azure_blob_container(ws, \n",
|
"mnist_data = Datastore.register_azure_blob_container(ws, \n",
|
||||||
" datastore_name=datastore_name, \n",
|
" datastore_name=datastore_name, \n",
|
||||||
" container_name= container_name, \n",
|
" container_name=container_name, \n",
|
||||||
" account_name=account_name,\n",
|
" account_name=account_name,\n",
|
||||||
" overwrite=True)"
|
" overwrite=True)"
|
||||||
]
|
]
|
||||||
@@ -181,6 +197,13 @@
|
|||||||
"input_mnist_ds = Dataset.File.from_files(path=path_on_datastore, validate=False)"
|
"input_mnist_ds = Dataset.File.from_files(path=path_on_datastore, validate=False)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"The input dataset can be specified as a pipeline parameter, so that you can pass in new data when rerun the PRS pipeline."
|
||||||
|
]
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
"execution_count": null,
|
"execution_count": null,
|
||||||
@@ -199,15 +222,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"### Intermediate/Output Data\n",
|
"### Intermediate/Output Data\n",
|
||||||
"Intermediate data (or output of a Step) is represented by [PipelineData](https://docs.microsoft.com/en-us/python/api/azureml-pipeline-core/azureml.pipeline.core.pipelinedata?view=azure-ml-py) object. PipelineData can be produced by one step and consumed in another step by providing the PipelineData object as an output of one step and the input of one or more steps.\n",
|
"Intermediate data (or output of a Step) is represented by [PipelineData](https://docs.microsoft.com/en-us/python/api/azureml-pipeline-core/azureml.pipeline.core.pipelinedata?view=azure-ml-py) object. PipelineData can be produced by one step and consumed in another step by providing the PipelineData object as an output of one step and the input of one or more steps."
|
||||||
"\n",
|
|
||||||
"**Constructing PipelineData**\n",
|
|
||||||
"- name: [Required] Name of the data item within the pipeline graph\n",
|
|
||||||
"- datastore_name: Name of the Datastore to write this output to\n",
|
|
||||||
"- output_name: Name of the output\n",
|
|
||||||
"- output_mode: Specifies \"upload\" or \"mount\" modes for producing output (default: mount)\n",
|
|
||||||
"- output_path_on_compute: For \"upload\" mode, the path to which the module writes this output during execution\n",
|
|
||||||
"- output_overwrite: Flag to overwrite pre-existing data"
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -218,9 +233,7 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"from azureml.pipeline.core import Pipeline, PipelineData\n",
|
"from azureml.pipeline.core import Pipeline, PipelineData\n",
|
||||||
"\n",
|
"\n",
|
||||||
"output_dir = PipelineData(name=\"inferences\", \n",
|
"output_dir = PipelineData(name=\"inferences\", datastore=def_data_store)"
|
||||||
" datastore=def_data_store, \n",
|
|
||||||
" output_path_on_compute=\"mnist/results\")"
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -273,11 +286,11 @@
|
|||||||
"from azureml.core.model import Model\n",
|
"from azureml.core.model import Model\n",
|
||||||
"\n",
|
"\n",
|
||||||
"# register downloaded model \n",
|
"# register downloaded model \n",
|
||||||
"model = Model.register(model_path = \"models/\",\n",
|
"model = Model.register(model_path=\"models/\",\n",
|
||||||
" model_name = \"mnist-prs\", # this is the name the model is registered as\n",
|
" model_name=\"mnist-prs\", # this is the name the model is registered as\n",
|
||||||
" tags = {'pretrained': \"mnist\"},\n",
|
" tags={'pretrained': \"mnist\"},\n",
|
||||||
" description = \"Mnist trained tensorflow model\",\n",
|
" description=\"Mnist trained tensorflow model\",\n",
|
||||||
" workspace = ws)"
|
" workspace=ws)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -298,11 +311,7 @@
|
|||||||
" \n",
|
" \n",
|
||||||
"\n",
|
"\n",
|
||||||
"#### Dependencies\n",
|
"#### Dependencies\n",
|
||||||
"Helper scripts or Python/Conda packages required to run the entry script or model.\n",
|
"Helper scripts or Python/Conda packages required to run the entry script."
|
||||||
"\n",
|
|
||||||
"The deployment configuration for the compute target that hosts the deployed model. This configuration describes things like memory and CPU requirements needed to run the model.\n",
|
|
||||||
"\n",
|
|
||||||
"These items are encapsulated into an inference configuration and a deployment configuration. The inference configuration references the entry script and other dependencies. You define these configurations programmatically when you use the SDK to perform the deployment. You define them in JSON files when you use the CLI."
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -332,7 +341,10 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"### Specify the environment to run the script\n",
|
"### Specify the environment to run the script\n",
|
||||||
"Specify the conda dependencies for your script. This will allow us to install pip packages as well as configure the inference environment."
|
"Specify the conda dependencies for your script. This will allow us to install pip packages as well as configure the inference environment.\n",
|
||||||
|
"* Always include **azureml-core** and **azureml-dataset-runtime\\[fuse\\]** in the pip package list to make ParallelRunStep run properly.\n",
|
||||||
|
"\n",
|
||||||
|
"If you're using custom image (`batch_env.python.user_managed_dependencies = True`), you need to install the package to your image."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -345,7 +357,7 @@
|
|||||||
"from azureml.core.runconfig import CondaDependencies, DEFAULT_CPU_IMAGE\n",
|
"from azureml.core.runconfig import CondaDependencies, DEFAULT_CPU_IMAGE\n",
|
||||||
"\n",
|
"\n",
|
||||||
"batch_conda_deps = CondaDependencies.create(pip_packages=[\"tensorflow==1.15.2\", \"pillow\", \n",
|
"batch_conda_deps = CondaDependencies.create(pip_packages=[\"tensorflow==1.15.2\", \"pillow\", \n",
|
||||||
" \"azureml-core\", \"azureml-dataprep[fuse]\"])\n",
|
" \"azureml-core\", \"azureml-dataset-runtime[fuse]\"])\n",
|
||||||
"batch_env = Environment(name=\"batch_environment\")\n",
|
"batch_env = Environment(name=\"batch_environment\")\n",
|
||||||
"batch_env.python.conda_dependencies = batch_conda_deps\n",
|
"batch_env.python.conda_dependencies = batch_conda_deps\n",
|
||||||
"batch_env.docker.enabled = True\n",
|
"batch_env.docker.enabled = True\n",
|
||||||
@@ -401,7 +413,7 @@
|
|||||||
" parallel_run_config=parallel_run_config,\n",
|
" parallel_run_config=parallel_run_config,\n",
|
||||||
" inputs=[ input_mnist_ds_consumption ],\n",
|
" inputs=[ input_mnist_ds_consumption ],\n",
|
||||||
" output=output_dir,\n",
|
" output=output_dir,\n",
|
||||||
" allow_reuse=True\n",
|
" allow_reuse=False\n",
|
||||||
")"
|
")"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -430,7 +442,9 @@
|
|||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"### Monitor the run"
|
"### Monitor the run\n",
|
||||||
|
"\n",
|
||||||
|
"The pipeline run status could be checked in Azure Machine Learning portal (https://ml.azure.com). The link to the pipeline run could be retrieved by inspecting the `pipeline_run` object."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -439,8 +453,8 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"from azureml.widgets import RunDetails\n",
|
"# This will output information of the pipeline run, including the link to the details page of portal.\n",
|
||||||
"RunDetails(pipeline_run).show()"
|
"pipeline_run"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -456,9 +470,40 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
|
"# Wait the run for completion and show output log to console\n",
|
||||||
"pipeline_run.wait_for_completion(show_output=True)"
|
"pipeline_run.wait_for_completion(show_output=True)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### View the prediction results per input image\n",
|
||||||
|
"In the digit_identification.py file above you can see that the ResultList with the filename and the prediction result gets returned. These are written to the DataStore specified in the PipelineData object as the output data, which in this case is called *inferences*. This containers the outputs from all of the worker nodes used in the compute cluster. You can download this data to view the results ... below just filters to the first 10 rows"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import pandas as pd\n",
|
||||||
|
"import tempfile\n",
|
||||||
|
"\n",
|
||||||
|
"batch_run = pipeline_run.find_step_run(parallelrun_step.name)[0]\n",
|
||||||
|
"batch_output = batch_run.get_output_data(output_dir.name)\n",
|
||||||
|
"\n",
|
||||||
|
"target_dir = tempfile.mkdtemp()\n",
|
||||||
|
"batch_output.download(local_path=target_dir)\n",
|
||||||
|
"result_file = os.path.join(target_dir, batch_output.path_on_datastore, parallel_run_config.append_row_file_name)\n",
|
||||||
|
"\n",
|
||||||
|
"df = pd.read_csv(result_file, delimiter=\":\", header=None)\n",
|
||||||
|
"df.columns = [\"Filename\", \"Prediction\"]\n",
|
||||||
|
"print(\"Prediction has \", df.shape[0], \" rows\")\n",
|
||||||
|
"df.head(10) "
|
||||||
|
]
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
@@ -496,15 +541,8 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"pipeline_run_2.wait_for_completion(show_output=True)"
|
"# This will output information of the pipeline run, including the link to the details page of portal.\n",
|
||||||
]
|
"pipeline_run_2"
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### View the prediction results per input image\n",
|
|
||||||
"In the score.py file above you can see that the ResultList with the filename and the prediction result gets returned. These are written to the DataStore specified in the PipelineData object as the output data, which in this case is called *inferences*. This containers the outputs from all of the worker nodes used in the compute cluster. You can download this data to view the results ... below just filters to the first 10 rows"
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -513,25 +551,8 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"import pandas as pd\n",
|
"# Wait the run for completion and show output log to console\n",
|
||||||
"import shutil\n",
|
"pipeline_run_2.wait_for_completion(show_output=True)"
|
||||||
"\n",
|
|
||||||
"# remove previous run results, if present\n",
|
|
||||||
"shutil.rmtree(\"mnist_results\", ignore_errors=True)\n",
|
|
||||||
"\n",
|
|
||||||
"batch_run = next(pipeline_run.get_children())\n",
|
|
||||||
"batch_output = batch_run.get_output_data(\"inferences\")\n",
|
|
||||||
"batch_output.download(local_path=\"mnist_results\")\n",
|
|
||||||
"\n",
|
|
||||||
"for root, dirs, files in os.walk(\"mnist_results\"):\n",
|
|
||||||
" for file in files:\n",
|
|
||||||
" if file.endswith('mnist_outputs.txt'):\n",
|
|
||||||
" result_file = os.path.join(root,file)\n",
|
|
||||||
"\n",
|
|
||||||
"df = pd.read_csv(result_file, delimiter=\":\", header=None)\n",
|
|
||||||
"df.columns = [\"Filename\", \"Prediction\"]\n",
|
|
||||||
"print(\"Prediction has \", df.shape[0], \" rows\")\n",
|
|
||||||
"df.head(10) "
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -49,7 +49,23 @@
|
|||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
"execution_count": null,
|
"execution_count": null,
|
||||||
"metadata": {},
|
"metadata": {
|
||||||
|
"tags": []
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# Check core SDK version number\n",
|
||||||
|
"import azureml.core\n",
|
||||||
|
"\n",
|
||||||
|
"print(\"SDK version:\", azureml.core.VERSION)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"tags": []
|
||||||
|
},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"from azureml.core import Workspace\n",
|
"from azureml.core import Workspace\n",
|
||||||
@@ -168,15 +184,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"### Intermediate/Output Data\n",
|
"### Intermediate/Output Data\n",
|
||||||
"Intermediate data (or output of a Step) is represented by [PipelineData](https://docs.microsoft.com/en-us/python/api/azureml-pipeline-core/azureml.pipeline.core.pipelinedata?view=azure-ml-py) object. PipelineData can be produced by one step and consumed in another step by providing the PipelineData object as an output of one step and the input of one or more steps.\n",
|
"Intermediate data (or output of a Step) is represented by [PipelineData](https://docs.microsoft.com/en-us/python/api/azureml-pipeline-core/azureml.pipeline.core.pipelinedata?view=azure-ml-py) object. PipelineData can be produced by one step and consumed in another step by providing the PipelineData object as an output of one step and the input of one or more steps."
|
||||||
"\n",
|
|
||||||
"**Constructing PipelineData**\n",
|
|
||||||
"- name: [Required] Name of the data item within the pipeline graph\n",
|
|
||||||
"- datastore_name: Name of the Datastore to write this output to\n",
|
|
||||||
"- output_name: Name of the output\n",
|
|
||||||
"- output_mode: Specifies \"upload\" or \"mount\" modes for producing output (default: mount)\n",
|
|
||||||
"- output_path_on_compute: For \"upload\" mode, the path to which the module writes this output during execution\n",
|
|
||||||
"- output_overwrite: Flag to overwrite pre-existing data"
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -250,11 +258,7 @@
|
|||||||
" \n",
|
" \n",
|
||||||
"\n",
|
"\n",
|
||||||
"#### Dependencies\n",
|
"#### Dependencies\n",
|
||||||
"Helper scripts or Python/Conda packages required to run the entry script or model.\n",
|
"Helper scripts or Python/Conda packages required to run the entry script.\n",
|
||||||
"\n",
|
|
||||||
"The deployment configuration for the compute target that hosts the deployed model. This configuration describes things like memory and CPU requirements needed to run the model.\n",
|
|
||||||
"\n",
|
|
||||||
"These items are encapsulated into an inference configuration and a deployment configuration. The inference configuration references the entry script and other dependencies. You define these configurations programmatically when you use the SDK to perform the deployment. You define them in JSON files when you use the CLI.\n",
|
|
||||||
"\n",
|
"\n",
|
||||||
"## Print inferencing script"
|
"## Print inferencing script"
|
||||||
]
|
]
|
||||||
@@ -286,7 +290,11 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"### Specify the environment to run the script\n",
|
"### Specify the environment to run the script\n",
|
||||||
"Specify the conda dependencies for your script. This will allow us to install pip packages as well as configure the inference environment."
|
"Specify the conda dependencies for your script. This will allow us to install pip packages as well as configure the inference environment.\n",
|
||||||
|
"* Always include **azureml-core** and **azureml-dataset-runtime\\[fuse\\]** in the pip package list to make ParallelRunStep run properly.\n",
|
||||||
|
"* For TabularDataset, add **pandas** as `run(mini_batch)` uses `pandas.DataFrame` as mini_batch type.\n",
|
||||||
|
"\n",
|
||||||
|
"If you're using custom image (`batch_env.python.user_managed_dependencies = True`), you need to install the package to your image."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -299,7 +307,7 @@
|
|||||||
"from azureml.core.runconfig import CondaDependencies\n",
|
"from azureml.core.runconfig import CondaDependencies\n",
|
||||||
"\n",
|
"\n",
|
||||||
"predict_conda_deps = CondaDependencies.create(pip_packages=[\"scikit-learn==0.20.3\",\n",
|
"predict_conda_deps = CondaDependencies.create(pip_packages=[\"scikit-learn==0.20.3\",\n",
|
||||||
" \"azureml-core\", \"azureml-dataprep[pandas,fuse]\"])\n",
|
" \"azureml-core\", \"azureml-dataset-runtime[pandas,fuse]\"])\n",
|
||||||
"\n",
|
"\n",
|
||||||
"predict_env = Environment(name=\"predict_environment\")\n",
|
"predict_env = Environment(name=\"predict_environment\")\n",
|
||||||
"predict_env.python.conda_dependencies = predict_conda_deps\n",
|
"predict_env.python.conda_dependencies = predict_conda_deps\n",
|
||||||
@@ -326,7 +334,7 @@
|
|||||||
"parallel_run_config = ParallelRunConfig(\n",
|
"parallel_run_config = ParallelRunConfig(\n",
|
||||||
" source_directory=scripts_folder,\n",
|
" source_directory=scripts_folder,\n",
|
||||||
" entry_script=script_file, # the user script to run against each input\n",
|
" entry_script=script_file, # the user script to run against each input\n",
|
||||||
" mini_batch_size='5MB',\n",
|
" mini_batch_size='1KB',\n",
|
||||||
" error_threshold=5,\n",
|
" error_threshold=5,\n",
|
||||||
" output_action='append_row',\n",
|
" output_action='append_row',\n",
|
||||||
" append_row_file_name=\"iris_outputs.txt\",\n",
|
" append_row_file_name=\"iris_outputs.txt\",\n",
|
||||||
@@ -357,7 +365,7 @@
|
|||||||
" output=output_folder,\n",
|
" output=output_folder,\n",
|
||||||
" parallel_run_config=parallel_run_config,\n",
|
" parallel_run_config=parallel_run_config,\n",
|
||||||
" arguments=['--model_name', 'iris-prs'],\n",
|
" arguments=['--model_name', 'iris-prs'],\n",
|
||||||
" allow_reuse=True\n",
|
" allow_reuse=False\n",
|
||||||
")"
|
")"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -383,13 +391,22 @@
|
|||||||
"pipeline_run = Experiment(ws, 'iris-prs').submit(pipeline)"
|
"pipeline_run = Experiment(ws, 'iris-prs').submit(pipeline)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## View progress of Pipeline run\n",
|
||||||
|
"\n",
|
||||||
|
"The pipeline run status could be checked in Azure Machine Learning portal (https://ml.azure.com). The link to the pipeline run could be retrieved by inspecting the `pipeline_run` object."
|
||||||
|
]
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
"execution_count": null,
|
"execution_count": null,
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"# this will output a table with link to the run details in azure portal\n",
|
"# This will output information of the pipeline run, including the link to the details page of portal.\n",
|
||||||
"pipeline_run"
|
"pipeline_run"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -397,29 +414,18 @@
|
|||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"## View progress of Pipeline run\n",
|
"### Optional: View detailed logs (streaming) "
|
||||||
"\n",
|
|
||||||
"The progress of the pipeline is able to be viewed either through azureml.widgets or a console feed from PipelineRun.wait_for_completion()."
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
"execution_count": null,
|
"execution_count": null,
|
||||||
"metadata": {},
|
"metadata": {
|
||||||
|
"tags": []
|
||||||
|
},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"# GUI\n",
|
"## Wait the run for completion and show output log to console\n",
|
||||||
"from azureml.widgets import RunDetails\n",
|
|
||||||
"RunDetails(pipeline_run).show() "
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# Console logs\n",
|
|
||||||
"pipeline_run.wait_for_completion(show_output=True)"
|
"pipeline_run.wait_for_completion(show_output=True)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -438,19 +444,14 @@
|
|||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"import pandas as pd\n",
|
"import pandas as pd\n",
|
||||||
"import shutil\n",
|
"import tempfile\n",
|
||||||
"\n",
|
"\n",
|
||||||
"shutil.rmtree(\"iris_results\", ignore_errors=True)\n",
|
"prediction_run = pipeline_run.find_step_run(distributed_csv_iris_step.name)[0]\n",
|
||||||
|
"prediction_output = prediction_run.get_output_data(output_folder.name)\n",
|
||||||
"\n",
|
"\n",
|
||||||
"prediction_run = next(pipeline_run.get_children())\n",
|
"target_dir = tempfile.mkdtemp()\n",
|
||||||
"prediction_output = prediction_run.get_output_data(\"inferences\")\n",
|
"prediction_output.download(local_path=target_dir)\n",
|
||||||
"prediction_output.download(local_path=\"iris_results\")\n",
|
"result_file = os.path.join(target_dir, prediction_output.path_on_datastore, parallel_run_config.append_row_file_name)\n",
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"for root, dirs, files in os.walk(\"iris_results\"):\n",
|
|
||||||
" for file in files:\n",
|
|
||||||
" if file.endswith('iris_outputs.txt'):\n",
|
|
||||||
" result_file = os.path.join(root,file)\n",
|
|
||||||
"\n",
|
"\n",
|
||||||
"# cleanup output format\n",
|
"# cleanup output format\n",
|
||||||
"df = pd.read_csv(result_file, delimiter=\" \", header=None)\n",
|
"df = pd.read_csv(result_file, delimiter=\" \", header=None)\n",
|
||||||
|
|||||||
@@ -120,7 +120,7 @@
|
|||||||
"\n",
|
"\n",
|
||||||
"def download_model(model_name):\n",
|
"def download_model(model_name):\n",
|
||||||
" # downloaded models from https://pytorch.org/tutorials/advanced/neural_style_tutorial.html are kept here\n",
|
" # downloaded models from https://pytorch.org/tutorials/advanced/neural_style_tutorial.html are kept here\n",
|
||||||
" url=\"https://pipelinedata.blob.core.windows.net/styletransfer/saved_models/\" + model_name\n",
|
" url = \"https://pipelinedata.blob.core.windows.net/styletransfer/saved_models/\" + model_name\n",
|
||||||
" local_path = os.path.join(model_dir, model_name)\n",
|
" local_path = os.path.join(model_dir, model_name)\n",
|
||||||
" urllib.request.urlretrieve(url, local_path)"
|
" urllib.request.urlretrieve(url, local_path)"
|
||||||
]
|
]
|
||||||
@@ -415,7 +415,7 @@
|
|||||||
"parallel_cd.add_conda_package(\"torchvision\")\n",
|
"parallel_cd.add_conda_package(\"torchvision\")\n",
|
||||||
"parallel_cd.add_conda_package(\"pillow<7\") # needed for torchvision==0.4.0\n",
|
"parallel_cd.add_conda_package(\"pillow<7\") # needed for torchvision==0.4.0\n",
|
||||||
"parallel_cd.add_pip_package(\"azureml-core\")\n",
|
"parallel_cd.add_pip_package(\"azureml-core\")\n",
|
||||||
"parallel_cd.add_pip_package(\"azureml-dataprep[fuse]\")\n",
|
"parallel_cd.add_pip_package(\"azureml-dataset-runtime[fuse]\")\n",
|
||||||
"\n",
|
"\n",
|
||||||
"styleenvironment = Environment(name=\"styleenvironment\")\n",
|
"styleenvironment = Environment(name=\"styleenvironment\")\n",
|
||||||
"styleenvironment.python.conda_dependencies=parallel_cd\n",
|
"styleenvironment.python.conda_dependencies=parallel_cd\n",
|
||||||
@@ -461,7 +461,7 @@
|
|||||||
" output=processed_images, # Output file share/blob container\n",
|
" output=processed_images, # Output file share/blob container\n",
|
||||||
" arguments=[\"--style\", style_param],\n",
|
" arguments=[\"--style\", style_param],\n",
|
||||||
" parallel_run_config=parallel_run_config,\n",
|
" parallel_run_config=parallel_run_config,\n",
|
||||||
" allow_reuse=True #[optional - default value True]\n",
|
" allow_reuse=False #[optional - default value True]\n",
|
||||||
")"
|
")"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -497,7 +497,10 @@
|
|||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"# Monitor using widget"
|
"# Monitor pipeline run\n",
|
||||||
|
"\n",
|
||||||
|
"The pipeline run status could be checked in Azure Machine Learning portal (https://ml.azure.com). The link to the pipeline run could be retrieved by inspecting the `pipeline_run` object.\n",
|
||||||
|
"\n"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -506,25 +509,25 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"# Track pipeline run progress\n",
|
"# This will output information of the pipeline run, including the link to the details page of portal.\n",
|
||||||
"from azureml.widgets import RunDetails\n",
|
"pipeline_run"
|
||||||
"RunDetails(pipeline_run).show()"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"pipeline_run.wait_for_completion()"
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"Downloads the video in `output_video` folder"
|
"### Optional: View detailed logs (streaming) "
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# Wait the run for completion and show output log to console\n",
|
||||||
|
"pipeline_run.wait_for_completion(show_output=True)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -534,6 +537,13 @@
|
|||||||
"# Download output video"
|
"# Download output video"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Downloads the video in `output_video` folder"
|
||||||
|
]
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
"execution_count": null,
|
"execution_count": null,
|
||||||
@@ -541,8 +551,8 @@
|
|||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"def download_video(run, target_dir=None):\n",
|
"def download_video(run, target_dir=None):\n",
|
||||||
" stitch_run = run.find_step_run(\"stitch\")[0]\n",
|
" stitch_run = run.find_step_run(stitch_video_step.name)[0]\n",
|
||||||
" port_data = stitch_run.get_output_data(\"output_video\")\n",
|
" port_data = stitch_run.get_output_data(output_video.name)\n",
|
||||||
" port_data.download(target_dir, show_progress=True)"
|
" port_data.download(target_dir, show_progress=True)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -674,7 +684,8 @@
|
|||||||
"from azureml.pipeline.core.run import PipelineRun\n",
|
"from azureml.pipeline.core.run import PipelineRun\n",
|
||||||
"published_pipeline_run_candy = PipelineRun(ws.experiments[experiment_name], run_id)\n",
|
"published_pipeline_run_candy = PipelineRun(ws.experiments[experiment_name], run_id)\n",
|
||||||
"\n",
|
"\n",
|
||||||
"RunDetails(published_pipeline_run_candy).show()"
|
"# Show detail information of run\n",
|
||||||
|
"published_pipeline_run_candy"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -418,7 +418,7 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"from azureml.train.hyperdrive.runconfig import HyperDriveRunConfig\n",
|
"from azureml.train.hyperdrive.runconfig import HyperDriveConfig\n",
|
||||||
"from azureml.train.hyperdrive.sampling import RandomParameterSampling\n",
|
"from azureml.train.hyperdrive.sampling import RandomParameterSampling\n",
|
||||||
"from azureml.train.hyperdrive.run import PrimaryMetricGoal\n",
|
"from azureml.train.hyperdrive.run import PrimaryMetricGoal\n",
|
||||||
"from azureml.train.hyperdrive.parameter_expressions import choice\n",
|
"from azureml.train.hyperdrive.parameter_expressions import choice\n",
|
||||||
@@ -430,12 +430,12 @@
|
|||||||
" }\n",
|
" }\n",
|
||||||
")\n",
|
")\n",
|
||||||
"\n",
|
"\n",
|
||||||
"hyperdrive_run_config = HyperDriveRunConfig(estimator=estimator,\n",
|
"hyperdrive_config = HyperDriveConfig(estimator=estimator,\n",
|
||||||
" hyperparameter_sampling=param_sampling, \n",
|
" hyperparameter_sampling=param_sampling, \n",
|
||||||
" primary_metric_name='Accuracy',\n",
|
" primary_metric_name='Accuracy',\n",
|
||||||
" primary_metric_goal=PrimaryMetricGoal.MAXIMIZE,\n",
|
" primary_metric_goal=PrimaryMetricGoal.MAXIMIZE,\n",
|
||||||
" max_total_runs=12,\n",
|
" max_total_runs=12,\n",
|
||||||
" max_concurrent_runs=4)"
|
" max_concurrent_runs=4)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -452,7 +452,7 @@
|
|||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"# start the HyperDrive run\n",
|
"# start the HyperDrive run\n",
|
||||||
"hyperdrive_run = experiment.submit(hyperdrive_run_config)"
|
"hyperdrive_run = experiment.submit(hyperdrive_config)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -459,7 +459,7 @@
|
|||||||
" entry_script='tf_mnist.py',\n",
|
" entry_script='tf_mnist.py',\n",
|
||||||
" use_gpu=True,\n",
|
" use_gpu=True,\n",
|
||||||
" framework_version='2.0',\n",
|
" framework_version='2.0',\n",
|
||||||
" pip_packages=['azureml-dataprep[pandas,fuse]'])"
|
" pip_packages=['azureml-dataset-runtime[pandas,fuse]'])"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -630,52 +630,44 @@
|
|||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"## Predict on the test set\n",
|
"## Predict on the test set (Optional)\n",
|
||||||
"Now load the saved TensorFlow graph, and list all operations under the `network` scope. This way we can discover the input tensor `network/X:0` and the output tensor `network/output/MatMul:0`, and use them in the scoring script in the next step.\n",
|
"Now load the saved TensorFlow graph, and list all operations under the `network` scope. This way we can discover the input tensor `network/X:0` and the output tensor `network/output/MatMul:0`, and use them in the scoring script in the next step.\n",
|
||||||
"\n",
|
"\n",
|
||||||
"Note: if your local TensorFlow version is different than the version running in the cluster where the model is trained, you might see a \"compiletime version mismatch\" warning. You can ignore it."
|
"Note: if your local TensorFlow version is different than the version running in the cluster where the model is trained, you might see a \"compiletime version mismatch\" warning. You can ignore it."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "markdown",
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
"source": [
|
||||||
"import tensorflow as tf\n",
|
" import tensorflow as tf\n",
|
||||||
"imported_model = tf.saved_model.load('./model')"
|
" imported_model = tf.saved_model.load('./model')"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "markdown",
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
"source": [
|
||||||
"pred =imported_model(X_test)\n",
|
" pred = imported_model(X_test)\n",
|
||||||
"y_hat = np.argmax(pred, axis=1)\n",
|
" y_hat = np.argmax(pred, axis=1)\n",
|
||||||
"\n",
|
"\n",
|
||||||
"# print the first 30 labels and predictions\n",
|
" # print the first 30 labels and predictions\n",
|
||||||
"print('labels: \\t', y_test[:30])\n",
|
" print('labels: \\t', y_test[:30])\n",
|
||||||
"print('predictions:\\t', y_hat[:30])"
|
" print('predictions:\\t', y_hat[:30])"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "markdown",
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
"source": [
|
||||||
"print(\"Accuracy on the test set:\", np.average(y_hat == y_test))"
|
" print(\"Accuracy on the test set:\", np.average(y_hat == y_test))"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "markdown",
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
"source": [
|
||||||
"print(\"Accuracy on the test set:\", np.average(y_hat == y_test))"
|
" print(\"Accuracy on the test set:\", np.average(y_hat == y_test))"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -724,7 +716,7 @@
|
|||||||
" entry_script='tf_mnist.py',\n",
|
" entry_script='tf_mnist.py',\n",
|
||||||
" framework_version='2.0',\n",
|
" framework_version='2.0',\n",
|
||||||
" use_gpu=True,\n",
|
" use_gpu=True,\n",
|
||||||
" pip_packages=['azureml-dataprep[pandas,fuse]'])"
|
" pip_packages=['azureml-dataset-runtime[pandas,fuse]'])"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -1055,7 +1047,7 @@
|
|||||||
" font_color = 'red' if y_test[s] != result[i] else 'black'\n",
|
" font_color = 'red' if y_test[s] != result[i] else 'black'\n",
|
||||||
" clr_map = plt.cm.gray if y_test[s] != result[i] else plt.cm.Greys\n",
|
" clr_map = plt.cm.gray if y_test[s] != result[i] else plt.cm.Greys\n",
|
||||||
" \n",
|
" \n",
|
||||||
" plt.text(x=10, y=-10, s=y_hat[s], fontsize=18, color=font_color)\n",
|
" plt.text(x=10, y=-10, s=result[i], fontsize=18, color=font_color)\n",
|
||||||
" plt.imshow(X_test[s].reshape(28, 28), cmap=clr_map)\n",
|
" plt.imshow(X_test[s].reshape(28, 28), cmap=clr_map)\n",
|
||||||
" \n",
|
" \n",
|
||||||
" i = i + 1\n",
|
" i = i + 1\n",
|
||||||
|
|||||||
@@ -158,7 +158,7 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"from azureml.core import Dataset\n",
|
"from azureml.core import Dataset\n",
|
||||||
"\n",
|
"\n",
|
||||||
"web_paths = ['http://mattmahoney.net/dc/text8.zip']\n",
|
"web_paths = ['https://azureopendatastorage.blob.core.windows.net/testpublic/text8.zip']\n",
|
||||||
"dataset = Dataset.File.from_files(path=web_paths)"
|
"dataset = Dataset.File.from_files(path=web_paths)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
@@ -285,7 +285,7 @@
|
|||||||
" distributed_training=Mpi(),\n",
|
" distributed_training=Mpi(),\n",
|
||||||
" framework_version='1.13', \n",
|
" framework_version='1.13', \n",
|
||||||
" use_gpu=True,\n",
|
" use_gpu=True,\n",
|
||||||
" pip_packages=['azureml-dataprep[pandas,fuse]'])"
|
" pip_packages=['azureml-dataset-runtime[pandas,fuse]'])"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ dependencies:
|
|||||||
- azureml-widgets
|
- azureml-widgets
|
||||||
- keras
|
- keras
|
||||||
- tensorflow-gpu==1.13.2
|
- tensorflow-gpu==1.13.2
|
||||||
- horovod==0.16.1
|
- horovod==0.19.1
|
||||||
- matplotlib
|
- matplotlib
|
||||||
- pandas
|
- pandas
|
||||||
- fuse
|
- fuse
|
||||||
|
|||||||
@@ -442,12 +442,12 @@
|
|||||||
"\n",
|
"\n",
|
||||||
"# set up environment\\n\n",
|
"# set up environment\\n\n",
|
||||||
"env = Environment('my_env')\n",
|
"env = Environment('my_env')\n",
|
||||||
"# ensure latest azureml-dataprep and other required packages installed in the environment\n",
|
"# ensure latest azureml-dataset-runtime and other required packages installed in the environment\n",
|
||||||
"cd = CondaDependencies.create(pip_packages=['keras',\n",
|
"cd = CondaDependencies.create(pip_packages=['keras',\n",
|
||||||
" 'azureml-sdk',\n",
|
" 'azureml-sdk',\n",
|
||||||
" 'tensorflow==2.0.0',\n",
|
" 'tensorflow==2.0.0',\n",
|
||||||
" 'matplotlib',\n",
|
" 'matplotlib',\n",
|
||||||
" 'azureml-dataprep[pandas,fuse]'])\n",
|
" 'azureml-dataset-runtime[pandas,fuse]'])\n",
|
||||||
"\n",
|
"\n",
|
||||||
"env.python.conda_dependencies = cd"
|
"env.python.conda_dependencies = cd"
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -300,7 +300,7 @@
|
|||||||
" script_params=script_params,\n",
|
" script_params=script_params,\n",
|
||||||
" entry_script='tf_mnist_with_checkpoint.py',\n",
|
" entry_script='tf_mnist_with_checkpoint.py',\n",
|
||||||
" use_gpu=True,\n",
|
" use_gpu=True,\n",
|
||||||
" pip_packages=['azureml-dataprep[pandas,fuse]'])"
|
" pip_packages=['azureml-dataset-runtime[pandas,fuse]'])"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -411,7 +411,7 @@
|
|||||||
" entry_script='tf_mnist_with_checkpoint.py',\n",
|
" entry_script='tf_mnist_with_checkpoint.py',\n",
|
||||||
" resume_from=model_location,\n",
|
" resume_from=model_location,\n",
|
||||||
" use_gpu=True,\n",
|
" use_gpu=True,\n",
|
||||||
" pip_packages=['azureml-dataprep[pandas,fuse]'])"
|
" pip_packages=['azureml-dataset-runtime[pandas,fuse]'])"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -1,31 +0,0 @@
|
|||||||
# imports
|
|
||||||
import pickle
|
|
||||||
from datetime import datetime
|
|
||||||
from azureml.opendatasets import NoaaIsdWeather
|
|
||||||
from sklearn.linear_model import LinearRegression
|
|
||||||
|
|
||||||
# get weather dataset
|
|
||||||
start = datetime(2019, 1, 1)
|
|
||||||
end = datetime(2019, 1, 14)
|
|
||||||
isd = NoaaIsdWeather(start, end)
|
|
||||||
|
|
||||||
# convert to pandas dataframe and filter down
|
|
||||||
df = isd.to_pandas_dataframe().fillna(0)
|
|
||||||
df = df[df['stationName'].str.contains('FLORIDA', regex=True, na=False)]
|
|
||||||
|
|
||||||
# features for training
|
|
||||||
X_features = ['latitude', 'longitude', 'temperature', 'windAngle', 'windSpeed']
|
|
||||||
y_features = ['elevation']
|
|
||||||
|
|
||||||
# write the training dataset to csv
|
|
||||||
training_dataset = df[X_features + y_features]
|
|
||||||
training_dataset.to_csv('training.csv', index=False)
|
|
||||||
|
|
||||||
# train the model
|
|
||||||
X = training_dataset[X_features]
|
|
||||||
y = training_dataset[y_features]
|
|
||||||
model = LinearRegression().fit(X, y)
|
|
||||||
|
|
||||||
# save the model as a .pkl file
|
|
||||||
with open('elevation-regression-model.pkl', 'wb') as f:
|
|
||||||
pickle.dump(model, f)
|
|
||||||
@@ -1,346 +0,0 @@
|
|||||||
latitude,longitude,temperature,windAngle,windSpeed,elevation
|
|
||||||
26.536,-81.755,17.8,10.0,2.1,9.0
|
|
||||||
26.536,-81.755,16.7,360.0,1.5,9.0
|
|
||||||
26.536,-81.755,16.1,350.0,1.5,9.0
|
|
||||||
26.536,-81.755,15.0,0.0,0.0,9.0
|
|
||||||
26.536,-81.755,14.4,350.0,1.5,9.0
|
|
||||||
26.536,-81.755,0.0,0.0,0.0,9.0
|
|
||||||
26.536,-81.755,13.9,360.0,2.1,9.0
|
|
||||||
26.536,-81.755,13.3,350.0,1.5,9.0
|
|
||||||
26.536,-81.755,13.3,10.0,2.1,9.0
|
|
||||||
26.536,-81.755,13.3,360.0,1.5,9.0
|
|
||||||
26.536,-81.755,13.3,0.0,0.0,9.0
|
|
||||||
26.536,-81.755,12.2,0.0,0.0,9.0
|
|
||||||
26.536,-81.755,11.7,0.0,0.0,9.0
|
|
||||||
26.536,-81.755,14.4,0.0,0.0,9.0
|
|
||||||
26.536,-81.755,17.2,10.0,2.6,9.0
|
|
||||||
26.536,-81.755,20.0,20.0,2.6,9.0
|
|
||||||
26.536,-81.755,22.2,10.0,3.6,9.0
|
|
||||||
26.536,-81.755,23.3,30.0,4.6,9.0
|
|
||||||
26.536,-81.755,23.3,330.0,2.6,9.0
|
|
||||||
26.536,-81.755,24.4,0.0,0.0,9.0
|
|
||||||
26.536,-81.755,25.0,360.0,3.1,9.0
|
|
||||||
26.536,-81.755,24.4,20.0,4.1,9.0
|
|
||||||
26.536,-81.755,23.3,10.0,2.6,9.0
|
|
||||||
26.536,-81.755,21.1,30.0,2.1,9.0
|
|
||||||
26.536,-81.755,18.3,0.0,0.0,9.0
|
|
||||||
26.536,-81.755,17.2,30.0,2.1,9.0
|
|
||||||
26.536,-81.755,15.6,60.0,2.6,9.0
|
|
||||||
26.536,-81.755,15.6,0.0,0.0,9.0
|
|
||||||
26.536,-81.755,13.9,60.0,2.6,9.0
|
|
||||||
26.536,-81.755,12.8,70.0,2.6,9.0
|
|
||||||
26.536,-81.755,0.0,0.0,0.0,9.0
|
|
||||||
26.536,-81.755,11.7,70.0,2.1,9.0
|
|
||||||
26.536,-81.755,12.2,20.0,2.1,9.0
|
|
||||||
26.536,-81.755,11.7,30.0,1.5,9.0
|
|
||||||
26.536,-81.755,11.1,40.0,2.1,9.0
|
|
||||||
26.536,-81.755,12.2,40.0,2.6,9.0
|
|
||||||
26.536,-81.755,12.2,30.0,2.6,9.0
|
|
||||||
26.536,-81.755,12.2,0.0,0.0,9.0
|
|
||||||
26.536,-81.755,15.0,30.0,6.2,9.0
|
|
||||||
26.536,-81.755,17.2,50.0,3.6,9.0
|
|
||||||
26.536,-81.755,20.6,60.0,5.1,9.0
|
|
||||||
26.536,-81.755,22.8,50.0,4.6,9.0
|
|
||||||
26.536,-81.755,24.4,80.0,6.2,9.0
|
|
||||||
26.536,-81.755,25.0,100.0,5.7,9.0
|
|
||||||
26.536,-81.755,25.6,60.0,3.1,9.0
|
|
||||||
26.536,-81.755,25.6,80.0,4.6,9.0
|
|
||||||
26.536,-81.755,25.0,90.0,5.1,9.0
|
|
||||||
26.536,-81.755,24.4,80.0,5.1,9.0
|
|
||||||
26.536,-81.755,21.1,60.0,2.6,9.0
|
|
||||||
26.536,-81.755,19.4,70.0,3.6,9.0
|
|
||||||
26.536,-81.755,18.3,70.0,2.6,9.0
|
|
||||||
26.536,-81.755,18.3,80.0,2.6,9.0
|
|
||||||
26.536,-81.755,17.2,60.0,1.5,9.0
|
|
||||||
26.536,-81.755,16.1,70.0,2.6,9.0
|
|
||||||
26.536,-81.755,15.6,70.0,2.6,9.0
|
|
||||||
26.536,-81.755,0.0,0.0,0.0,9.0
|
|
||||||
26.536,-81.755,16.1,50.0,2.6,9.0
|
|
||||||
26.536,-81.755,15.6,50.0,2.1,9.0
|
|
||||||
26.536,-81.755,15.0,50.0,1.5,9.0
|
|
||||||
26.536,-81.755,15.0,0.0,0.0,9.0
|
|
||||||
26.536,-81.755,15.0,0.0,0.0,9.0
|
|
||||||
26.536,-81.755,14.4,0.0,0.0,9.0
|
|
||||||
26.536,-81.755,14.4,30.0,4.1,9.0
|
|
||||||
26.536,-81.755,16.1,40.0,1.5,9.0
|
|
||||||
26.536,-81.755,19.4,0.0,1.5,9.0
|
|
||||||
26.536,-81.755,22.8,90.0,2.6,9.0
|
|
||||||
26.536,-81.755,24.4,130.0,3.6,9.0
|
|
||||||
26.536,-81.755,25.6,100.0,4.6,9.0
|
|
||||||
26.536,-81.755,26.1,120.0,3.1,9.0
|
|
||||||
26.536,-81.755,26.7,0.0,2.6,9.0
|
|
||||||
26.536,-81.755,27.2,0.0,0.0,9.0
|
|
||||||
26.536,-81.755,27.2,40.0,3.1,9.0
|
|
||||||
26.536,-81.755,26.1,30.0,1.5,9.0
|
|
||||||
26.536,-81.755,22.8,310.0,2.1,9.0
|
|
||||||
26.536,-81.755,23.3,330.0,2.1,9.0
|
|
||||||
-34.067,-56.238,17.5,30.0,3.1,68.0
|
|
||||||
-34.067,-56.238,21.2,30.0,5.7,68.0
|
|
||||||
-34.067,-56.238,24.5,30.0,3.1,68.0
|
|
||||||
-34.067,-56.238,27.5,330.0,3.6,68.0
|
|
||||||
-34.067,-56.238,29.2,30.0,4.1,68.0
|
|
||||||
-34.067,-56.238,31.0,20.0,4.6,68.0
|
|
||||||
-34.067,-56.238,33.0,360.0,2.6,68.0
|
|
||||||
-34.067,-56.238,33.6,60.0,3.1,68.0
|
|
||||||
-34.067,-56.238,33.6,30.0,3.6,68.0
|
|
||||||
-34.067,-56.238,18.6,40.0,3.1,68.0
|
|
||||||
-34.067,-56.238,22.0,120.0,1.5,68.0
|
|
||||||
-34.067,-56.238,25.0,120.0,2.6,68.0
|
|
||||||
-34.067,-56.238,28.6,50.0,3.1,68.0
|
|
||||||
-34.067,-56.238,30.6,50.0,4.1,68.0
|
|
||||||
-34.067,-56.238,31.5,30.0,6.7,68.0
|
|
||||||
-34.067,-56.238,32.0,40.0,7.2,68.0
|
|
||||||
-34.067,-56.238,33.0,30.0,5.7,68.0
|
|
||||||
-34.067,-56.238,33.2,360.0,3.6,68.0
|
|
||||||
-34.067,-56.238,20.6,30.0,3.1,68.0
|
|
||||||
-34.067,-56.238,21.2,0.0,0.0,68.0
|
|
||||||
-34.067,-56.238,22.0,210.0,3.1,68.0
|
|
||||||
-34.067,-56.238,23.0,210.0,3.6,68.0
|
|
||||||
-34.067,-56.238,24.0,180.0,6.7,68.0
|
|
||||||
-34.067,-56.238,24.5,210.0,7.2,68.0
|
|
||||||
-34.067,-56.238,21.0,180.0,8.2,68.0
|
|
||||||
-34.067,-56.238,20.0,180.0,6.7,68.0
|
|
||||||
-34.083,-56.233,20.2,180.0,7.2,68.0
|
|
||||||
-29.917,-71.2,16.6,290.0,4.1,146.0
|
|
||||||
-29.916,-71.2,17.0,290.0,4.1,147.0
|
|
||||||
-29.916,-71.2,16.0,310.0,3.1,147.0
|
|
||||||
-29.916,-71.2,16.0,300.0,2.1,147.0
|
|
||||||
-29.917,-71.2,15.1,0.0,0.0,146.0
|
|
||||||
-29.916,-71.2,15.0,0.0,1.0,147.0
|
|
||||||
-29.916,-71.2,15.0,160.0,1.0,147.0
|
|
||||||
-29.916,-71.2,15.0,120.0,1.0,147.0
|
|
||||||
-29.917,-71.2,14.3,190.0,1.0,146.0
|
|
||||||
-29.916,-71.2,14.0,190.0,1.0,147.0
|
|
||||||
-29.916,-71.2,14.0,0.0,0.0,147.0
|
|
||||||
-29.916,-71.2,14.0,100.0,3.1,147.0
|
|
||||||
-29.917,-71.2,12.9,0.0,0.0,146.0
|
|
||||||
-29.916,-71.2,13.0,0.0,1.0,147.0
|
|
||||||
-29.916,-71.2,14.0,0.0,0.5,147.0
|
|
||||||
-29.916,-71.2,15.0,0.0,0.5,147.0
|
|
||||||
-29.917,-71.2,15.9,0.0,0.0,146.0
|
|
||||||
-29.916,-71.2,16.0,0.0,0.0,147.0
|
|
||||||
-29.916,-71.2,17.0,270.0,4.6,147.0
|
|
||||||
-29.916,-71.2,19.0,260.0,4.1,147.0
|
|
||||||
-29.917,-71.2,18.1,270.0,6.2,146.0
|
|
||||||
-29.916,-71.2,18.0,270.0,6.2,147.0
|
|
||||||
-29.916,-71.2,19.0,270.0,6.2,147.0
|
|
||||||
-29.916,-71.2,20.0,260.0,5.1,147.0
|
|
||||||
-29.917,-71.2,19.6,280.0,6.2,146.0
|
|
||||||
-29.916,-71.2,20.0,280.0,6.2,147.0
|
|
||||||
-29.916,-71.2,20.0,270.0,6.2,147.0
|
|
||||||
-29.916,-71.2,19.0,280.0,6.7,147.0
|
|
||||||
-29.917,-71.2,18.3,270.0,5.7,146.0
|
|
||||||
-29.916,-71.2,18.0,270.0,5.7,147.0
|
|
||||||
-29.916,-71.2,18.0,0.0,0.0,147.0
|
|
||||||
-29.916,-71.2,17.0,280.0,4.6,147.0
|
|
||||||
-29.917,-71.2,15.9,280.0,4.1,146.0
|
|
||||||
-29.916,-71.2,16.0,280.0,4.1,147.0
|
|
||||||
-29.916,-71.2,15.0,280.0,3.6,147.0
|
|
||||||
-29.916,-71.2,15.0,280.0,3.6,147.0
|
|
||||||
-29.917,-71.2,15.4,280.0,4.1,146.0
|
|
||||||
-29.916,-71.2,15.0,280.0,4.1,147.0
|
|
||||||
-29.916,-71.2,16.0,240.0,2.1,147.0
|
|
||||||
-29.916,-71.2,15.0,0.0,0.5,147.0
|
|
||||||
-29.917,-71.2,15.8,80.0,3.6,146.0
|
|
||||||
-29.916,-71.2,16.0,80.0,3.6,147.0
|
|
||||||
-29.916,-71.2,16.0,10.0,1.5,147.0
|
|
||||||
-29.916,-71.2,16.0,100.0,1.5,147.0
|
|
||||||
-29.917,-71.2,15.3,130.0,1.5,146.0
|
|
||||||
-29.916,-71.2,15.0,130.0,1.5,147.0
|
|
||||||
-29.916,-71.2,15.0,110.0,1.0,147.0
|
|
||||||
-29.916,-71.2,16.0,280.0,6.2,147.0
|
|
||||||
-29.917,-71.2,15.9,240.0,3.6,146.0
|
|
||||||
-29.916,-71.2,16.0,240.0,3.6,147.0
|
|
||||||
-29.916,-71.2,16.0,240.0,3.1,147.0
|
|
||||||
-29.916,-71.2,16.0,220.0,3.1,147.0
|
|
||||||
-29.917,-71.2,16.4,260.0,3.1,146.0
|
|
||||||
-29.916,-71.2,16.0,260.0,3.1,147.0
|
|
||||||
-29.916,-71.2,17.0,230.0,2.6,147.0
|
|
||||||
-29.916,-71.2,18.0,0.0,1.5,147.0
|
|
||||||
-29.917,-71.2,20.3,340.0,2.6,146.0
|
|
||||||
-29.916,-71.2,20.0,340.0,2.6,147.0
|
|
||||||
-29.916,-71.2,21.0,270.0,5.1,147.0
|
|
||||||
-29.916,-71.2,20.0,270.0,6.7,147.0
|
|
||||||
-29.917,-71.2,19.2,280.0,6.7,146.0
|
|
||||||
-29.916,-71.2,19.0,280.0,6.7,147.0
|
|
||||||
-29.916,-71.2,19.0,310.0,2.6,147.0
|
|
||||||
-29.916,-71.2,18.0,270.0,5.1,147.0
|
|
||||||
-29.917,-71.2,17.0,300.0,4.6,146.0
|
|
||||||
-29.916,-71.2,17.0,300.0,4.6,147.0
|
|
||||||
-29.916,-71.2,17.0,300.0,3.6,147.0
|
|
||||||
-29.916,-71.2,17.0,290.0,3.1,147.0
|
|
||||||
-29.917,-71.2,16.3,290.0,2.1,146.0
|
|
||||||
-29.916,-71.2,16.0,290.0,2.1,147.0
|
|
||||||
-29.916,-71.2,17.0,270.0,1.0,147.0
|
|
||||||
-29.916,-71.2,17.0,0.0,0.5,147.0
|
|
||||||
-29.917,-71.2,16.5,160.0,2.1,146.0
|
|
||||||
-29.916,-71.2,17.0,160.0,2.1,147.0
|
|
||||||
-29.916,-71.2,15.0,120.0,3.1,147.0
|
|
||||||
-29.916,-71.2,16.0,180.0,1.5,147.0
|
|
||||||
-29.917,-71.2,14.7,0.0,0.0,146.0
|
|
||||||
-29.916,-71.2,15.0,0.0,1.0,147.0
|
|
||||||
-29.916,-71.2,15.0,300.0,1.0,147.0
|
|
||||||
-29.916,-71.2,16.0,0.0,0.0,147.0
|
|
||||||
-29.917,-71.2,18.5,110.0,1.0,146.0
|
|
||||||
-29.916,-71.2,19.0,110.0,1.0,147.0
|
|
||||||
-29.916,-71.2,20.0,270.0,3.6,147.0
|
|
||||||
-29.916,-71.2,20.0,270.0,5.7,147.0
|
|
||||||
-29.917,-71.2,20.0,280.0,6.2,146.0
|
|
||||||
-29.916,-71.2,20.0,280.0,6.2,147.0
|
|
||||||
-29.916,-71.2,21.0,290.0,6.7,147.0
|
|
||||||
-29.916,-71.2,20.0,270.0,6.2,147.0
|
|
||||||
-29.917,-71.2,21.0,260.0,6.7,146.0
|
|
||||||
-29.916,-71.2,21.0,260.0,6.7,147.0
|
|
||||||
-29.916,-71.2,20.0,270.0,6.2,147.0
|
|
||||||
-29.916,-71.2,19.0,260.0,5.1,147.0
|
|
||||||
-29.916,-71.2,18.0,280.0,4.6,147.0
|
|
||||||
-29.917,-71.2,17.5,280.0,3.1,146.0
|
|
||||||
-29.916,-71.2,18.0,280.0,3.1,147.0
|
|
||||||
30.349,-85.788,11.1,0.0,0.0,21.0
|
|
||||||
30.349,-85.788,11.1,0.0,0.0,21.0
|
|
||||||
30.349,-85.788,9.4,0.0,0.0,21.0
|
|
||||||
30.349,-85.788,9.4,0.0,0.0,21.0
|
|
||||||
30.349,-85.788,8.3,300.0,2.1,21.0
|
|
||||||
30.349,-85.788,11.1,280.0,1.5,21.0
|
|
||||||
30.349,-85.788,0.0,0.0,0.0,21.0
|
|
||||||
30.349,-85.788,10.6,320.0,3.1,21.0
|
|
||||||
30.349,-85.788,9.4,310.0,3.1,21.0
|
|
||||||
30.349,-85.788,7.8,320.0,2.6,21.0
|
|
||||||
30.349,-85.788,6.1,340.0,2.1,21.0
|
|
||||||
30.349,-85.788,6.7,330.0,2.6,21.0
|
|
||||||
30.349,-85.788,6.1,310.0,1.5,21.0
|
|
||||||
30.349,-85.788,7.2,310.0,2.1,21.0
|
|
||||||
30.349,-85.788,12.8,360.0,3.1,21.0
|
|
||||||
30.349,-85.788,15.0,0.0,3.1,21.0
|
|
||||||
30.349,-85.788,16.7,20.0,4.6,21.0
|
|
||||||
30.349,-85.788,18.9,30.0,5.1,21.0
|
|
||||||
30.349,-85.788,19.4,10.0,4.1,21.0
|
|
||||||
30.349,-85.788,21.1,330.0,2.6,21.0
|
|
||||||
30.349,-85.788,21.1,10.0,4.6,21.0
|
|
||||||
30.349,-85.788,21.7,360.0,4.1,21.0
|
|
||||||
30.349,-85.788,21.7,30.0,2.1,21.0
|
|
||||||
30.349,-85.788,21.7,330.0,2.6,21.0
|
|
||||||
30.349,-85.788,16.1,350.0,2.1,21.0
|
|
||||||
30.349,-85.788,11.7,0.0,0.0,21.0
|
|
||||||
30.349,-85.788,8.9,0.0,0.0,21.0
|
|
||||||
30.349,-85.788,9.4,0.0,0.0,21.0
|
|
||||||
30.349,-85.788,7.8,0.0,0.0,21.0
|
|
||||||
30.349,-85.788,11.1,30.0,3.1,21.0
|
|
||||||
30.349,-85.788,7.2,0.0,0.0,21.0
|
|
||||||
30.349,-85.788,7.2,0.0,0.0,21.0
|
|
||||||
30.349,-85.788,0.0,0.0,0.0,21.0
|
|
||||||
30.349,-85.788,7.8,30.0,2.1,21.0
|
|
||||||
30.349,-85.788,8.3,40.0,2.6,21.0
|
|
||||||
30.349,-85.788,7.2,50.0,1.5,21.0
|
|
||||||
30.349,-85.788,8.3,60.0,1.5,21.0
|
|
||||||
30.349,-85.788,5.6,40.0,2.1,21.0
|
|
||||||
30.349,-85.788,6.7,40.0,2.1,21.0
|
|
||||||
30.349,-85.788,7.8,50.0,3.1,21.0
|
|
||||||
30.349,-85.788,11.7,70.0,2.6,21.0
|
|
||||||
30.349,-85.788,15.6,70.0,3.1,21.0
|
|
||||||
30.349,-85.788,18.9,100.0,3.6,21.0
|
|
||||||
30.349,-85.788,20.0,130.0,3.6,21.0
|
|
||||||
30.349,-85.788,21.1,140.0,4.1,21.0
|
|
||||||
30.349,-85.788,21.7,150.0,4.1,21.0
|
|
||||||
30.349,-85.788,21.7,170.0,3.1,21.0
|
|
||||||
30.349,-85.788,22.2,170.0,3.1,21.0
|
|
||||||
30.349,-85.788,20.6,0.0,0.0,21.0
|
|
||||||
30.349,-85.788,17.2,0.0,0.0,21.0
|
|
||||||
30.349,-85.788,14.4,0.0,0.0,21.0
|
|
||||||
30.349,-85.788,12.8,100.0,1.5,21.0
|
|
||||||
30.349,-85.788,13.3,100.0,1.5,21.0
|
|
||||||
30.349,-85.788,10.6,0.0,0.0,21.0
|
|
||||||
30.349,-85.788,9.4,0.0,0.0,21.0
|
|
||||||
30.349,-85.788,7.8,0.0,0.0,21.0
|
|
||||||
30.358,-85.799,8.3,0.0,0.0,21.0
|
|
||||||
30.349,-85.788,0.0,0.0,0.0,21.0
|
|
||||||
30.358,-85.799,6.7,0.0,0.0,21.0
|
|
||||||
30.358,-85.799,7.2,0.0,0.0,21.0
|
|
||||||
30.358,-85.799,7.2,0.0,0.0,21.0
|
|
||||||
30.358,-85.799,8.3,50.0,1.5,21.0
|
|
||||||
30.358,-85.799,9.4,0.0,0.0,21.0
|
|
||||||
30.358,-85.799,8.9,0.0,0.0,21.0
|
|
||||||
30.358,-85.799,10.0,340.0,1.5,21.0
|
|
||||||
30.358,-85.799,12.8,40.0,1.5,21.0
|
|
||||||
30.358,-85.799,16.7,100.0,2.1,21.0
|
|
||||||
30.358,-85.799,21.1,100.0,1.5,21.0
|
|
||||||
30.358,-85.799,23.3,0.0,0.0,21.0
|
|
||||||
30.358,-85.799,25.0,180.0,4.6,21.0
|
|
||||||
30.358,-85.799,24.4,230.0,3.6,21.0
|
|
||||||
30.358,-85.799,25.0,210.0,4.1,21.0
|
|
||||||
30.358,-85.799,23.9,170.0,4.1,21.0
|
|
||||||
30.358,-85.799,22.8,0.0,0.0,21.0
|
|
||||||
30.358,-85.799,19.4,0.0,0.0,21.0
|
|
||||||
30.358,-85.799,17.8,140.0,2.1,21.0
|
|
||||||
60.383,5.333,-0.7,0.0,0.0,36.0
|
|
||||||
60.383,5.333,0.6,270.0,2.0,36.0
|
|
||||||
60.383,5.333,-0.9,120.0,1.0,36.0
|
|
||||||
60.383,5.333,-1.6,130.0,2.0,36.0
|
|
||||||
60.383,5.333,-1.4,150.0,1.0,36.0
|
|
||||||
60.383,5.333,-1.7,0.0,0.0,36.0
|
|
||||||
60.383,5.333,-1.7,140.0,1.0,36.0
|
|
||||||
60.383,5.333,-1.4,0.0,0.0,36.0
|
|
||||||
60.383,5.333,-1.0,0.0,0.0,36.0
|
|
||||||
60.383,5.333,-1.0,150.0,1.0,36.0
|
|
||||||
60.383,5.333,-0.7,140.0,1.0,36.0
|
|
||||||
60.383,5.333,0.5,150.0,1.0,36.0
|
|
||||||
60.383,5.333,1.9,0.0,0.0,36.0
|
|
||||||
60.383,5.333,1.7,0.0,0.0,36.0
|
|
||||||
60.383,5.333,2.1,310.0,2.0,36.0
|
|
||||||
60.383,5.333,1.5,90.0,1.0,36.0
|
|
||||||
60.383,5.333,1.9,290.0,1.0,36.0
|
|
||||||
60.383,5.333,2.0,320.0,1.0,36.0
|
|
||||||
60.383,5.333,1.9,330.0,1.0,36.0
|
|
||||||
60.383,5.333,1.3,350.0,1.0,36.0
|
|
||||||
60.383,5.333,1.5,120.0,1.0,36.0
|
|
||||||
60.383,5.333,1.3,150.0,2.0,36.0
|
|
||||||
60.383,5.333,0.8,140.0,1.0,36.0
|
|
||||||
60.383,5.333,0.3,300.0,1.0,36.0
|
|
||||||
60.383,5.333,0.2,140.0,1.0,36.0
|
|
||||||
60.383,5.333,0.4,140.0,1.0,36.0
|
|
||||||
60.383,5.333,0.5,320.0,1.0,36.0
|
|
||||||
60.383,5.333,1.5,330.0,1.0,36.0
|
|
||||||
60.383,5.333,1.8,40.0,1.0,36.0
|
|
||||||
60.383,5.333,2.3,170.0,1.0,36.0
|
|
||||||
60.383,5.333,2.7,140.0,1.0,36.0
|
|
||||||
60.383,5.333,3.1,330.0,1.0,36.0
|
|
||||||
60.383,5.333,3.8,350.0,1.0,36.0
|
|
||||||
60.383,5.333,3.8,140.0,1.0,36.0
|
|
||||||
60.383,5.333,4.1,150.0,1.0,36.0
|
|
||||||
60.383,5.333,4.4,180.0,1.0,36.0
|
|
||||||
60.383,5.333,4.9,300.0,1.0,36.0
|
|
||||||
60.383,5.333,5.2,320.0,1.0,36.0
|
|
||||||
60.383,5.333,6.7,340.0,1.0,36.0
|
|
||||||
60.383,5.333,6.9,250.0,1.0,36.0
|
|
||||||
60.383,5.333,7.9,300.0,2.0,36.0
|
|
||||||
60.383,5.333,5.5,140.0,1.0,36.0
|
|
||||||
60.383,5.333,7.1,140.0,2.0,36.0
|
|
||||||
60.383,5.333,7.0,280.0,2.0,36.0
|
|
||||||
60.383,5.333,4.6,170.0,1.0,36.0
|
|
||||||
60.383,5.333,4.8,330.0,1.0,36.0
|
|
||||||
60.383,5.333,6.4,260.0,2.0,36.0
|
|
||||||
60.383,5.333,6.2,340.0,1.0,36.0
|
|
||||||
60.383,5.333,5.7,320.0,2.0,36.0
|
|
||||||
60.383,5.333,5.2,100.0,1.0,36.0
|
|
||||||
60.383,5.333,5.1,310.0,1.0,36.0
|
|
||||||
60.383,5.333,4.9,290.0,2.0,36.0
|
|
||||||
60.383,5.333,4.9,310.0,2.0,36.0
|
|
||||||
60.383,5.333,6.1,320.0,2.0,36.0
|
|
||||||
60.383,5.333,7.0,250.0,1.0,36.0
|
|
||||||
60.383,5.333,5.3,140.0,1.0,36.0
|
|
||||||
60.383,5.333,6.9,350.0,1.0,36.0
|
|
||||||
60.383,5.333,9.7,110.0,3.0,36.0
|
|
||||||
60.383,5.333,10.3,300.0,3.0,36.0
|
|
||||||
60.383,5.333,8.7,310.0,1.0,36.0
|
|
||||||
60.383,5.333,9.0,270.0,3.0,36.0
|
|
||||||
60.383,5.333,11.6,80.0,3.0,36.0
|
|
||||||
60.383,5.333,11.4,80.0,4.0,36.0
|
|
||||||
60.383,5.333,9.7,70.0,5.0,36.0
|
|
||||||
60.383,5.333,9.5,80.0,6.0,36.0
|
|
||||||
60.383,5.333,8.7,80.0,5.0,36.0
|
|
||||||
60.383,5.333,7.7,80.0,5.0,36.0
|
|
||||||
60.383,5.333,8.2,80.0,4.0,36.0
|
|
||||||
60.383,5.333,7.7,30.0,1.0,36.0
|
|
||||||
60.383,5.333,7.2,310.0,1.0,36.0
|
|
||||||
60.383,5.333,6.8,300.0,2.0,36.0
|
|
||||||
60.383,5.333,6.7,140.0,1.0,36.0
|
|
||||||
|
File diff suppressed because it is too large
Load Diff
@@ -1,547 +0,0 @@
|
|||||||
{
|
|
||||||
"cells": [
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"Copyright (c) Microsoft Corporation. All rights reserved.\n",
|
|
||||||
"\n",
|
|
||||||
"Licensed under the MIT License."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
""
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"# Monitor data drift on models deployed to Azure Kubernetes Service \n",
|
|
||||||
"\n",
|
|
||||||
"In this tutorial, you will setup a data drift monitor on a toy model that predicts elevation based on a few weather factors which will send email alerts if drift is detected."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Prerequisites\n",
|
|
||||||
"If you are using an Azure Machine Learning Compute instance, you are all set. Otherwise, go through the [configuration notebook](../../../configuration.ipynb) first if you haven't already established your connection to the AzureML Workspace."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# Check core SDK version number\n",
|
|
||||||
"import azureml.core\n",
|
|
||||||
"\n",
|
|
||||||
"print('SDK version:', azureml.core.VERSION)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Initialize Workspace\n",
|
|
||||||
"\n",
|
|
||||||
"Initialize a workspace object from persisted configuration."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"from azureml.core import Workspace\n",
|
|
||||||
"\n",
|
|
||||||
"ws = Workspace.from_config()\n",
|
|
||||||
"ws"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Setup training dataset and model\n",
|
|
||||||
"\n",
|
|
||||||
"Setup the training dataset and model in preparation for deployment to the Azure Kubernetes Service. \n",
|
|
||||||
"\n",
|
|
||||||
"The next few cells will:\n",
|
|
||||||
" * get the default datastore and upload the `training.csv` dataset to the datastore\n",
|
|
||||||
" * create and register the dataset \n",
|
|
||||||
" * register the model with the dataset\n",
|
|
||||||
" \n",
|
|
||||||
"See the `config.py` script in this folder for details on how `training.csv` and `elevation-regression-model.pkl` are created. If you train your model in Azure ML using a Dataset, it will be automatically captured when registering the model from the run. "
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# use default datastore\n",
|
|
||||||
"dstore = ws.get_default_datastore()\n",
|
|
||||||
"\n",
|
|
||||||
"# upload weather data\n",
|
|
||||||
"dstore.upload('dataset', 'drift-on-aks-data', overwrite=True, show_progress=False)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"from azureml.core import Dataset\n",
|
|
||||||
"\n",
|
|
||||||
"# create dataset \n",
|
|
||||||
"dset = Dataset.Tabular.from_delimited_files(dstore.path('drift-on-aks-data/training.csv'))\n",
|
|
||||||
"# register dataset\n",
|
|
||||||
"dset = dset.register(ws, 'drift-demo-dataset')\n",
|
|
||||||
"# get the dataset by name from the workspace\n",
|
|
||||||
"dset = Dataset.get_by_name(ws, 'drift-demo-dataset')"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"from azureml.core.model import Model\n",
|
|
||||||
"\n",
|
|
||||||
"# register the model\n",
|
|
||||||
"model = Model.register(model_path='elevation-regression-model.pkl',\n",
|
|
||||||
" model_name='elevation-regression-model.pkl',\n",
|
|
||||||
" tags={'area': \"weather\", 'type': \"linear regression\"},\n",
|
|
||||||
" description='Linear regression model to predict elevation based on the weather',\n",
|
|
||||||
" workspace=ws,\n",
|
|
||||||
" datasets=[(Dataset.Scenario.TRAINING, dset)]) # need to register the dataset with the model"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Create the inference config\n",
|
|
||||||
"\n",
|
|
||||||
"Create the environment and inference config from the `myenv.yml` and `score.py` files. Notice the [Model Data Collector](https://docs.microsoft.com/azure/machine-learning/service/how-to-enable-data-collection) code included in the scoring script. This dependency is currently required to collect model data, but will be removed in the near future as data collection in Azure Machine Learning webservice endpoints is automated."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"from azureml.core import Environment\n",
|
|
||||||
"\n",
|
|
||||||
"# create the environment from the yml file \n",
|
|
||||||
"env = Environment.from_conda_specification(name='deploytocloudenv', file_path='myenv.yml')"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"from azureml.core.model import InferenceConfig\n",
|
|
||||||
"\n",
|
|
||||||
"# create an inference config, combining the environment and entry script \n",
|
|
||||||
"inference_config = InferenceConfig(entry_script='score.py', environment=env)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Create the AKS compute target\n",
|
|
||||||
"\n",
|
|
||||||
"Create an Azure Kubernetes Service compute target to deploy the model to. "
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"from azureml.core.compute import AksCompute, ComputeTarget\n",
|
|
||||||
"\n",
|
|
||||||
"# Use the default configuration (you can also provide parameters to customize this).\n",
|
|
||||||
"# For example, to create a dev/test cluster, use:\n",
|
|
||||||
"# prov_config = AksCompute.provisioning_configuration(cluster_purpose = AksCompute.ClusterPurpose.DEV_TEST)\n",
|
|
||||||
"prov_config = AksCompute.provisioning_configuration()\n",
|
|
||||||
"\n",
|
|
||||||
"aks_name = 'drift-aks'\n",
|
|
||||||
"aks_target = ws.compute_targets.get(aks_name)\n",
|
|
||||||
"\n",
|
|
||||||
"# Create the cluster\n",
|
|
||||||
"if not aks_target:\n",
|
|
||||||
" aks_target = ComputeTarget.create(workspace = ws,\n",
|
|
||||||
" name = aks_name,\n",
|
|
||||||
" provisioning_configuration = prov_config)\n",
|
|
||||||
"\n",
|
|
||||||
" # Wait for the create process to complete\n",
|
|
||||||
" aks_target.wait_for_completion(show_output = True)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Deploy the model to AKS \n",
|
|
||||||
"\n",
|
|
||||||
"Deploy the model as a webservice endpoint. Be sure to enable the `collect_model_data` flag so that serving data is collected in blob storage for use by the data drift capability."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"from azureml.core.webservice import AksWebservice\n",
|
|
||||||
"\n",
|
|
||||||
"deployment_config = AksWebservice.deploy_configuration(cpu_cores=1, memory_gb=1, collect_model_data=True)\n",
|
|
||||||
"service_name = 'drift-aks-service'\n",
|
|
||||||
"\n",
|
|
||||||
"service = Model.deploy(ws, service_name, [model], inference_config, deployment_config, aks_target)\n",
|
|
||||||
"\n",
|
|
||||||
"service.wait_for_deployment(True)\n",
|
|
||||||
"print(service.state)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Run recent weather data through the webservice \n",
|
|
||||||
"\n",
|
|
||||||
"The below cells take the weather data of Florida from 2019-11-20 to 2019-11-12, filter and transform using the same processes as the training dataset, and runs the data through the service."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# create dataset \n",
|
|
||||||
"tset = Dataset.Tabular.from_delimited_files(dstore.path('drift-on-aks-data/testing.csv'))\n",
|
|
||||||
"\n",
|
|
||||||
"df = tset.to_pandas_dataframe().fillna(0)\n",
|
|
||||||
"\n",
|
|
||||||
"X_features = ['latitude', 'longitude', 'temperature', 'windAngle', 'windSpeed']\n",
|
|
||||||
"y_features = ['elevation']\n",
|
|
||||||
"\n",
|
|
||||||
"X = df[X_features]\n",
|
|
||||||
"y = df[y_features]"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"import json\n",
|
|
||||||
"\n",
|
|
||||||
"data = json.dumps({'data': X.values.tolist()})\n",
|
|
||||||
"\n",
|
|
||||||
"data_encoded = bytes(data, encoding='utf8')\n",
|
|
||||||
"prediction = service.run(input_data=data_encoded)\n",
|
|
||||||
"print(prediction)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Create an Azure Machine Learning Compute cluster\n",
|
|
||||||
"\n",
|
|
||||||
"The data drift capability needs a compute target for computing drift and other data metrics. "
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"from azureml.core.compute import AmlCompute\n",
|
|
||||||
"\n",
|
|
||||||
"compute_name = 'cpu-cluster'\n",
|
|
||||||
"\n",
|
|
||||||
"if compute_name in ws.compute_targets:\n",
|
|
||||||
" compute_target = ws.compute_targets[compute_name]\n",
|
|
||||||
" if compute_target and type(compute_target) is AmlCompute:\n",
|
|
||||||
" print('found compute target. just use it. ' + compute_name)\n",
|
|
||||||
"else:\n",
|
|
||||||
" print('creating a new compute target...')\n",
|
|
||||||
" provisioning_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_D3_V2', min_nodes=0, max_nodes=2)\n",
|
|
||||||
"\n",
|
|
||||||
" # create the cluster\n",
|
|
||||||
" compute_target = ComputeTarget.create(ws, compute_name, provisioning_config)\n",
|
|
||||||
"\n",
|
|
||||||
" # can poll for a minimum number of nodes and for a specific timeout.\n",
|
|
||||||
" # if no min node count is provided it will use the scale settings for the cluster\n",
|
|
||||||
" compute_target.wait_for_completion(show_output=True, min_node_count=None, timeout_in_minutes=20)\n",
|
|
||||||
"\n",
|
|
||||||
" # For a more detailed view of current AmlCompute status, use get_status()\n",
|
|
||||||
" print(compute_target.get_status().serialize())"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Wait 10 minutes\n",
|
|
||||||
"\n",
|
|
||||||
"From the Model Data Collector, it can take up to (but usually less than) 10 minutes for data to arrive in your blob storage account. Wait 10 minutes to ensure cells below will run."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"import time\n",
|
|
||||||
"\n",
|
|
||||||
"time.sleep(600)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Create and update the data drift object"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"from datetime import datetime, timedelta\n",
|
|
||||||
"from azureml.datadrift import DataDriftDetector, AlertConfiguration\n",
|
|
||||||
"\n",
|
|
||||||
"services = [service_name]\n",
|
|
||||||
"start = datetime.now() - timedelta(days=2)\n",
|
|
||||||
"feature_list = X_features\n",
|
|
||||||
"alert_config = AlertConfiguration(['user@contoso.com'])\n",
|
|
||||||
"\n",
|
|
||||||
"try:\n",
|
|
||||||
" monitor = DataDriftDetector.create_from_model(ws, model.name, model.version, services, \n",
|
|
||||||
" frequency='Day', \n",
|
|
||||||
" schedule_start=datetime.utcnow() + timedelta(days=1), \n",
|
|
||||||
" alert_config=alert_config, \n",
|
|
||||||
" compute_target='cpu-cluster')\n",
|
|
||||||
"except KeyError:\n",
|
|
||||||
" monitor = DataDriftDetector.get(ws, model.name, model.version)\n",
|
|
||||||
" \n",
|
|
||||||
"monitor"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# many monitor settings can be updated \n",
|
|
||||||
"monitor = monitor.update(drift_threshold = 0.1)\n",
|
|
||||||
"\n",
|
|
||||||
"monitor"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Run the monitor on today's scoring data\n",
|
|
||||||
"\n",
|
|
||||||
"Perform a data drift run on the data sent to the service earlier in this notebook. If you set your email address in the alert configuration and the drift threshold <=0.1 you should recieve an email alert to drift from this run.\n",
|
|
||||||
"\n",
|
|
||||||
"Wait for the run to complete before getting the results. "
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"now = datetime.utcnow()\n",
|
|
||||||
"target_date = datetime(now.year, now.month, now.day)\n",
|
|
||||||
"run = monitor.run(target_date, services, feature_list=feature_list, compute_target='cpu-cluster')"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Get and view results and metrics\n",
|
|
||||||
"\n",
|
|
||||||
"For enterprise workspaces, the UI in the Azure Machine Learning studio can be used. Otherwise, the metrics can be queried in Python and plotted. "
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# The run() API initiates a pipeline run for each service in the services list. \n",
|
|
||||||
"# Here we retrieve the individual service run to get its output results and metrics. \n",
|
|
||||||
"\n",
|
|
||||||
"child_run = list(run.get_children())[0]\n",
|
|
||||||
"child_run"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"child_run.wait_for_completion(wait_post_processing=True)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"results, metrics = monitor.get_output(run_id=child_run.id)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"drift_plots = monitor.show()"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Enable the monitor's pipeline schedule\n",
|
|
||||||
"\n",
|
|
||||||
"Turn on a scheduled pipeline which will anlayze the serving dataset for drift. "
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"monitor.enable_schedule()"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Delete the DataDriftDetector\n",
|
|
||||||
"\n",
|
|
||||||
"Invoking the `delete()` method on the object deletes the the drift monitor permanently and cannot be undone. You will no longer be able to find it in the UI and the `list()` or `get()` methods. The object on which delete() was called will have its state set to deleted and name suffixed with deleted. The baseline and target datasets and model data that was collected, if any, are not deleted. The compute is not deleted. The DataDrift schedule pipeline is disabled and archived."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"monitor.delete()"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Next steps\n",
|
|
||||||
"\n",
|
|
||||||
" * See [our documentation](https://aka.ms/datadrift/aks) or [Python SDK reference](https://docs.microsoft.com/python/api/overview/azure/ml/intro)\n",
|
|
||||||
" * [Send requests or feedback](mailto:driftfeedback@microsoft.com) on data drift directly to the team\n",
|
|
||||||
" * Please open issues with data drift here on GitHub or on StackOverflow if others are likely to run into the same issue"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": []
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"authors": [
|
|
||||||
{
|
|
||||||
"name": "jamgan"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"category": "tutorial",
|
|
||||||
"compute": [
|
|
||||||
"Remote"
|
|
||||||
],
|
|
||||||
"datasets": [
|
|
||||||
"NOAA"
|
|
||||||
],
|
|
||||||
"deployment": [
|
|
||||||
"AKS"
|
|
||||||
],
|
|
||||||
"exclude_from_index": false,
|
|
||||||
"framework": [
|
|
||||||
"Azure ML"
|
|
||||||
],
|
|
||||||
"friendly_name": "Data drift on aks",
|
|
||||||
"index_order": 1.0,
|
|
||||||
"kernelspec": {
|
|
||||||
"display_name": "Python 3.6",
|
|
||||||
"language": "python",
|
|
||||||
"name": "python36"
|
|
||||||
},
|
|
||||||
"language_info": {
|
|
||||||
"codemirror_mode": {
|
|
||||||
"name": "ipython",
|
|
||||||
"version": 3
|
|
||||||
},
|
|
||||||
"file_extension": ".py",
|
|
||||||
"mimetype": "text/x-python",
|
|
||||||
"name": "python",
|
|
||||||
"nbconvert_exporter": "python",
|
|
||||||
"pygments_lexer": "ipython3",
|
|
||||||
"version": "3.7.4"
|
|
||||||
},
|
|
||||||
"star_tag": [
|
|
||||||
"featured"
|
|
||||||
],
|
|
||||||
"tags": [
|
|
||||||
"Dataset",
|
|
||||||
"Timeseries",
|
|
||||||
"Drift"
|
|
||||||
],
|
|
||||||
"task": "Filtering"
|
|
||||||
},
|
|
||||||
"nbformat": 4,
|
|
||||||
"nbformat_minor": 4
|
|
||||||
}
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
name: drift-on-aks
|
|
||||||
dependencies:
|
|
||||||
- pip:
|
|
||||||
- azureml-sdk
|
|
||||||
- azureml-datadrift
|
|
||||||
- azureml-monitoring
|
|
||||||
- azureml-opendatasets
|
|
||||||
- azureml-widgets
|
|
||||||
Binary file not shown.
@@ -1,11 +0,0 @@
|
|||||||
name: project_environment
|
|
||||||
dependencies:
|
|
||||||
- python=3.6.2
|
|
||||||
- pip:
|
|
||||||
- azureml-core
|
|
||||||
- azureml-defaults
|
|
||||||
- azureml-monitoring
|
|
||||||
- scikit-learn
|
|
||||||
- numpy
|
|
||||||
- packaging
|
|
||||||
- inference-schema[numpy-support]
|
|
||||||
@@ -1,44 +0,0 @@
|
|||||||
import os
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
from azureml.monitoring import ModelDataCollector
|
|
||||||
from inference_schema.parameter_types.numpy_parameter_type import NumpyParameterType
|
|
||||||
from inference_schema.schema_decorators import input_schema, output_schema
|
|
||||||
# sklearn.externals.joblib is removed in 0.23
|
|
||||||
from sklearn import __version__ as sklearnver
|
|
||||||
from packaging.version import Version
|
|
||||||
if Version(sklearnver) < Version("0.23.0"):
|
|
||||||
from sklearn.externals import joblib
|
|
||||||
else:
|
|
||||||
import joblib
|
|
||||||
|
|
||||||
|
|
||||||
def init():
|
|
||||||
global model
|
|
||||||
global inputs_dc
|
|
||||||
inputs_dc = ModelDataCollector('elevation-regression-model.pkl', designation='inputs',
|
|
||||||
feature_names=['latitude', 'longitude', 'temperature', 'windAngle', 'windSpeed'])
|
|
||||||
# note here "elevation-regression-model.pkl" is the name of the model registered under
|
|
||||||
# this is a different behavior than before when the code is run locally, even though the code is the same.
|
|
||||||
# AZUREML_MODEL_DIR is an environment variable created during deployment.
|
|
||||||
# It is the path to the model folder (./azureml-models/$MODEL_NAME/$VERSION)
|
|
||||||
# For multiple models, it points to the folder containing all deployed models (./azureml-models)
|
|
||||||
model_path = os.path.join(os.getenv('AZUREML_MODEL_DIR'), 'elevation-regression-model.pkl')
|
|
||||||
model = joblib.load(model_path)
|
|
||||||
|
|
||||||
|
|
||||||
input_sample = np.array([[30, -85, 21, 150, 6]])
|
|
||||||
output_sample = np.array([8.995])
|
|
||||||
|
|
||||||
|
|
||||||
@input_schema('data', NumpyParameterType(input_sample))
|
|
||||||
@output_schema(NumpyParameterType(output_sample))
|
|
||||||
def run(data):
|
|
||||||
try:
|
|
||||||
inputs_dc.collect(data)
|
|
||||||
result = model.predict(data)
|
|
||||||
# you can return any datatype as long as it is JSON-serializable
|
|
||||||
return result.tolist()
|
|
||||||
except Exception as e:
|
|
||||||
error = str(e)
|
|
||||||
return error
|
|
||||||
@@ -582,7 +582,7 @@
|
|||||||
" rl_framework=Ray(),\n",
|
" rl_framework=Ray(),\n",
|
||||||
" \n",
|
" \n",
|
||||||
" # Additional pip packages to install\n",
|
" # Additional pip packages to install\n",
|
||||||
" pip_packages = ['azureml-dataprep[fuse,pandas]'])"
|
" pip_packages = ['azureml-dataset-runtime[fuse,pandas]'])"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -254,6 +254,7 @@
|
|||||||
" dockerfile=f.read()\n",
|
" dockerfile=f.read()\n",
|
||||||
"\n",
|
"\n",
|
||||||
" xvfb_env = Environment(name='xvfb-vdisplay')\n",
|
" xvfb_env = Environment(name='xvfb-vdisplay')\n",
|
||||||
|
" xvfb_env.docker.enabled = True\n",
|
||||||
" xvfb_env.docker.base_image = None\n",
|
" xvfb_env.docker.base_image = None\n",
|
||||||
" xvfb_env.docker.base_dockerfile = dockerfile\n",
|
" xvfb_env.docker.base_dockerfile = dockerfile\n",
|
||||||
" \n",
|
" \n",
|
||||||
@@ -547,29 +548,18 @@
|
|||||||
"source": [
|
"source": [
|
||||||
"import shutil\n",
|
"import shutil\n",
|
||||||
"\n",
|
"\n",
|
||||||
"# A helper function to download (copy) movies from a dataset to local directory\n",
|
"# A helper function to download movies from a dataset to local directory\n",
|
||||||
"def download_movies(artifacts_ds, movies, destination):\n",
|
"def download_movies(artifacts_ds, movies, destination):\n",
|
||||||
" # Create the local destination directory \n",
|
" # Create the local destination directory \n",
|
||||||
" if path.exists(destination):\n",
|
" if path.exists(destination):\n",
|
||||||
" dir_util.remove_tree(destination)\n",
|
" dir_util.remove_tree(destination)\n",
|
||||||
" dir_util.mkpath(destination)\n",
|
" dir_util.mkpath(destination)\n",
|
||||||
" \n",
|
"\n",
|
||||||
" try:\n",
|
" for i, artifact in enumerate(artifacts_ds.to_path()):\n",
|
||||||
" print(\"Trying mounting dataset and copying movies.\")\n",
|
" if artifact in movies:\n",
|
||||||
" # Note: We assume movie paths start with '\\'\n",
|
" print('Downloading {} ...'.format(artifact))\n",
|
||||||
" mount_context = artifacts_ds.mount()\n",
|
" artifacts_ds.skip(i).take(1).download(target_path=destination, overwrite=True)\n",
|
||||||
" mount_context.start()\n",
|
"\n",
|
||||||
" for movie in movies:\n",
|
|
||||||
" print('Copying {} ...'.format(movie))\n",
|
|
||||||
" shutil.copy2(path.join(mount_context.mount_point, movie[1:]), destination)\n",
|
|
||||||
" mount_context.stop()\n",
|
|
||||||
" except OSError as e:\n",
|
|
||||||
" print(\"Mounting failed with error '{0}'. Going with dataset download.\".format(e))\n",
|
|
||||||
" for i, artifact in enumerate(artifacts_ds.to_path()):\n",
|
|
||||||
" if artifact in movies:\n",
|
|
||||||
" print('Downloading {} ...'.format(artifact))\n",
|
|
||||||
" artifacts_ds.skip(i).take(1).download(target_path=destination, overwrite=True)\n",
|
|
||||||
" \n",
|
|
||||||
" print('Downloading movies completed!')\n",
|
" print('Downloading movies completed!')\n",
|
||||||
"\n",
|
"\n",
|
||||||
"\n",
|
"\n",
|
||||||
@@ -782,6 +772,7 @@
|
|||||||
" dockerfile=f.read()\n",
|
" dockerfile=f.read()\n",
|
||||||
"\n",
|
"\n",
|
||||||
"xvfb_env = Environment(name='xvfb-vdisplay')\n",
|
"xvfb_env = Environment(name='xvfb-vdisplay')\n",
|
||||||
|
"xvfb_env.docker.enabled = True\n",
|
||||||
"xvfb_env.docker.base_image = None\n",
|
"xvfb_env.docker.base_image = None\n",
|
||||||
"xvfb_env.docker.base_dockerfile = dockerfile\n",
|
"xvfb_env.docker.base_dockerfile = dockerfile\n",
|
||||||
" \n",
|
" \n",
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
FROM mcr.microsoft.com/azureml/base:openmpi3.1.2-ubuntu18.04
|
FROM mcr.microsoft.com/azureml/openmpi3.1.2-ubuntu18.04:20200423.v1
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
python-opengl \
|
python-opengl \
|
||||||
@@ -8,10 +8,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
|||||||
rm -rf /var/lib/apt/lists/* && \
|
rm -rf /var/lib/apt/lists/* && \
|
||||||
rm -rf /usr/share/man/*
|
rm -rf /usr/share/man/*
|
||||||
|
|
||||||
RUN conda install -y conda=4.7.12 python=3.6.2 && conda clean -ay && \
|
RUN conda install -y conda=4.7.12 python=3.7 && conda clean -ay && \
|
||||||
pip install --no-cache-dir \
|
pip install --no-cache-dir \
|
||||||
azureml-defaults \
|
azureml-defaults \
|
||||||
azureml-dataprep[fuse,pandas] \
|
azureml-dataset-runtime[fuse,pandas] \
|
||||||
azureml-contrib-reinforcementlearning \
|
azureml-contrib-reinforcementlearning \
|
||||||
gputil \
|
gputil \
|
||||||
cloudpickle==1.3.0 \
|
cloudpickle==1.3.0 \
|
||||||
@@ -26,4 +26,4 @@ RUN conda install -y conda=4.7.12 python=3.6.2 && conda clean -ay && \
|
|||||||
setproctitle \
|
setproctitle \
|
||||||
gym[atari] && \
|
gym[atari] && \
|
||||||
conda install -y -c conda-forge x264='1!152.20180717' ffmpeg=4.0.2 && \
|
conda install -y -c conda-forge x264='1!152.20180717' ffmpeg=4.0.2 && \
|
||||||
conda install opencv
|
conda install -c anaconda opencv
|
||||||
|
|||||||
@@ -10,10 +10,10 @@
|
|||||||
[MLflow](https://mlflow.org/) is an open-source platform for tracking machine learning experiments and managing models. You can use MLflow logging APIs with Azure Machine Learning service: the metrics and artifacts are logged to your Azure ML Workspace.
|
[MLflow](https://mlflow.org/) is an open-source platform for tracking machine learning experiments and managing models. You can use MLflow logging APIs with Azure Machine Learning service: the metrics and artifacts are logged to your Azure ML Workspace.
|
||||||
|
|
||||||
Try out the sample notebooks:
|
Try out the sample notebooks:
|
||||||
1. [Use MLflow with Azure Machine Learning for Local Training Run](./train-local/train-local.ipynb)
|
1. [Use MLflow with Azure Machine Learning for Local Training Run](./using-mlflow/train-local/train-local.ipynb)
|
||||||
1. [Use MLflow with Azure Machine Learning for Remote Training Run](./train-remote/train-remote.ipynb)
|
1. [Use MLflow with Azure Machine Learning for Remote Training Run](./using-mlflow/train-remote/train-remote.ipynb)
|
||||||
1. [Deploy Model as Azure Machine Learning Web Service using MLflow](./deploy-model/deploy-model.ipynb)
|
1. [Train and Deploy PyTorch Image Classifier](./using-mlflow/train-and-deploy-pytorch/train-and-deploy-pytorch.ipynb)
|
||||||
1. [Train and Deploy PyTorch Image Classifier](./train-deploy-pytorch/train-deploy-pytorch.ipynb)
|
1. [Train and Deploy Keras Image Classifier with MLflow auto logging](./using-mlflow/train-and-deploy-keras-auto-logging/train-and-deploy-keras-auto-logging.ipynb)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -100,7 +100,7 @@
|
|||||||
"\n",
|
"\n",
|
||||||
"# Check core SDK version number\n",
|
"# Check core SDK version number\n",
|
||||||
"\n",
|
"\n",
|
||||||
"print(\"This notebook was created using SDK version 1.10.0, you are currently running version\", azureml.core.VERSION)"
|
"print(\"This notebook was created using SDK version 1.13.0, you are currently running version\", azureml.core.VERSION)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -0,0 +1,78 @@
|
|||||||
|
# Copyright (c) Microsoft Corporation. All rights reserved.
|
||||||
|
# Licensed under the MIT License.
|
||||||
|
|
||||||
|
import mlflow
|
||||||
|
import mlflow.keras
|
||||||
|
import numpy as np
|
||||||
|
import warnings
|
||||||
|
|
||||||
|
import keras
|
||||||
|
from keras.models import Sequential
|
||||||
|
from keras.layers import Dense
|
||||||
|
from keras.optimizers import RMSprop
|
||||||
|
|
||||||
|
print("Keras version:", keras.__version__)
|
||||||
|
|
||||||
|
# Enable auto-logging to MLflow to capture Keras metrics.
|
||||||
|
mlflow.keras.autolog()
|
||||||
|
|
||||||
|
# Model / data parameters
|
||||||
|
n_inputs = 28 * 28
|
||||||
|
n_h1 = 300
|
||||||
|
n_h2 = 100
|
||||||
|
n_outputs = 10
|
||||||
|
learning_rate = 0.001
|
||||||
|
|
||||||
|
# the data, split between train and test sets
|
||||||
|
(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()
|
||||||
|
|
||||||
|
# Scale images to the [0, 1] range
|
||||||
|
x_train = x_train.astype("float32") / 255
|
||||||
|
x_test = x_test.astype("float32") / 255
|
||||||
|
|
||||||
|
# Flatten image to be (n, 28 * 28)
|
||||||
|
x_train = x_train.reshape(len(x_train), -1)
|
||||||
|
x_test = x_test.reshape(len(x_test), -1)
|
||||||
|
|
||||||
|
print("x_train shape:", x_train.shape)
|
||||||
|
print(x_train.shape[0], "train samples")
|
||||||
|
print(x_test.shape[0], "test samples")
|
||||||
|
|
||||||
|
# convert class vectors to binary class matrices
|
||||||
|
y_train = keras.utils.to_categorical(y_train, n_outputs)
|
||||||
|
y_test = keras.utils.to_categorical(y_test, n_outputs)
|
||||||
|
|
||||||
|
|
||||||
|
def driver():
|
||||||
|
warnings.filterwarnings("ignore")
|
||||||
|
|
||||||
|
with mlflow.start_run() as run:
|
||||||
|
|
||||||
|
# Build a simple MLP model
|
||||||
|
model = Sequential()
|
||||||
|
# first hidden layer
|
||||||
|
model.add(Dense(n_h1, activation='relu', input_shape=(n_inputs,)))
|
||||||
|
# second hidden layer
|
||||||
|
model.add(Dense(n_h2, activation='relu'))
|
||||||
|
# output layer
|
||||||
|
model.add(Dense(n_outputs, activation='softmax'))
|
||||||
|
model.summary()
|
||||||
|
|
||||||
|
batch_size = 128
|
||||||
|
epochs = 5
|
||||||
|
|
||||||
|
model.compile(loss='categorical_crossentropy',
|
||||||
|
optimizer=RMSprop(lr=learning_rate),
|
||||||
|
metrics=['accuracy'])
|
||||||
|
|
||||||
|
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_split=0.1)
|
||||||
|
|
||||||
|
score = model.evaluate(x_test, y_test, verbose=0)
|
||||||
|
print('Test loss:', score[0])
|
||||||
|
print('Test accuracy:', score[1])
|
||||||
|
|
||||||
|
return run
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
driver()
|
||||||
@@ -0,0 +1,455 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Copyright (c) Microsoft Corporation. All rights reserved.\n",
|
||||||
|
"\n",
|
||||||
|
"Licensed under the MIT License."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
""
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Use MLflow with Azure Machine Learning to Train and Deploy Keras Image Classifier\n",
|
||||||
|
"\n",
|
||||||
|
"This example shows you how to use MLflow together with Azure Machine Learning services for tracking the metrics and artifacts while training a Keras model to classify MNIST digit images and deploy the model as a web service. You'll learn how to:\n",
|
||||||
|
"\n",
|
||||||
|
" 1. Set up MLflow tracking URI so as to use Azure ML\n",
|
||||||
|
" 2. Create experiment\n",
|
||||||
|
" 3. Instrument your model with MLflow tracking\n",
|
||||||
|
" 4. Train a Keras model locally with MLflow auto logging\n",
|
||||||
|
" 5. Train a model on GPU compute on Azure with MLflow auto logging\n",
|
||||||
|
" 6. View your experiment within your Azure ML Workspace in Azure Portal\n",
|
||||||
|
" 7. Deploy the model as a web service on Azure Container Instance\n",
|
||||||
|
" 8. Call the model to make predictions\n",
|
||||||
|
" \n",
|
||||||
|
"### Pre-requisites\n",
|
||||||
|
" \n",
|
||||||
|
"If you are using a Notebook VM, you are all set. Otherwise, go through the [Configuration](../../../../configuration.ipnyb) notebook to set up your Azure Machine Learning workspace and ensure other common prerequisites are met.\n",
|
||||||
|
"\n",
|
||||||
|
"Install TensorFlow and Keras, this notebook has been tested with TensorFlow version 2.1.0 and Keras version 2.3.1.\n",
|
||||||
|
"\n",
|
||||||
|
"Also, install azureml-mlflow package using ```pip install azureml-mlflow```. Note that azureml-mlflow installs mlflow package itself as a dependency if you haven't done so previously.\n",
|
||||||
|
"\n",
|
||||||
|
"### Set-up\n",
|
||||||
|
"\n",
|
||||||
|
"Import packages and check versions of Azure ML SDK and MLflow installed on your computer. Then connect to your Workspace."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import sys, os\n",
|
||||||
|
"import mlflow\n",
|
||||||
|
"import mlflow.azureml\n",
|
||||||
|
"\n",
|
||||||
|
"import azureml.core\n",
|
||||||
|
"from azureml.core import Workspace\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"print(\"SDK version:\", azureml.core.VERSION)\n",
|
||||||
|
"print(\"MLflow version:\", mlflow.version.VERSION)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"ws = Workspace.from_config()\n",
|
||||||
|
"ws.get_details()"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Set tracking URI\n",
|
||||||
|
"\n",
|
||||||
|
"Set the MLflow tracking URI to point to your Azure ML Workspace. The subsequent logging calls from MLflow APIs will go to Azure ML services and will be tracked under your Workspace."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"mlflow.set_tracking_uri(ws.get_mlflow_tracking_uri())"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Create Experiment\n",
|
||||||
|
"\n",
|
||||||
|
"In both MLflow and Azure ML, training runs are grouped into experiments. Let's create one for our experimentation."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"experiment_name = \"keras-with-mlflow\"\n",
|
||||||
|
"mlflow.set_experiment(experiment_name)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Train model locally while logging metrics and artifacts\n",
|
||||||
|
"\n",
|
||||||
|
"The ```scripts/train.py``` program contains the code to load the image dataset, train and test the model. Within this program, the train.driver function wraps the end-to-end workflow.\n",
|
||||||
|
"\n",
|
||||||
|
"Within the driver, the ```mlflow.start_run``` starts MLflow tracking. Then, MLflow's automatic logging is used to log metrics, parameters and model for the Keras run.\n",
|
||||||
|
"\n",
|
||||||
|
"Let's add the program to search path, import it as a module and invoke the driver function. Note that the training can take few minutes."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"lib_path = os.path.abspath(\"scripts\")\n",
|
||||||
|
"sys.path.append(lib_path)\n",
|
||||||
|
"\n",
|
||||||
|
"import train"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"run = train.driver()"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Train model on GPU compute on Azure\n",
|
||||||
|
"\n",
|
||||||
|
"Next, let's run the same script on GPU-enabled compute for faster training. If you've completed the the [Configuration](../../../configuration.ipnyb) notebook, you should have a GPU cluster named \"gpu-cluster\" available in your workspace. Otherwise, follow the instructions in the notebook to create one. For simplicity, this example uses single process on single VM to train the model.\n",
|
||||||
|
"\n",
|
||||||
|
"Clone an environment object from the Tensorflow 2.1 Azure ML curated environment. Azure ML curated environments are pre-configured environments to simplify ML setup, reference [this doc](https://docs.microsoft.com/en-us/azure/machine-learning/how-to-use-environments#use-a-curated-environment) for more information. To enable MLflow tracking, add ```azureml-mlflow``` as pip package."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"from azureml.core import Environment\n",
|
||||||
|
"\n",
|
||||||
|
"env = Environment.get(workspace=ws, name=\"AzureML-TensorFlow-2.1-GPU\").clone(\"mlflow-env\")\n",
|
||||||
|
"\n",
|
||||||
|
"env.python.conda_dependencies.add_pip_package(\"azureml-mlflow\")\n",
|
||||||
|
"env.python.conda_dependencies.add_pip_package(\"keras==2.3.1\")\n",
|
||||||
|
"env.python.conda_dependencies.add_pip_package(\"numpy\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Create a ScriptRunConfig to specify the training configuration: script, compute as well as environment."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"from azureml.core import ScriptRunConfig\n",
|
||||||
|
"\n",
|
||||||
|
"src = ScriptRunConfig(source_directory=\"./scripts\", script=\"train.py\")\n",
|
||||||
|
"src.run_config.environment = env\n",
|
||||||
|
"src.run_config.target = \"gpu-cluster\""
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Get a reference to the experiment you created previously, but this time, as an Azure Machine Learning experiment object.\n",
|
||||||
|
"\n",
|
||||||
|
"Then, use the ```Experiment.submit``` method to start the remote training run. Note that the first training run often takes longer as Azure Machine Learning service builds the Docker image for executing the script. Subsequent runs will be faster as the cached image is used."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"from azureml.core import Experiment\n",
|
||||||
|
"\n",
|
||||||
|
"exp = Experiment(ws, experiment_name)\n",
|
||||||
|
"run = exp.submit(src)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"You can monitor the run and its metrics on Azure Portal."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"run"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Also, you can wait for run to complete."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"run.wait_for_completion(show_output=True)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Deploy model as web service\n",
|
||||||
|
"\n",
|
||||||
|
"The ```mlflow.azureml.deploy``` function registers the logged Keras+Tensorflow model and deploys the model in a framework-aware manner. It automatically creates the Tensorflow-specific inferencing wrapper code and specifies package dependencies for you. See [this doc](https://mlflow.org/docs/latest/models.html#id34) for more information on deploying models on Azure ML using MLflow.\n",
|
||||||
|
"\n",
|
||||||
|
"In this example, we deploy the Docker image to Azure Container Instance: a serverless compute capable of running a single container. You can tag and add descriptions to help keep track of your web service. \n",
|
||||||
|
"\n",
|
||||||
|
"[Other inferencing compute choices](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-deploy-and-where) include Azure Kubernetes Service which provides scalable endpoint suitable for production use.\n",
|
||||||
|
"\n",
|
||||||
|
"Note that the service deployment can take several minutes."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"from azureml.core.webservice import AciWebservice, Webservice\n",
|
||||||
|
"\n",
|
||||||
|
"model_path = \"model\"\n",
|
||||||
|
"\n",
|
||||||
|
"aci_config = AciWebservice.deploy_configuration(cpu_cores=2, \n",
|
||||||
|
" memory_gb=5, \n",
|
||||||
|
" tags={\"data\": \"MNIST\", \"method\" : \"keras\"}, \n",
|
||||||
|
" description=\"Predict using webservice\")\n",
|
||||||
|
"\n",
|
||||||
|
"webservice, azure_model = mlflow.azureml.deploy(model_uri='runs:/{}/{}'.format(run.id, model_path),\n",
|
||||||
|
" workspace=ws,\n",
|
||||||
|
" deployment_config=aci_config,\n",
|
||||||
|
" service_name=\"keras-mnist-1\",\n",
|
||||||
|
" model_name=\"keras_mnist\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Once the deployment has completed you can check the scoring URI of the web service."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"print(\"Scoring URI is: {}\".format(webservice.scoring_uri))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"In case of a service creation issue, you can use ```webservice.get_logs()``` to get logs to debug."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Make predictions using a web service\n",
|
||||||
|
"\n",
|
||||||
|
"To make the web service, create a test data set as normalized NumPy array. \n",
|
||||||
|
"\n",
|
||||||
|
"Then, let's define a utility function that takes a random image and converts it into a format and shape suitable for input to the Keras inferencing end-point. The conversion is done by: \n",
|
||||||
|
"\n",
|
||||||
|
" 1. Select a random (image, label) tuple\n",
|
||||||
|
" 2. Take the image and converting to to NumPy array \n",
|
||||||
|
" 3. Reshape array into 1 x 1 x N array\n",
|
||||||
|
" * 1 image in batch, 1 color channel, N = 784 pixels for MNIST images\n",
|
||||||
|
" * Note also ```x = x.view(-1, 1, 28, 28)``` in net definition in ```train.py``` program to shape incoming scoring requests.\n",
|
||||||
|
" 4. Convert the NumPy array to list to make it into a built-in type.\n",
|
||||||
|
" 5. Create a dictionary {\"data\", <list>} that can be converted to JSON string for web service requests."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import keras\n",
|
||||||
|
"import random\n",
|
||||||
|
"import numpy as np\n",
|
||||||
|
"\n",
|
||||||
|
"# the data, split between train and test sets\n",
|
||||||
|
"(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()\n",
|
||||||
|
"\n",
|
||||||
|
"# Scale images to the [0, 1] range\n",
|
||||||
|
"x_test = x_test.astype(\"float32\") / 255\n",
|
||||||
|
"x_test = x_test.reshape(len(x_test), -1)\n",
|
||||||
|
"\n",
|
||||||
|
"# convert class vectors to binary class matrices\n",
|
||||||
|
"y_test = keras.utils.to_categorical(y_test, 10)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"%matplotlib inline\n",
|
||||||
|
"\n",
|
||||||
|
"import json\n",
|
||||||
|
"import matplotlib.pyplot as plt\n",
|
||||||
|
"\n",
|
||||||
|
"# send a random row from the test set to score\n",
|
||||||
|
"random_index = np.random.randint(0, len(x_test)-1)\n",
|
||||||
|
"input_data = \"{\\\"data\\\": [\" + str(list(x_test[random_index])) + \"]}\"\n",
|
||||||
|
"\n",
|
||||||
|
"response = webservice.run(input_data)\n",
|
||||||
|
"\n",
|
||||||
|
"response = sorted(response[0].items(), key = lambda x: x[1], reverse = True)\n",
|
||||||
|
"\n",
|
||||||
|
"print(\"Predicted label:\", response[0][0])\n",
|
||||||
|
"plt.imshow(x_test[random_index].reshape(28,28), cmap = \"gray\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"You can also call the web service using a raw POST method against the web service"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import requests\n",
|
||||||
|
"\n",
|
||||||
|
"response = requests.post(url=webservice.scoring_uri, data=input_data,headers={\"Content-type\": \"application/json\"})\n",
|
||||||
|
"print(response.text)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Clean up\n",
|
||||||
|
"You can delete the ACI deployment with a delete API call."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"webservice.delete()"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"authors": [
|
||||||
|
{
|
||||||
|
"name": "hancwang"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"category": "tutorial",
|
||||||
|
"celltoolbar": "Edit Metadata",
|
||||||
|
"compute": [
|
||||||
|
"Local",
|
||||||
|
"AML Compute"
|
||||||
|
],
|
||||||
|
"datasets": [
|
||||||
|
"MNIST"
|
||||||
|
],
|
||||||
|
"deployment": [
|
||||||
|
"Azure Container Instance"
|
||||||
|
],
|
||||||
|
"exclude_from_index": false,
|
||||||
|
"framework": [
|
||||||
|
"Keras"
|
||||||
|
],
|
||||||
|
"friendly_name": "Use MLflow with Azure Machine Learning to Train and Deploy Keras Image Classifier",
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3.6",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python36"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.7.7"
|
||||||
|
},
|
||||||
|
"tags": [
|
||||||
|
"mlflow",
|
||||||
|
"keras"
|
||||||
|
],
|
||||||
|
"task": "Use MLflow with Azure Machine Learning to Train and Deploy Keras Image Classifier, leveraging MLflow auto logging"
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 4
|
||||||
|
}
|
||||||
@@ -0,0 +1,150 @@
|
|||||||
|
# Copyright (c) 2017, PyTorch Team
|
||||||
|
# All rights reserved
|
||||||
|
# Licensed under BSD 3-Clause License.
|
||||||
|
|
||||||
|
# This example is based on PyTorch MNIST example:
|
||||||
|
# https://github.com/pytorch/examples/blob/master/mnist/main.py
|
||||||
|
|
||||||
|
import mlflow
|
||||||
|
import mlflow.pytorch
|
||||||
|
from mlflow.utils.environment import _mlflow_conda_env
|
||||||
|
import warnings
|
||||||
|
import cloudpickle
|
||||||
|
import torch
|
||||||
|
import torch.nn as nn
|
||||||
|
import torch.nn.functional as F
|
||||||
|
import torch.optim as optim
|
||||||
|
import torchvision
|
||||||
|
from torchvision import datasets, transforms
|
||||||
|
|
||||||
|
|
||||||
|
class Net(nn.Module):
|
||||||
|
def __init__(self):
|
||||||
|
super(Net, self).__init__()
|
||||||
|
self.conv1 = nn.Conv2d(1, 20, 5, 1)
|
||||||
|
self.conv2 = nn.Conv2d(20, 50, 5, 1)
|
||||||
|
self.fc1 = nn.Linear(4 * 4 * 50, 500)
|
||||||
|
self.fc2 = nn.Linear(500, 10)
|
||||||
|
|
||||||
|
def forward(self, x):
|
||||||
|
# Added the view for reshaping score requests
|
||||||
|
x = x.view(-1, 1, 28, 28)
|
||||||
|
x = F.relu(self.conv1(x))
|
||||||
|
x = F.max_pool2d(x, 2, 2)
|
||||||
|
x = F.relu(self.conv2(x))
|
||||||
|
x = F.max_pool2d(x, 2, 2)
|
||||||
|
x = x.view(-1, 4 * 4 * 50)
|
||||||
|
x = F.relu(self.fc1(x))
|
||||||
|
x = self.fc2(x)
|
||||||
|
return F.log_softmax(x, dim=1)
|
||||||
|
|
||||||
|
|
||||||
|
def train(args, model, device, train_loader, optimizer, epoch):
|
||||||
|
model.train()
|
||||||
|
for batch_idx, (data, target) in enumerate(train_loader):
|
||||||
|
data, target = data.to(device), target.to(device)
|
||||||
|
optimizer.zero_grad()
|
||||||
|
output = model(data)
|
||||||
|
loss = F.nll_loss(output, target)
|
||||||
|
loss.backward()
|
||||||
|
optimizer.step()
|
||||||
|
if batch_idx % args.log_interval == 0:
|
||||||
|
print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
|
||||||
|
epoch, batch_idx * len(data), len(train_loader.dataset),
|
||||||
|
100. * batch_idx / len(train_loader), loss.item()))
|
||||||
|
# Use MLflow logging
|
||||||
|
mlflow.log_metric("epoch_loss", loss.item())
|
||||||
|
|
||||||
|
|
||||||
|
def test(args, model, device, test_loader):
|
||||||
|
model.eval()
|
||||||
|
test_loss = 0
|
||||||
|
correct = 0
|
||||||
|
with torch.no_grad():
|
||||||
|
for data, target in test_loader:
|
||||||
|
data, target = data.to(device), target.to(device)
|
||||||
|
output = model(data)
|
||||||
|
# sum up batch loss
|
||||||
|
test_loss += F.nll_loss(output, target, reduction="sum").item()
|
||||||
|
# get the index of the max log-probability
|
||||||
|
pred = output.argmax(dim=1, keepdim=True)
|
||||||
|
correct += pred.eq(target.view_as(pred)).sum().item()
|
||||||
|
|
||||||
|
test_loss /= len(test_loader.dataset)
|
||||||
|
print("\n")
|
||||||
|
print("Test set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n".format(
|
||||||
|
test_loss, correct, len(test_loader.dataset),
|
||||||
|
100. * correct / len(test_loader.dataset)))
|
||||||
|
# Use MLflow logging
|
||||||
|
mlflow.log_metric("average_loss", test_loss)
|
||||||
|
|
||||||
|
|
||||||
|
class Args(object):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
# Training settings
|
||||||
|
args = Args()
|
||||||
|
setattr(args, 'batch_size', 64)
|
||||||
|
setattr(args, 'test_batch_size', 1000)
|
||||||
|
setattr(args, 'epochs', 3) # Higher number for better convergence
|
||||||
|
setattr(args, 'lr', 0.01)
|
||||||
|
setattr(args, 'momentum', 0.5)
|
||||||
|
setattr(args, 'no_cuda', True)
|
||||||
|
setattr(args, 'seed', 1)
|
||||||
|
setattr(args, 'log_interval', 10)
|
||||||
|
setattr(args, 'save_model', True)
|
||||||
|
|
||||||
|
use_cuda = not args.no_cuda and torch.cuda.is_available()
|
||||||
|
|
||||||
|
torch.manual_seed(args.seed)
|
||||||
|
|
||||||
|
device = torch.device("cuda" if use_cuda else "cpu")
|
||||||
|
|
||||||
|
kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
|
||||||
|
train_loader = torch.utils.data.DataLoader(
|
||||||
|
datasets.MNIST('../data', train=True, download=True,
|
||||||
|
transform=transforms.Compose([
|
||||||
|
transforms.ToTensor(),
|
||||||
|
transforms.Normalize((0.1307,), (0.3081,))
|
||||||
|
])),
|
||||||
|
batch_size=args.batch_size, shuffle=True, **kwargs)
|
||||||
|
test_loader = torch.utils.data.DataLoader(
|
||||||
|
datasets.MNIST(
|
||||||
|
'../data',
|
||||||
|
train=False,
|
||||||
|
transform=transforms.Compose([
|
||||||
|
transforms.ToTensor(),
|
||||||
|
transforms.Normalize((0.1307,), (0.3081,))])),
|
||||||
|
batch_size=args.test_batch_size, shuffle=True, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def driver():
|
||||||
|
warnings.filterwarnings("ignore")
|
||||||
|
# Dependencies for deploying the model
|
||||||
|
pytorch_index = "https://download.pytorch.org/whl/"
|
||||||
|
pytorch_version = "cpu/torch-1.1.0-cp36-cp36m-linux_x86_64.whl"
|
||||||
|
deps = [
|
||||||
|
"cloudpickle=={}".format(cloudpickle.__version__),
|
||||||
|
pytorch_index + pytorch_version,
|
||||||
|
"torchvision=={}".format(torchvision.__version__),
|
||||||
|
"Pillow=={}".format("6.0.0")
|
||||||
|
]
|
||||||
|
with mlflow.start_run() as run:
|
||||||
|
model = Net().to(device)
|
||||||
|
optimizer = optim.SGD(
|
||||||
|
model.parameters(),
|
||||||
|
lr=args.lr,
|
||||||
|
momentum=args.momentum)
|
||||||
|
for epoch in range(1, args.epochs + 1):
|
||||||
|
train(args, model, device, train_loader, optimizer, epoch)
|
||||||
|
test(args, model, device, test_loader)
|
||||||
|
# Log model to run history using MLflow
|
||||||
|
if args.save_model:
|
||||||
|
model_env = _mlflow_conda_env(additional_pip_deps=deps)
|
||||||
|
mlflow.pytorch.log_model(model, "model", conda_env=model_env)
|
||||||
|
return run
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
driver()
|
||||||
@@ -0,0 +1,464 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Copyright (c) Microsoft Corporation. All rights reserved.\n",
|
||||||
|
"\n",
|
||||||
|
"Licensed under the MIT License."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
""
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"# Use MLflow with Azure Machine Learning to Train and Deploy PyTorch Image Classifier\n",
|
||||||
|
"\n",
|
||||||
|
"This example shows you how to use MLflow together with Azure Machine Learning services for tracking the metrics and artifacts while training a PyTorch model to classify MNIST digit images and deploy the model as a web service. You'll learn how to:\n",
|
||||||
|
"\n",
|
||||||
|
" 1. Set up MLflow tracking URI so as to use Azure ML\n",
|
||||||
|
" 2. Create experiment\n",
|
||||||
|
" 3. Instrument your model with MLflow tracking\n",
|
||||||
|
" 4. Train a PyTorch model locally\n",
|
||||||
|
" 5. Train a model on GPU compute on Azure\n",
|
||||||
|
" 6. View your experiment within your Azure ML Workspace in Azure Portal\n",
|
||||||
|
" 7. Deploy the model as a web service on Azure Container Instance\n",
|
||||||
|
" 8. Call the model to make predictions\n",
|
||||||
|
" \n",
|
||||||
|
"## Pre-requisites\n",
|
||||||
|
" \n",
|
||||||
|
"If you are using a Notebook VM, you are all set. Otherwise, go through the [Configuration](../../../../configuration.ipnyb) notebook to set up your Azure Machine Learning workspace and ensure other common prerequisites are met.\n",
|
||||||
|
"\n",
|
||||||
|
"Install PyTorch, this notebook has been tested with torch==1.4\n",
|
||||||
|
"\n",
|
||||||
|
"Also, install azureml-mlflow package using ```pip install azureml-mlflow```. Note that azureml-mlflow installs mlflow package itself as a dependency if you haven't done so previously.\n",
|
||||||
|
"\n",
|
||||||
|
"## Set-up\n",
|
||||||
|
"\n",
|
||||||
|
"Import packages and check versions of Azure ML SDK and MLflow installed on your computer. Then connect to your Workspace."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import sys, os\n",
|
||||||
|
"import mlflow\n",
|
||||||
|
"import mlflow.azureml\n",
|
||||||
|
"\n",
|
||||||
|
"import azureml.core\n",
|
||||||
|
"from azureml.core import Workspace\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"print(\"SDK version:\", azureml.core.VERSION)\n",
|
||||||
|
"print(\"MLflow version:\", mlflow.version.VERSION)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"ws = Workspace.from_config()\n",
|
||||||
|
"ws.get_details()"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Set tracking URI\n",
|
||||||
|
"\n",
|
||||||
|
"Set the MLflow tracking URI to point to your Azure ML Workspace. The subsequent logging calls from MLflow APIs will go to Azure ML services and will be tracked under your Workspace."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"mlflow.set_tracking_uri(ws.get_mlflow_tracking_uri())"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Create Experiment\n",
|
||||||
|
"\n",
|
||||||
|
"In both MLflow and Azure ML, training runs are grouped into experiments. Let's create one for our experimentation."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"experiment_name = \"pytorch-with-mlflow\"\n",
|
||||||
|
"mlflow.set_experiment(experiment_name)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Train model locally while logging metrics and artifacts\n",
|
||||||
|
"\n",
|
||||||
|
"The ```scripts/train.py``` program contains the code to load the image dataset, train and test the model. Within this program, the train.driver function wraps the end-to-end workflow.\n",
|
||||||
|
"\n",
|
||||||
|
"Within the driver, the ```mlflow.start_run``` starts MLflow tracking. Then, ```mlflow.log_metric``` functions are used to track the convergence of the neural network training iterations. Finally ```mlflow.pytorch.save_model``` is used to save the trained model in framework-aware manner.\n",
|
||||||
|
"\n",
|
||||||
|
"Let's add the program to search path, import it as a module and invoke the driver function. Note that the training can take few minutes."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"lib_path = os.path.abspath(\"scripts\")\n",
|
||||||
|
"sys.path.append(lib_path)\n",
|
||||||
|
"\n",
|
||||||
|
"import train"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"run = train.driver()"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Train model on GPU compute on Azure\n",
|
||||||
|
"\n",
|
||||||
|
"Next, let's run the same script on GPU-enabled compute for faster training. If you've completed the the [Configuration](../../../configuration.ipnyb) notebook, you should have a GPU cluster named \"gpu-cluster\" available in your workspace. Otherwise, follow the instructions in the notebook to create one. For simplicity, this example uses single process on single VM to train the model.\n",
|
||||||
|
"\n",
|
||||||
|
"Clone an environment object from the PyTorch 1.4 Azure ML curated environment. Azure ML curated environments are pre-configured environments to simplify ML setup, reference [this doc](https://docs.microsoft.com/en-us/azure/machine-learning/how-to-use-environments#use-a-curated-environment) for more information. To enable MLflow tracking, add ```azureml-mlflow``` as pip package."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"from azureml.core import Environment\n",
|
||||||
|
"\n",
|
||||||
|
"env = Environment.get(workspace=ws, name=\"AzureML-PyTorch-1.4-GPU\").clone(\"mlflow-env\")\n",
|
||||||
|
"\n",
|
||||||
|
"env.python.conda_dependencies.add_pip_package(\"azureml-mlflow\")\n",
|
||||||
|
"env.python.conda_dependencies.add_pip_package(\"Pillow==6.0.0\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Create a ScriptRunConfig to specify the training configuration: script, compute as well as environment."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"from azureml.core import ScriptRunConfig\n",
|
||||||
|
"\n",
|
||||||
|
"src = ScriptRunConfig(source_directory=\"./scripts\", script=\"train.py\")\n",
|
||||||
|
"src.run_config.environment = env\n",
|
||||||
|
"src.run_config.target = \"gpu-cluster\""
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Get a reference to the experiment you created previously, but this time, as an Azure Machine Learning experiment object.\n",
|
||||||
|
"\n",
|
||||||
|
"Then, use the ```Experiment.submit``` method to start the remote training run. Note that the first training run often takes longer as Azure Machine Learning service builds the Docker image for executing the script. Subsequent runs will be faster as the cached image is used."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"from azureml.core import Experiment\n",
|
||||||
|
"\n",
|
||||||
|
"exp = Experiment(ws, experiment_name)\n",
|
||||||
|
"run = exp.submit(src)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"You can monitor the run and its metrics on Azure Portal."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"run"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Also, you can wait for run to complete."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"run.wait_for_completion(show_output=True)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Deploy model as web service\n",
|
||||||
|
"\n",
|
||||||
|
"The ```mlflow.azureml.deploy``` function registers the logged PyTorch model and deploys the model in a framework-aware manner. It automatically creates the PyTorch-specific inferencing wrapper code and specifies package dependencies for you. See [this doc](https://mlflow.org/docs/latest/models.html#id34) for more information on deploying models on Azure ML using MLflow.\n",
|
||||||
|
"\n",
|
||||||
|
"In this example, we deploy the Docker image to Azure Container Instance: a serverless compute capable of running a single container. You can tag and add descriptions to help keep track of your web service. \n",
|
||||||
|
"\n",
|
||||||
|
"[Other inferencing compute choices](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-deploy-and-where) include Azure Kubernetes Service which provides scalable endpoint suitable for production use.\n",
|
||||||
|
"\n",
|
||||||
|
"Note that the service deployment can take several minutes."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"from azureml.core.webservice import AciWebservice, Webservice\n",
|
||||||
|
"\n",
|
||||||
|
"model_path = \"model\"\n",
|
||||||
|
"\n",
|
||||||
|
"aci_config = AciWebservice.deploy_configuration(cpu_cores=2, \n",
|
||||||
|
" memory_gb=5, \n",
|
||||||
|
" tags={\"data\": \"MNIST\", \"method\" : \"pytorch\"}, \n",
|
||||||
|
" description=\"Predict using webservice\")\n",
|
||||||
|
"\n",
|
||||||
|
"webservice, azure_model = mlflow.azureml.deploy(model_uri='runs:/{}/{}'.format(run.id, model_path),\n",
|
||||||
|
" workspace=ws,\n",
|
||||||
|
" deployment_config=aci_config,\n",
|
||||||
|
" service_name=\"pytorch-mnist-1\",\n",
|
||||||
|
" model_name=\"pytorch_mnist\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Once the deployment has completed you can check the scoring URI of the web service."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"print(\"Scoring URI is: {}\".format(webservice.scoring_uri))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"In case of a service creation issue, you can use ```webservice.get_logs()``` to get logs to debug."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Make predictions using a web service\n",
|
||||||
|
"\n",
|
||||||
|
"To make the web service, create a test data set as normalized PyTorch tensors. \n",
|
||||||
|
"\n",
|
||||||
|
"Then, let's define a utility function that takes a random image and converts it into a format and shape suitable for input to the PyTorch inferencing end-point. The conversion is done by: \n",
|
||||||
|
"\n",
|
||||||
|
" 1. Select a random (image, label) tuple\n",
|
||||||
|
" 2. Take the image and converting the tensor to NumPy array \n",
|
||||||
|
" 3. Reshape array into 1 x 1 x N array\n",
|
||||||
|
" * 1 image in batch, 1 color channel, N = 784 pixels for MNIST images\n",
|
||||||
|
" * Note also ```x = x.view(-1, 1, 28, 28)``` in net definition in ```train.py``` program to shape incoming scoring requests.\n",
|
||||||
|
" 4. Convert the NumPy array to list to make it into a built-in type.\n",
|
||||||
|
" 5. Create a dictionary {\"data\", <list>} that can be converted to JSON string for web service requests."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"from torchvision import datasets, transforms\n",
|
||||||
|
"import random\n",
|
||||||
|
"import numpy as np\n",
|
||||||
|
"\n",
|
||||||
|
"test_data = datasets.MNIST('../data', train=False, transform=transforms.Compose([\n",
|
||||||
|
" transforms.ToTensor(),\n",
|
||||||
|
" transforms.Normalize((0.1307,), (0.3081,))]))\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"def get_random_image():\n",
|
||||||
|
" image_idx = random.randint(0,len(test_data))\n",
|
||||||
|
" image_as_tensor = test_data[image_idx][0]\n",
|
||||||
|
" return {\"data\": elem for elem in image_as_tensor.numpy().reshape(1,1,-1).tolist()}"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Then, invoke the web service using a random test image. Convert the dictionary containing the image to JSON string before passing it to web service.\n",
|
||||||
|
"\n",
|
||||||
|
"The response contains the raw scores for each label, with greater value indicating higher probability. Sort the labels and select the one with greatest score to get the prediction. Let's also plot the image sent to web service for comparison purposes."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"%matplotlib inline\n",
|
||||||
|
"\n",
|
||||||
|
"import json\n",
|
||||||
|
"import matplotlib.pyplot as plt\n",
|
||||||
|
"\n",
|
||||||
|
"test_image = get_random_image()\n",
|
||||||
|
"\n",
|
||||||
|
"response = webservice.run(json.dumps(test_image))\n",
|
||||||
|
"\n",
|
||||||
|
"response = sorted(response[0].items(), key = lambda x: x[1], reverse = True)\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"print(\"Predicted label:\", response[0][0])\n",
|
||||||
|
"plt.imshow(np.array(test_image[\"data\"]).reshape(28,28), cmap = \"gray\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"You can also call the web service using a raw POST method against the web service"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import requests\n",
|
||||||
|
"\n",
|
||||||
|
"response = requests.post(url=webservice.scoring_uri, data=json.dumps(test_image),headers={\"Content-type\": \"application/json\"})\n",
|
||||||
|
"print(response.text)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Clean up\n",
|
||||||
|
"You can delete the ACI deployment with a delete API call."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"webservice.delete()"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"authors": [
|
||||||
|
{
|
||||||
|
"name": "shipatel"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"category": "tutorial",
|
||||||
|
"celltoolbar": "Edit Metadata",
|
||||||
|
"compute": [
|
||||||
|
"Local",
|
||||||
|
"AML Compute"
|
||||||
|
],
|
||||||
|
"datasets": [
|
||||||
|
"MNIST"
|
||||||
|
],
|
||||||
|
"deployment": [
|
||||||
|
"Azure Container Instance"
|
||||||
|
],
|
||||||
|
"exclude_from_index": false,
|
||||||
|
"framework": [
|
||||||
|
"PyTorch"
|
||||||
|
],
|
||||||
|
"friendly_name": "Use MLflow with Azure Machine Learning to Train and Deploy PyTorch Image Classifier",
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3.6",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python36"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.7.7"
|
||||||
|
},
|
||||||
|
"name": "mlflow-sparksummit-pytorch",
|
||||||
|
"notebookId": 2495374963457641,
|
||||||
|
"tags": [
|
||||||
|
"mlflow",
|
||||||
|
"pytorch"
|
||||||
|
],
|
||||||
|
"task": "Use MLflow with Azure Machine Learning to train and deploy PyTorch image classifier model"
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 4
|
||||||
|
}
|
||||||
@@ -449,7 +449,7 @@
|
|||||||
" compute_target=compute_target, \n",
|
" compute_target=compute_target, \n",
|
||||||
" entry_script='keras_mnist.py',\n",
|
" entry_script='keras_mnist.py',\n",
|
||||||
" framework_version='2.0', \n",
|
" framework_version='2.0', \n",
|
||||||
" pip_packages=['keras<=2.3.1','azureml-dataprep[pandas,fuse]','matplotlib'])"
|
" pip_packages=['keras<=2.3.1','azureml-dataset-runtime[pandas,fuse]','matplotlib'])"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -604,21 +604,19 @@
|
|||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"## Predict on the test set\n",
|
"## Predict on the test set (Optional)\n",
|
||||||
"Let's check the version of the local Keras. Make sure it matches with the version number printed out in the training script. Otherwise you might not be able to load the model properly."
|
"Let's check the version of the local Keras. Make sure it matches with the version number printed out in the training script. Otherwise you might not be able to load the model properly."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "markdown",
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
"source": [
|
||||||
"import keras\n",
|
" import keras\n",
|
||||||
"import tensorflow as tf\n",
|
" import tensorflow as tf\n",
|
||||||
"\n",
|
"\n",
|
||||||
"print(\"Keras version:\", keras.__version__)\n",
|
" print(\"Keras version:\", keras.__version__)\n",
|
||||||
"print(\"Tensorflow version:\", tf.__version__)"
|
" print(\"Tensorflow version:\", tf.__version__)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -629,21 +627,19 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "markdown",
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
"source": [
|
||||||
"from keras.models import model_from_json\n",
|
" from keras.models import model_from_json\n",
|
||||||
"\n",
|
"\n",
|
||||||
"# load json and create model\n",
|
" # load json and create model\n",
|
||||||
"json_file = open('model/model.json', 'r')\n",
|
" json_file = open('model/model.json', 'r')\n",
|
||||||
"loaded_model_json = json_file.read()\n",
|
" loaded_model_json = json_file.read()\n",
|
||||||
"json_file.close()\n",
|
" json_file.close()\n",
|
||||||
"loaded_model = model_from_json(loaded_model_json)\n",
|
" loaded_model = model_from_json(loaded_model_json)\n",
|
||||||
"# load weights into new model\n",
|
" # load weights into new model\n",
|
||||||
"loaded_model.load_weights(\"model/model.h5\")\n",
|
" loaded_model.load_weights(\"model/model.h5\")\n",
|
||||||
"print(\"Model loaded from disk.\")"
|
" print(\"Model loaded from disk.\")"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -654,19 +650,17 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "markdown",
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
"source": [
|
||||||
"# evaluate loaded model on test data\n",
|
" # evaluate loaded model on test data\n",
|
||||||
"loaded_model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy'])\n",
|
" loaded_model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy'])\n",
|
||||||
"y_test_ohe = one_hot_encode(y_test, 10)\n",
|
" y_test_ohe = one_hot_encode(y_test, 10)\n",
|
||||||
"y_hat = np.argmax(loaded_model.predict(X_test), axis=1)\n",
|
" y_hat = np.argmax(loaded_model.predict(X_test), axis=1)\n",
|
||||||
"\n",
|
"\n",
|
||||||
"# print the first 30 labels and predictions\n",
|
" # print the first 30 labels and predictions\n",
|
||||||
"print('labels: \\t', y_test[:30])\n",
|
" print('labels: \\t', y_test[:30])\n",
|
||||||
"print('predictions:\\t', y_hat[:30])"
|
" print('predictions:\\t', y_hat[:30])"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -677,12 +671,10 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "markdown",
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
"source": [
|
||||||
"print(\"Accuracy on the test set:\", np.average(y_hat == y_test))"
|
" print(\"Accuracy on the test set:\", np.average(y_hat == y_test))"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -730,7 +722,7 @@
|
|||||||
" compute_target=compute_target,\n",
|
" compute_target=compute_target,\n",
|
||||||
" entry_script='keras_mnist.py',\n",
|
" entry_script='keras_mnist.py',\n",
|
||||||
" framework_version='2.0',\n",
|
" framework_version='2.0',\n",
|
||||||
" pip_packages=['keras<=2.3.1','azureml-dataprep[pandas,fuse]','matplotlib'])"
|
" pip_packages=['keras<=2.3.1','azureml-dataset-runtime[pandas,fuse]','matplotlib'])"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -1056,7 +1048,7 @@
|
|||||||
" font_color = 'red' if y_test[s] != result[i] else 'black'\n",
|
" font_color = 'red' if y_test[s] != result[i] else 'black'\n",
|
||||||
" clr_map = plt.cm.gray if y_test[s] != result[i] else plt.cm.Greys\n",
|
" clr_map = plt.cm.gray if y_test[s] != result[i] else plt.cm.Greys\n",
|
||||||
" \n",
|
" \n",
|
||||||
" plt.text(x=10, y=-10, s=y_test[s], fontsize=18, color=font_color)\n",
|
" plt.text(x=10, y=-10, s=result[i], fontsize=18, color=font_color)\n",
|
||||||
" plt.imshow(X_test[s].reshape(28, 28), cmap=clr_map)\n",
|
" plt.imshow(X_test[s].reshape(28, 28), cmap=clr_map)\n",
|
||||||
" \n",
|
" \n",
|
||||||
" i = i + 1\n",
|
" i = i + 1\n",
|
||||||
|
|||||||
@@ -1,406 +0,0 @@
|
|||||||
{
|
|
||||||
"cells": [
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"Copyright (c) Microsoft Corporation. All rights reserved.\n",
|
|
||||||
"\n",
|
|
||||||
"Licensed under the MIT License."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
""
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"# Train using Azure Machine Learning Compute Instance\n",
|
|
||||||
"\n",
|
|
||||||
"* Initialize Workspace\n",
|
|
||||||
"* Introduction to ComputeInstance\n",
|
|
||||||
"* Create an Experiment\n",
|
|
||||||
"* Submit ComputeInstance run\n",
|
|
||||||
"* Additional operations to perform on ComputeInstance"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Prerequisites\n",
|
|
||||||
"If you are using an Azure Machine Learning ComputeInstance, you are all set. Otherwise, go through the [configuration](../../../configuration.ipynb) notebook first if you haven't already to establish your connection to the AzureML Workspace."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# Check core SDK version number\n",
|
|
||||||
"import azureml.core\n",
|
|
||||||
"\n",
|
|
||||||
"print(\"SDK version:\", azureml.core.VERSION)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Initialize Workspace\n",
|
|
||||||
"\n",
|
|
||||||
"Initialize a workspace object"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {
|
|
||||||
"tags": [
|
|
||||||
"create workspace"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"from azureml.core import Workspace\n",
|
|
||||||
"\n",
|
|
||||||
"ws = Workspace.from_config()\n",
|
|
||||||
"print(ws.name, ws.resource_group, ws.location, ws.subscription_id, sep = '\\n')"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Introduction to ComputeInstance\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"Azure Machine Learning compute instance is a fully-managed cloud-based workstation optimized for your machine learning development environment. It is created **within your workspace region**.\n",
|
|
||||||
"\n",
|
|
||||||
"For more information on ComputeInstance, please read [this article](https://docs.microsoft.com/en-us/azure/machine-learning/concept-compute-instance)\n",
|
|
||||||
"\n",
|
|
||||||
"**Note**: As with other Azure services, there are limits on certain resources (for eg. AmlCompute quota) associated with the Azure Machine Learning service. Please read [this article](https://docs.microsoft.com/azure/machine-learning/service/how-to-manage-quotas) on the default limits and how to request more quota."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### Create ComputeInstance\n",
|
|
||||||
"First lets check which VM families are available in your region. Azure is a regional service and some specialized SKUs (especially GPUs) are only available in certain regions. Since ComputeInstance is created in the region of your workspace, we will use the supported_vms () function to see if the VM family we want to use ('STANDARD_D3_V2') is supported.\n",
|
|
||||||
"\n",
|
|
||||||
"You can also pass a different region to check availability and then re-create your workspace in that region through the [configuration notebook](../../../configuration.ipynb)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {
|
|
||||||
"msdoc": "how-to-auto-train-remote.md",
|
|
||||||
"name": "check_region"
|
|
||||||
},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"from azureml.core.compute import ComputeTarget, ComputeInstance\n",
|
|
||||||
"\n",
|
|
||||||
"ComputeInstance.supported_vmsizes(workspace = ws)\n",
|
|
||||||
"# ComputeInstance.supported_vmsizes(workspace = ws, location='eastus')"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {
|
|
||||||
"msdoc": "how-to-auto-train-remote.md",
|
|
||||||
"name": "create_instance"
|
|
||||||
},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"import datetime\n",
|
|
||||||
"import time\n",
|
|
||||||
"\n",
|
|
||||||
"from azureml.core.compute import ComputeTarget, ComputeInstance\n",
|
|
||||||
"from azureml.core.compute_target import ComputeTargetException\n",
|
|
||||||
"\n",
|
|
||||||
"# Choose a name for your instance\n",
|
|
||||||
"# Compute instance name should be unique across the azure region\n",
|
|
||||||
"compute_name = \"ci{}\".format(ws._workspace_id)[:10]\n",
|
|
||||||
"\n",
|
|
||||||
"# Verify that instance does not exist already\n",
|
|
||||||
"try:\n",
|
|
||||||
" instance = ComputeInstance(workspace=ws, name=compute_name)\n",
|
|
||||||
" print('Found existing instance, use it.')\n",
|
|
||||||
"except ComputeTargetException:\n",
|
|
||||||
" compute_config = ComputeInstance.provisioning_configuration(\n",
|
|
||||||
" vm_size='STANDARD_D3_V2',\n",
|
|
||||||
" ssh_public_access=False,\n",
|
|
||||||
" # vnet_resourcegroup_name='<my-resource-group>',\n",
|
|
||||||
" # vnet_name='<my-vnet-name>',\n",
|
|
||||||
" # subnet_name='default',\n",
|
|
||||||
" # admin_user_ssh_public_key='<my-sshkey>'\n",
|
|
||||||
" )\n",
|
|
||||||
" instance = ComputeInstance.create(ws, compute_name, compute_config)\n",
|
|
||||||
" instance.wait_for_completion(show_output=True)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Create An Experiment\n",
|
|
||||||
"\n",
|
|
||||||
"**Experiment** is a logical container in an Azure ML Workspace. It hosts run records which can include run metrics and output artifacts from your experiments."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"from azureml.core import Experiment\n",
|
|
||||||
"experiment_name = 'train-on-computeinstance'\n",
|
|
||||||
"experiment = Experiment(workspace = ws, name = experiment_name)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Submit ComputeInstance run\n",
|
|
||||||
"The training script `train.py` is already created for you"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### Create environment\n",
|
|
||||||
"\n",
|
|
||||||
"Create an environment with scikit-learn installed."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"from azureml.core import Environment\n",
|
|
||||||
"from azureml.core.conda_dependencies import CondaDependencies\n",
|
|
||||||
"\n",
|
|
||||||
"myenv = Environment(\"myenv\")\n",
|
|
||||||
"myenv.python.conda_dependencies = CondaDependencies.create(conda_packages=['scikit-learn'])"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### Configure & Run"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"from azureml.core import ScriptRunConfig\n",
|
|
||||||
"from azureml.core.runconfig import DEFAULT_CPU_IMAGE\n",
|
|
||||||
"\n",
|
|
||||||
"src = ScriptRunConfig(source_directory='', script='train.py')\n",
|
|
||||||
"\n",
|
|
||||||
"# Set compute target to the one created in previous step\n",
|
|
||||||
"src.run_config.target = instance\n",
|
|
||||||
"\n",
|
|
||||||
"# Set environment\n",
|
|
||||||
"src.run_config.environment = myenv\n",
|
|
||||||
" \n",
|
|
||||||
"run = experiment.submit(config=src)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"Note: if you need to cancel a run, you can follow [these instructions](https://aka.ms/aml-docs-cancel-run)."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"from azureml.widgets import RunDetails\n",
|
|
||||||
"RunDetails(run).show()"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"You can use the get_active_runs() to get the currently running or queued jobs on the compute instance"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# wait for the run to reach Queued or Running state if it is in Preparing state\n",
|
|
||||||
"status = run.get_status()\n",
|
|
||||||
"while status not in ['Queued', 'Running', 'Completed', 'Failed', 'Canceled']:\n",
|
|
||||||
" state = run.get_status()\n",
|
|
||||||
" print('Run status: {}'.format(status))\n",
|
|
||||||
" time.sleep(10)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# get active runs which are in Queued or Running state\n",
|
|
||||||
"active_runs = instance.get_active_runs()\n",
|
|
||||||
"for active_run in active_runs:\n",
|
|
||||||
" print(active_run.run_id, ',', active_run.status)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"run.wait_for_completion()\n",
|
|
||||||
"print(run.get_metrics())"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### Additional operations to perform on ComputeInstance\n",
|
|
||||||
"\n",
|
|
||||||
"You can perform more operations on ComputeInstance such as get status, change the state or deleting the compute."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {
|
|
||||||
"msdoc": "how-to-auto-train-remote.md",
|
|
||||||
"name": "get_status"
|
|
||||||
},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# get_status() gets the latest status of the ComputeInstance target\n",
|
|
||||||
"instance.get_status()"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {
|
|
||||||
"msdoc": "how-to-auto-train-remote.md",
|
|
||||||
"name": "stop"
|
|
||||||
},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# stop() is used to stop the ComputeInstance\n",
|
|
||||||
"# Stopping ComputeInstance will stop the billing meter and persist the state on the disk.\n",
|
|
||||||
"# Available Quota will not be changed with this operation.\n",
|
|
||||||
"instance.stop(wait_for_completion=True, show_output=True)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {
|
|
||||||
"msdoc": "how-to-auto-train-remote.md",
|
|
||||||
"name": "start"
|
|
||||||
},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# start() is used to start the ComputeInstance if it is in stopped state\n",
|
|
||||||
"instance.start(wait_for_completion=True, show_output=True)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# restart() is used to restart the ComputeInstance\n",
|
|
||||||
"instance.restart(wait_for_completion=True, show_output=True)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# delete() is used to delete the ComputeInstance target. Useful if you want to re-use the compute name \n",
|
|
||||||
"# instance.delete(wait_for_completion=True, show_output=True)"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"authors": [
|
|
||||||
{
|
|
||||||
"name": "ramagott"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"category": "training",
|
|
||||||
"compute": [
|
|
||||||
"Compute Instance"
|
|
||||||
],
|
|
||||||
"datasets": [
|
|
||||||
"Diabetes"
|
|
||||||
],
|
|
||||||
"deployment": [
|
|
||||||
"None"
|
|
||||||
],
|
|
||||||
"exclude_from_index": false,
|
|
||||||
"framework": [
|
|
||||||
"None"
|
|
||||||
],
|
|
||||||
"friendly_name": "Train on Azure Machine Learning Compute Instance",
|
|
||||||
"index_order": 1,
|
|
||||||
"kernelspec": {
|
|
||||||
"display_name": "Python 3.6",
|
|
||||||
"language": "python",
|
|
||||||
"name": "python36"
|
|
||||||
},
|
|
||||||
"language_info": {
|
|
||||||
"codemirror_mode": {
|
|
||||||
"name": "ipython",
|
|
||||||
"version": 3
|
|
||||||
},
|
|
||||||
"file_extension": ".py",
|
|
||||||
"mimetype": "text/x-python",
|
|
||||||
"name": "python",
|
|
||||||
"nbconvert_exporter": "python",
|
|
||||||
"pygments_lexer": "ipython3",
|
|
||||||
"version": "3.7.7"
|
|
||||||
},
|
|
||||||
"tags": [
|
|
||||||
"None"
|
|
||||||
],
|
|
||||||
"task": "Submit a run on Azure Machine Learning Compute Instance."
|
|
||||||
},
|
|
||||||
"nbformat": 4,
|
|
||||||
"nbformat_minor": 2
|
|
||||||
}
|
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
name: train-on-computeinstance
|
|
||||||
dependencies:
|
|
||||||
- scikit-learn
|
|
||||||
- pip:
|
|
||||||
- azureml-sdk
|
|
||||||
- azureml-widgets
|
|
||||||
@@ -1,48 +0,0 @@
|
|||||||
# Copyright (c) Microsoft. All rights reserved.
|
|
||||||
# Licensed under the MIT license.
|
|
||||||
|
|
||||||
from sklearn.datasets import load_diabetes
|
|
||||||
from sklearn.linear_model import Ridge
|
|
||||||
from sklearn.metrics import mean_squared_error
|
|
||||||
from sklearn.model_selection import train_test_split
|
|
||||||
from azureml.core.run import Run
|
|
||||||
import os
|
|
||||||
import numpy as np
|
|
||||||
# sklearn.externals.joblib is removed in 0.23
|
|
||||||
try:
|
|
||||||
from sklearn.externals import joblib
|
|
||||||
except ImportError:
|
|
||||||
import joblib
|
|
||||||
|
|
||||||
os.makedirs('./outputs', exist_ok=True)
|
|
||||||
|
|
||||||
X, y = load_diabetes(return_X_y=True)
|
|
||||||
|
|
||||||
run = Run.get_context()
|
|
||||||
|
|
||||||
X_train, X_test, y_train, y_test = train_test_split(X, y,
|
|
||||||
test_size=0.2,
|
|
||||||
random_state=0)
|
|
||||||
data = {"train": {"X": X_train, "y": y_train},
|
|
||||||
"test": {"X": X_test, "y": y_test}}
|
|
||||||
|
|
||||||
# list of numbers from 0.0 to 1.0 with a 0.05 interval
|
|
||||||
alphas = np.arange(0.0, 1.0, 0.05)
|
|
||||||
|
|
||||||
for alpha in alphas:
|
|
||||||
# Use Ridge algorithm to create a regression model
|
|
||||||
reg = Ridge(alpha=alpha)
|
|
||||||
reg.fit(data["train"]["X"], data["train"]["y"])
|
|
||||||
|
|
||||||
preds = reg.predict(data["test"]["X"])
|
|
||||||
mse = mean_squared_error(preds, data["test"]["y"])
|
|
||||||
run.log('alpha', alpha)
|
|
||||||
run.log('mse', mse)
|
|
||||||
|
|
||||||
model_file_name = 'ridge_{0:.2f}.pkl'.format(alpha)
|
|
||||||
# save model in the outputs folder so it automatically get uploaded
|
|
||||||
with open(model_file_name, "wb") as file:
|
|
||||||
joblib.dump(value=reg, filename=os.path.join('./outputs/',
|
|
||||||
model_file_name))
|
|
||||||
|
|
||||||
print('alpha is {0:.2f}, and mse is {1:0.2f}'.format(alpha, mse))
|
|
||||||
@@ -309,7 +309,7 @@
|
|||||||
"conda_env = Environment(\"conda-env\")\n",
|
"conda_env = Environment(\"conda-env\")\n",
|
||||||
"conda_env.python.conda_dependencies = CondaDependencies.create(pip_packages=['scikit-learn',\n",
|
"conda_env.python.conda_dependencies = CondaDependencies.create(pip_packages=['scikit-learn',\n",
|
||||||
" 'azureml-sdk',\n",
|
" 'azureml-sdk',\n",
|
||||||
" 'azureml-dataprep[pandas,fuse]>=1.1.21'])"
|
" 'azureml-dataset-runtime[pandas,fuse]'])"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -1,33 +0,0 @@
|
|||||||
import pickle
|
|
||||||
import json
|
|
||||||
import numpy as np
|
|
||||||
from sklearn.linear_model import Ridge
|
|
||||||
from azureml.core.model import Model
|
|
||||||
# sklearn.externals.joblib is removed in 0.23
|
|
||||||
try:
|
|
||||||
from sklearn.externals import joblib
|
|
||||||
except ImportError:
|
|
||||||
import joblib
|
|
||||||
|
|
||||||
|
|
||||||
def init():
|
|
||||||
global model
|
|
||||||
# note here "best_model" is the name of the model registered under the workspace
|
|
||||||
# this call should return the path to the model.pkl file on the local disk.
|
|
||||||
model_path = Model.get_model_path(model_name='best_model')
|
|
||||||
# deserialize the model file back into a sklearn model
|
|
||||||
model = joblib.load(model_path)
|
|
||||||
|
|
||||||
|
|
||||||
# note you can pass in multiple rows for scoring
|
|
||||||
def run(raw_data):
|
|
||||||
try:
|
|
||||||
data = json.loads(raw_data)['data']
|
|
||||||
data = np.array(data)
|
|
||||||
result = model.predict(data)
|
|
||||||
|
|
||||||
# you can return any data type as long as it is JSON-serializable
|
|
||||||
return result.tolist()
|
|
||||||
except Exception as e:
|
|
||||||
result = str(e)
|
|
||||||
return result
|
|
||||||
@@ -1,721 +0,0 @@
|
|||||||
{
|
|
||||||
"cells": [
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"Copyright (c) Microsoft Corporation. All rights reserved.\n",
|
|
||||||
"\n",
|
|
||||||
"Licensed under the MIT License."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
""
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"# Train and deploy a model\n",
|
|
||||||
"_**Create and deploy a model directly from a notebook**_\n",
|
|
||||||
"\n",
|
|
||||||
"---\n",
|
|
||||||
"---\n",
|
|
||||||
"\n",
|
|
||||||
"## Contents\n",
|
|
||||||
"1. [Introduction](#Introduction)\n",
|
|
||||||
"1. [Setup](#Setup)\n",
|
|
||||||
"1. [Data](#Data)\n",
|
|
||||||
"1. [Train](#Train)\n",
|
|
||||||
" 1. Viewing run results\n",
|
|
||||||
" 1. Simple parameter sweep\n",
|
|
||||||
" 1. Viewing experiment results\n",
|
|
||||||
" 1. Select the best model\n",
|
|
||||||
"1. [Deploy](#Deploy)\n",
|
|
||||||
" 1. Register the model\n",
|
|
||||||
" 1. Create a scoring file\n",
|
|
||||||
" 1. Describe your environment\n",
|
|
||||||
" 1. Descrice your target compute\n",
|
|
||||||
" 1. Deploy your webservice\n",
|
|
||||||
" 1. Test your webservice\n",
|
|
||||||
" 1. Clean up\n",
|
|
||||||
"1. [Next Steps](#nextsteps)\n",
|
|
||||||
"\n",
|
|
||||||
"---\n",
|
|
||||||
"\n",
|
|
||||||
"## Introduction\n",
|
|
||||||
"Azure Machine Learning provides capabilities to control all aspects of model training and deployment directly from a notebook using the AML Python SDK. In this notebook we will\n",
|
|
||||||
"* connect to our AML Workspace\n",
|
|
||||||
"* create an experiment that contains multiple runs with tracked metrics\n",
|
|
||||||
"* choose the best model created across all runs\n",
|
|
||||||
"* deploy that model as a service\n",
|
|
||||||
"\n",
|
|
||||||
"In the end we will have a model deployed as a web service which we can call from an HTTP endpoint"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"---\n",
|
|
||||||
"\n",
|
|
||||||
"## Setup\n",
|
|
||||||
"If you are using an Azure Machine Learning Notebook VM, you are all set. Otherwise, go through the [configuration](../../../configuration.ipynb) Notebook first if you haven't already to establish your connection to the AzureML Workspace. From the configuration, the important sections are the workspace configuration and ACI regristration.\n",
|
|
||||||
"\n",
|
|
||||||
"We will also need the following libraries install to our conda environment. If these are not installed, use the following command to do so and restart the notebook.\n",
|
|
||||||
"```shell\n",
|
|
||||||
"(myenv) $ conda install -y matplotlib tqdm scikit-learn\n",
|
|
||||||
"```\n",
|
|
||||||
"\n",
|
|
||||||
"For this notebook we need the Azure ML SDK and access to our workspace. The following cell imports the SDK, checks the version, and accesses our already configured AzureML workspace."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {
|
|
||||||
"tags": [
|
|
||||||
"install"
|
|
||||||
],
|
|
||||||
"name": "load_ws",
|
|
||||||
"msdoc": "how-to-track-experiments.md"
|
|
||||||
},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"import azureml.core\n",
|
|
||||||
"from azureml.core import Experiment, Workspace\n",
|
|
||||||
"\n",
|
|
||||||
"# Check core SDK version number\n",
|
|
||||||
"print(\"This notebook was created using version 1.0.2 of the Azure ML SDK\")\n",
|
|
||||||
"print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")\n",
|
|
||||||
"print(\"\")\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"ws = Workspace.from_config()\n",
|
|
||||||
"print('Workspace name: ' + ws.name, \n",
|
|
||||||
" 'Azure region: ' + ws.location, \n",
|
|
||||||
" 'Subscription id: ' + ws.subscription_id, \n",
|
|
||||||
" 'Resource group: ' + ws.resource_group, sep='\\n')"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"---\n",
|
|
||||||
"\n",
|
|
||||||
"## Data\n",
|
|
||||||
"We will use the diabetes dataset for this experiement, a well-known small dataset that comes with scikit-learn. This cell loads the dataset and splits it into random training and testing sets.\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {
|
|
||||||
"name": "load_data",
|
|
||||||
"msdoc": "how-to-track-experiments.md"
|
|
||||||
},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"from sklearn.datasets import load_diabetes\n",
|
|
||||||
"from sklearn.linear_model import Ridge\n",
|
|
||||||
"from sklearn.metrics import mean_squared_error\n",
|
|
||||||
"from sklearn.model_selection import train_test_split\n",
|
|
||||||
"from sklearn.externals import joblib\n",
|
|
||||||
"\n",
|
|
||||||
"X, y = load_diabetes(return_X_y = True)\n",
|
|
||||||
"columns = ['age', 'gender', 'bmi', 'bp', 's1', 's2', 's3', 's4', 's5', 's6']\n",
|
|
||||||
"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)\n",
|
|
||||||
"data = {\n",
|
|
||||||
" \"train\":{\"X\": X_train, \"y\": y_train}, \n",
|
|
||||||
" \"test\":{\"X\": X_test, \"y\": y_test}\n",
|
|
||||||
"}\n",
|
|
||||||
"\n",
|
|
||||||
"print (\"Data contains\", len(data['train']['X']), \"training samples and\",len(data['test']['X']), \"test samples\")"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"---\n",
|
|
||||||
"## Train\n",
|
|
||||||
"\n",
|
|
||||||
"Let's use scikit-learn to train a simple Ridge regression model. We use AML to record interesting information about the model in an Experiment. An Experiment contains a series of trials called Runs. During this trial we use AML in the following way:\n",
|
|
||||||
"* We access an experiment from our AML workspace by name, which will be created if it doesn't exist\n",
|
|
||||||
"* We use `start_logging` to create a new run in this experiment\n",
|
|
||||||
"* We use `run.log()` to record a parameter, alpha, and an accuracy measure - the Mean Squared Error (MSE) to the run. We will be able to review and compare these measures in the Azure Portal at a later time.\n",
|
|
||||||
"* We store the resulting model in the **outputs** directory, which is automatically captured by AML when the run is complete.\n",
|
|
||||||
"* We use `run.complete()` to indicate that the run is over and results can be captured and finalized"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {
|
|
||||||
"tags": [
|
|
||||||
"local run",
|
|
||||||
"outputs upload"
|
|
||||||
],
|
|
||||||
"name": "create_experiment",
|
|
||||||
"msdoc": "how-to-track-experiments.md"
|
|
||||||
},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# Get an experiment object from Azure Machine Learning\n",
|
|
||||||
"experiment = Experiment(workspace=ws, name=\"train-within-notebook\")\n",
|
|
||||||
"\n",
|
|
||||||
"# Create a run object in the experiment\n",
|
|
||||||
"run = experiment.start_logging()\n",
|
|
||||||
"# Log the algorithm parameter alpha to the run\n",
|
|
||||||
"run.log('alpha', 0.03)\n",
|
|
||||||
"\n",
|
|
||||||
"# Create, fit, and test the scikit-learn Ridge regression model\n",
|
|
||||||
"regression_model = Ridge(alpha=0.03)\n",
|
|
||||||
"regression_model.fit(data['train']['X'], data['train']['y'])\n",
|
|
||||||
"preds = regression_model.predict(data['test']['X'])\n",
|
|
||||||
"\n",
|
|
||||||
"# Output the Mean Squared Error to the notebook and to the run\n",
|
|
||||||
"print('Mean Squared Error is', mean_squared_error(data['test']['y'], preds))\n",
|
|
||||||
"run.log('mse', mean_squared_error(data['test']['y'], preds))\n",
|
|
||||||
"\n",
|
|
||||||
"# Save the model to the outputs directory for capture\n",
|
|
||||||
"model_file_name = 'outputs/model.pkl'\n",
|
|
||||||
"\n",
|
|
||||||
"joblib.dump(value = regression_model, filename = model_file_name)\n",
|
|
||||||
"\n",
|
|
||||||
"# upload the model file explicitly into artifacts \n",
|
|
||||||
"run.upload_file(name = model_file_name, path_or_stream = model_file_name)\n",
|
|
||||||
"\n",
|
|
||||||
"# Complete the run\n",
|
|
||||||
"run.complete()"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### Viewing run results\n",
|
|
||||||
"Azure Machine Learning stores all the details about the run in the Azure cloud. Let's access those details by retrieving a link to the run using the default run output. Clicking on the resulting link will take you to an interactive page presenting all run information."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"run"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### Simple parameter sweep\n",
|
|
||||||
"Now let's take the same concept from above and modify the **alpha** parameter. For each value of alpha we will create a run that will store metrics and the resulting model. In the end we can use the captured run history to determine which model was the best for us to deploy. \n",
|
|
||||||
"\n",
|
|
||||||
"Note that by using `with experiment.start_logging() as run` AML will automatically call `run.complete()` at the end of each loop.\n",
|
|
||||||
"\n",
|
|
||||||
"This example also uses the **tqdm** library to provide a thermometer feedback"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"import numpy as np\n",
|
|
||||||
"from tqdm import tqdm\n",
|
|
||||||
"\n",
|
|
||||||
"# list of numbers from 0 to 1.0 with a 0.05 interval\n",
|
|
||||||
"alphas = np.arange(0.0, 1.0, 0.05)\n",
|
|
||||||
"\n",
|
|
||||||
"# try a bunch of alpha values in a Linear Regression (Ridge) model\n",
|
|
||||||
"for alpha in tqdm(alphas):\n",
|
|
||||||
" # create a bunch of runs, each train a model with a different alpha value\n",
|
|
||||||
" with experiment.start_logging() as run:\n",
|
|
||||||
" # Use Ridge algorithm to build a regression model\n",
|
|
||||||
" regression_model = Ridge(alpha=alpha)\n",
|
|
||||||
" regression_model.fit(X=data[\"train\"][\"X\"], y=data[\"train\"][\"y\"])\n",
|
|
||||||
" preds = regression_model.predict(X=data[\"test\"][\"X\"])\n",
|
|
||||||
" mse = mean_squared_error(y_true=data[\"test\"][\"y\"], y_pred=preds)\n",
|
|
||||||
"\n",
|
|
||||||
" # log alpha, mean_squared_error and feature names in run history\n",
|
|
||||||
" run.log(name=\"alpha\", value=alpha)\n",
|
|
||||||
" run.log(name=\"mse\", value=mse)\n",
|
|
||||||
"\n",
|
|
||||||
" # Save the model to the outputs directory for capture\n",
|
|
||||||
" joblib.dump(value=regression_model, filename='outputs/model.pkl')\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### Viewing experiment results\n",
|
|
||||||
"Similar to viewing the run, we can also view the entire experiment. The experiment report view in the Azure portal lets us view all the runs in a table, and also allows us to customize charts. This way, we can see how the alpha parameter impacts the quality of the model"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# now let's take a look at the experiment in Azure portal.\n",
|
|
||||||
"experiment"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### Select the best model \n",
|
|
||||||
"Now that we've created many runs with different parameters, we need to determine which model is the best for deployment. For this, we will iterate over the set of runs. From each run we will take the *run id* using the `id` property, and examine the metrics by calling `run.get_metrics()`. \n",
|
|
||||||
"\n",
|
|
||||||
"Since each run may be different, we do need to check if the run has the metric that we are looking for, in this case, **mse**. To find the best run, we create a dictionary mapping the run id's to the metrics.\n",
|
|
||||||
"\n",
|
|
||||||
"Finally, we use the `tag` method to mark the best run to make it easier to find later. "
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"runs = {}\n",
|
|
||||||
"run_metrics = {}\n",
|
|
||||||
"\n",
|
|
||||||
"# Create dictionaries containing the runs and the metrics for all runs containing the 'mse' metric\n",
|
|
||||||
"for r in tqdm(experiment.get_runs()):\n",
|
|
||||||
" metrics = r.get_metrics()\n",
|
|
||||||
" if 'mse' in metrics.keys():\n",
|
|
||||||
" runs[r.id] = r\n",
|
|
||||||
" run_metrics[r.id] = metrics\n",
|
|
||||||
"\n",
|
|
||||||
"# Find the run with the best (lowest) mean squared error and display the id and metrics\n",
|
|
||||||
"best_run_id = min(run_metrics, key = lambda k: run_metrics[k]['mse'])\n",
|
|
||||||
"best_run = runs[best_run_id]\n",
|
|
||||||
"print('Best run is:', best_run_id)\n",
|
|
||||||
"print('Metrics:', run_metrics[best_run_id])\n",
|
|
||||||
"\n",
|
|
||||||
"# Tag the best run for identification later\n",
|
|
||||||
"best_run.tag(\"Best Run\")"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"---\n",
|
|
||||||
"## Deploy\n",
|
|
||||||
"Now that we have trained a set of models and identified the run containing the best model, we want to deploy the model for real time inference. The process of deploying a model involves\n",
|
|
||||||
"* registering a model in your workspace\n",
|
|
||||||
"* creating a scoring file containing init and run methods\n",
|
|
||||||
"* creating an environment dependency file describing packages necessary for your scoring file\n",
|
|
||||||
"* deploying the model and packages as a web service"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### Register a model\n",
|
|
||||||
"We have already identified which run contains the \"best model\" by our evaluation criteria. Each run has a file structure associated with it that contains various files collected during the run. Since a run can have many outputs we need to tell AML which file from those outputs represents the model that we want to use for our deployment. We can use the `run.get_file_names()` method to list the files associated with the run, and then use the `run.register_model()` method to place the model in the workspace's model registry.\n",
|
|
||||||
"\n",
|
|
||||||
"When using `run.register_model()` we supply a `model_name` that is meaningful for our scenario and the `model_path` of the model relative to the run. In this case, the model path is what is returned from `run.get_file_names()`"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {
|
|
||||||
"tags": [
|
|
||||||
"query history"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# View the files in the run\n",
|
|
||||||
"for f in best_run.get_file_names():\n",
|
|
||||||
" print(f)\n",
|
|
||||||
" \n",
|
|
||||||
"# Register the model with the workspace\n",
|
|
||||||
"model = best_run.register_model(model_name='best_model', model_path='outputs/model.pkl')"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"Once a model is registered, it is accessible from the list of models on the AML workspace. If you register models with the same name multiple times, AML keeps a version history of those models for you. The `Model.list()` lists all models in a workspace, and can be filtered by name, tags, or model properties. "
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {
|
|
||||||
"tags": [
|
|
||||||
"register model from history"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# Find all models called \"best_model\" and display their version numbers\n",
|
|
||||||
"from azureml.core.model import Model\n",
|
|
||||||
"models = Model.list(ws, name='best_model')\n",
|
|
||||||
"for m in models:\n",
|
|
||||||
" print(m.name, m.version)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### Create a scoring file\n",
|
|
||||||
"\n",
|
|
||||||
"Since your model file can essentially be anything you want it to be, you need to supply a scoring script that can load your model and then apply the model to new data. This script is your 'scoring file'. This scoring file is a python program containing, at a minimum, two methods `init()` and `run()`. The `init()` method is called once when your deployment is started so you can load your model and any other required objects. This method uses the `get_model_path` function to locate the registered model inside the docker container. The `run()` method is called interactively when the web service is called with one or more data samples to predict.\n",
|
|
||||||
"\n",
|
|
||||||
"The scoring file used for this exercise is [here](score.py). \n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### Describe your environment\n",
|
|
||||||
"\n",
|
|
||||||
"Each modelling process may require a unique set of packages. Therefore we need to create an environment object describing the dependencies. \n",
|
|
||||||
"\n",
|
|
||||||
"Next we create an inference configuration using this environment object and the scoring script that we created previously."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"from azureml.core.conda_dependencies import CondaDependencies\n",
|
|
||||||
"from azureml.core.environment import Environment\n",
|
|
||||||
"from azureml.core.model import InferenceConfig\n",
|
|
||||||
"\n",
|
|
||||||
"env = Environment('deploytocloudenv')\n",
|
|
||||||
"env.python.conda_dependencies = CondaDependencies.create(conda_packages=['scikit-learn'],pip_packages=['azureml-defaults'])\n",
|
|
||||||
"inference_config = InferenceConfig(entry_script=\"score.py\", environment=env)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### Describe your target compute\n",
|
|
||||||
"In addition to the inference configuration, we also need to describe the type of compute we want to allocate for our webservice. In in this example we are using an [Azure Container Instance](https://azure.microsoft.com/en-us/services/container-instances/) which is a good choice for quick and cost-effective dev/test deployment scenarios. ACI instances require the number of cores you want to run and memory you need. Tags and descriptions are available for you to identify the instances in AML when viewing the Compute tab in the AML Portal.\n",
|
|
||||||
"\n",
|
|
||||||
"For production workloads, it is better to use [Azure Kubernentes Service (AKS)](https://azure.microsoft.com/en-us/services/kubernetes-service/) instead. Try [this notebook](https://github.com/Azure/MachineLearningNotebooks/blob/master/how-to-use-azureml/deployment/production-deploy-to-aks/production-deploy-to-aks.ipynb) to see how that can be done from Azure ML.\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {
|
|
||||||
"tags": [
|
|
||||||
"deploy service",
|
|
||||||
"aci"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"from azureml.core.webservice import AciWebservice\n",
|
|
||||||
"\n",
|
|
||||||
"aciconfig = AciWebservice.deploy_configuration(cpu_cores=1, \n",
|
|
||||||
" memory_gb=1, \n",
|
|
||||||
" tags={'sample name': 'AML 101'}, \n",
|
|
||||||
" description='This is a great example.')"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### Deploy your webservice\n",
|
|
||||||
"The final step to deploying your webservice is to call `Model.deploy()`. This function uses the deployment and inference configurations created above to perform the following:\n",
|
|
||||||
"* Build a docker image\n",
|
|
||||||
"* Deploy to the docker image to an Azure Container Instance\n",
|
|
||||||
"* Copy your model files to the Azure Container Instance\n",
|
|
||||||
"* Call the `init()` function in your scoring file\n",
|
|
||||||
"* Provide an HTTP endpoint for scoring calls\n",
|
|
||||||
"\n",
|
|
||||||
"The `Model.deploy` method requires the following parameters\n",
|
|
||||||
"* `workspace` - the workspace containing the service\n",
|
|
||||||
"* `name` - a unique named used to identify the service in the workspace\n",
|
|
||||||
"* `models` - an array of models to be deployed into the container\n",
|
|
||||||
"* `inference_config` - a configuration object describing the image environment\n",
|
|
||||||
"* `deployment_config` - a configuration object describing the compute type\n",
|
|
||||||
" \n",
|
|
||||||
"**Note:** The web service creation can take several minutes. "
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {
|
|
||||||
"tags": [
|
|
||||||
"deploy service",
|
|
||||||
"aci"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"%%time\n",
|
|
||||||
"from azureml.core.model import Model\n",
|
|
||||||
"from azureml.core.webservice import Webservice\n",
|
|
||||||
"\n",
|
|
||||||
"# Create the webservice using all of the precreated configurations and our best model\n",
|
|
||||||
"service = Model.deploy(workspace=ws,\n",
|
|
||||||
" name='my-aci-svc',\n",
|
|
||||||
" models=[model],\n",
|
|
||||||
" inference_config=inference_config,\n",
|
|
||||||
" deployment_config=aciconfig)\n",
|
|
||||||
"\n",
|
|
||||||
"# Wait for the service deployment to complete while displaying log output\n",
|
|
||||||
"service.wait_for_deployment(show_output=True)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"\n",
|
|
||||||
"### Test your webservice"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"Now that your web service is runing you can send JSON data directly to the service using the `run` method. This cell pulls the first test sample from the original dataset into JSON and then sends it to the service."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {
|
|
||||||
"tags": [
|
|
||||||
"deploy service",
|
|
||||||
"aci"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"import json\n",
|
|
||||||
"\n",
|
|
||||||
"service = ws.webservices['my-aci-svc']\n",
|
|
||||||
"\n",
|
|
||||||
"# scrape the first row from the test set.\n",
|
|
||||||
"test_samples = json.dumps({\"data\": X_test[0:1, :].tolist()})\n",
|
|
||||||
"\n",
|
|
||||||
"#score on our service\n",
|
|
||||||
"service.run(input_data = test_samples)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"This cell shows how you can send multiple rows to the webservice at once. It then calculates the residuals - that is, the errors - by subtracting out the actual values from the results. These residuals are used later to show a plotted result."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {
|
|
||||||
"tags": [
|
|
||||||
"deploy service",
|
|
||||||
"aci"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# score the entire test set.\n",
|
|
||||||
"test_samples = json.dumps({'data': X_test.tolist()})\n",
|
|
||||||
"\n",
|
|
||||||
"result = service.run(input_data = test_samples)\n",
|
|
||||||
"residual = result - y_test"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"This cell shows how you can use the `service.scoring_uri` property to access the HTTP endpoint of the service and call it using standard POST operations."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {
|
|
||||||
"tags": [
|
|
||||||
"deploy service",
|
|
||||||
"aci"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"import requests\n",
|
|
||||||
"\n",
|
|
||||||
"# use the first row from the test set again\n",
|
|
||||||
"test_samples = json.dumps({\"data\": X_test[0:1, :].tolist()})\n",
|
|
||||||
"\n",
|
|
||||||
"# create the required header\n",
|
|
||||||
"headers = {'Content-Type':'application/json'}\n",
|
|
||||||
"\n",
|
|
||||||
"# post the request to the service and display the result\n",
|
|
||||||
"resp = requests.post(service.scoring_uri, test_samples, headers = headers)\n",
|
|
||||||
"print(resp.text)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### Residual graph\n",
|
|
||||||
"One way to understand the behavior of your model is to see how the data performs against data with known results. This cell uses matplotlib to create a histogram of the residual values, or errors, created from scoring the test samples.\n",
|
|
||||||
"\n",
|
|
||||||
"A good model should have residual values that cluster around 0 - that is, no error. Observing the resulting histogram can also show you if the model is skewed in any particular direction."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"%matplotlib inline\n",
|
|
||||||
"import matplotlib.pyplot as plt\n",
|
|
||||||
"\n",
|
|
||||||
"f, (a0, a1) = plt.subplots(1, 2, gridspec_kw={'width_ratios':[3, 1], 'wspace':0, 'hspace': 0})\n",
|
|
||||||
"f.suptitle('Residual Values', fontsize = 18)\n",
|
|
||||||
"\n",
|
|
||||||
"f.set_figheight(6)\n",
|
|
||||||
"f.set_figwidth(14)\n",
|
|
||||||
"\n",
|
|
||||||
"a0.plot(residual, 'bo', alpha=0.4)\n",
|
|
||||||
"a0.plot([0,90], [0,0], 'r', lw=2)\n",
|
|
||||||
"a0.set_ylabel('residue values', fontsize=14)\n",
|
|
||||||
"a0.set_xlabel('test data set', fontsize=14)\n",
|
|
||||||
"\n",
|
|
||||||
"a1.hist(residual, orientation='horizontal', color='blue', bins=10, histtype='step')\n",
|
|
||||||
"a1.hist(residual, orientation='horizontal', color='blue', alpha=0.2, bins=10)\n",
|
|
||||||
"a1.set_yticklabels([])\n",
|
|
||||||
"\n",
|
|
||||||
"plt.show()"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### Clean up"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"Delete the ACI instance to stop the compute and any associated billing."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {
|
|
||||||
"tags": [
|
|
||||||
"deploy service",
|
|
||||||
"aci"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"%%time\n",
|
|
||||||
"service.delete()"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"<a id='nextsteps'></a>\n",
|
|
||||||
"## Next Steps"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"In this example, you created a series of models inside the notebook using local data, stored them inside an AML experiment, found the best one and deployed it as a live service! From here you can continue to use Azure Machine Learning in this regard to run your own experiments and deploy your own models, or you can expand into further capabilities of AML!\n",
|
|
||||||
"\n",
|
|
||||||
"If you have a model that is difficult to process locally, either because the data is remote or the model is large, try the [train-on-remote-vm](../train-on-remote-vm) notebook to learn about submitting remote jobs.\n",
|
|
||||||
"\n",
|
|
||||||
"If you want to take advantage of multiple cloud machines to perform large parameter sweeps try the [train-hyperparameter-tune-deploy-with-pytorch](../../training-with-deep-learning/train-hyperparameter-tune-deploy-with-pytorch\n",
|
|
||||||
") sample.\n",
|
|
||||||
"\n",
|
|
||||||
"If you want to deploy models to a production cluster try the [production-deploy-to-aks](../../deployment/production-deploy-to-aks\n",
|
|
||||||
") notebook."
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"authors": [
|
|
||||||
{
|
|
||||||
"name": "roastala"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"category": "tutorial",
|
|
||||||
"compute": [
|
|
||||||
"Local"
|
|
||||||
],
|
|
||||||
"datasets": [
|
|
||||||
"Diabetes"
|
|
||||||
],
|
|
||||||
"deployment": [
|
|
||||||
"Azure Container Instance"
|
|
||||||
],
|
|
||||||
"exclude_from_index": false,
|
|
||||||
"framework": [
|
|
||||||
"None"
|
|
||||||
],
|
|
||||||
"friendly_name": "Train and deploy a model using Python SDK",
|
|
||||||
"index_order": 1,
|
|
||||||
"kernelspec": {
|
|
||||||
"display_name": "Python 3.6",
|
|
||||||
"language": "python",
|
|
||||||
"name": "python36"
|
|
||||||
},
|
|
||||||
"language_info": {
|
|
||||||
"codemirror_mode": {
|
|
||||||
"name": "ipython",
|
|
||||||
"version": 3
|
|
||||||
},
|
|
||||||
"file_extension": ".py",
|
|
||||||
"mimetype": "text/x-python",
|
|
||||||
"name": "python",
|
|
||||||
"nbconvert_exporter": "python",
|
|
||||||
"pygments_lexer": "ipython3",
|
|
||||||
"version": "3.6.5"
|
|
||||||
},
|
|
||||||
"tags": [
|
|
||||||
"None"
|
|
||||||
],
|
|
||||||
"task": "Training and deploying a model from a notebook"
|
|
||||||
},
|
|
||||||
"nbformat": 4,
|
|
||||||
"nbformat_minor": 2
|
|
||||||
}
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
name: train-within-notebook
|
|
||||||
dependencies:
|
|
||||||
- tqdm
|
|
||||||
- scikit-learn
|
|
||||||
- matplotlib
|
|
||||||
- pip:
|
|
||||||
- azureml-sdk
|
|
||||||
- azureml-widgets
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user