From df332c20b7a348f0af248b9ba44144be666aaca4 Mon Sep 17 00:00:00 2001 From: Arnav Das Date: Sat, 16 Feb 2019 22:31:44 +0530 Subject: [PATCH] Assignment-5 Completed!!!(Hopefully) --- feature_crosses.ipynb | 959 ++++++++++ feature_sets.ipynb | 1193 +++++++++++++ first_steps_with_tensor_flow.ipynb | 971 ++++++++++ improving_neural_net_performance.ipynb | 1058 +++++++++++ intro_to_neural_nets.ipynb | 619 +++++++ intro_to_pandas.ipynb | 672 +++++++ logistic_regression.ipynb | 1554 +++++++++++++++++ ...classification_of_handwritten_digits.ipynb | 1120 ++++++++++++ sparsity_and_l1_regularization.ipynb | 598 +++++++ synthetic_features_and_outliers.ipynb | 604 +++++++ validation.ipynb | 1538 ++++++++++++++++ 11 files changed, 10886 insertions(+) create mode 100644 feature_crosses.ipynb create mode 100644 feature_sets.ipynb create mode 100644 first_steps_with_tensor_flow.ipynb create mode 100644 improving_neural_net_performance.ipynb create mode 100644 intro_to_neural_nets.ipynb create mode 100644 intro_to_pandas.ipynb create mode 100644 logistic_regression.ipynb create mode 100644 multi_class_classification_of_handwritten_digits.ipynb create mode 100644 sparsity_and_l1_regularization.ipynb create mode 100644 synthetic_features_and_outliers.ipynb create mode 100644 validation.ipynb diff --git a/feature_crosses.ipynb b/feature_crosses.ipynb new file mode 100644 index 0000000..5cc099d --- /dev/null +++ b/feature_crosses.ipynb @@ -0,0 +1,959 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "feature_crosses.ipynb", + "version": "0.3.2", + "provenance": [], + "collapsed_sections": [ + "JndnmDMp66FL", + "ZTDHHM61NPTw", + "0i7vGo9PTaZl" + ], + "include_colab_link": true + }, + "kernelspec": { + "name": "python2", + "display_name": "Python 2" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "metadata": { + "id": "JndnmDMp66FL", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "#### Copyright 2017 Google LLC." + ] + }, + { + "metadata": { + "id": "hMqWDc_m6rUC", + "colab_type": "code", + "cellView": "both", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "g4T-_IsVbweU", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Feature Crosses" + ] + }, + { + "metadata": { + "id": "F7dke6skIK-k", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "**Learning Objectives:**\n", + " * Improve a linear regression model with the addition of additional synthetic features (this is a continuation of the previous exercise)\n", + " * Use an input function to convert pandas `DataFrame` objects to `Tensors` and invoke the input function in `fit()` and `predict()` operations\n", + " * Use the FTRL optimization algorithm for model training\n", + " * Create new synthetic features through one-hot encoding, binning, and feature crosses" + ] + }, + { + "metadata": { + "id": "NS_fcQRd8B97", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Setup" + ] + }, + { + "metadata": { + "id": "4IdzD8IdIK-l", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "First, as we've done in previous exercises, let's define the input and create the data-loading code." + ] + }, + { + "metadata": { + "id": "CsfdiLiDIK-n", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "from __future__ import print_function\n", + "\n", + "import math\n", + "\n", + "from IPython import display\n", + "from matplotlib import cm\n", + "from matplotlib import gridspec\n", + "from matplotlib import pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "from sklearn import metrics\n", + "import tensorflow as tf\n", + "from tensorflow.python.data import Dataset\n", + "\n", + "tf.logging.set_verbosity(tf.logging.ERROR)\n", + "pd.options.display.max_rows = 10\n", + "pd.options.display.float_format = '{:.1f}'.format\n", + "\n", + "california_housing_dataframe = pd.read_csv(\"https://download.mlcc.google.com/mledu-datasets/california_housing_train.csv\", sep=\",\")\n", + "\n", + "california_housing_dataframe = california_housing_dataframe.reindex(\n", + " np.random.permutation(california_housing_dataframe.index))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "10rhoflKIK-s", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def preprocess_features(california_housing_dataframe):\n", + " \"\"\"Prepares input features from California housing data set.\n", + "\n", + " Args:\n", + " california_housing_dataframe: A Pandas DataFrame expected to contain data\n", + " from the California housing data set.\n", + " Returns:\n", + " A DataFrame that contains the features to be used for the model, including\n", + " synthetic features.\n", + " \"\"\"\n", + " selected_features = california_housing_dataframe[\n", + " [\"latitude\",\n", + " \"longitude\",\n", + " \"housing_median_age\",\n", + " \"total_rooms\",\n", + " \"total_bedrooms\",\n", + " \"population\",\n", + " \"households\",\n", + " \"median_income\"]]\n", + " processed_features = selected_features.copy()\n", + " # Create a synthetic feature.\n", + " processed_features[\"rooms_per_person\"] = (\n", + " california_housing_dataframe[\"total_rooms\"] /\n", + " california_housing_dataframe[\"population\"])\n", + " return processed_features\n", + "\n", + "def preprocess_targets(california_housing_dataframe):\n", + " \"\"\"Prepares target features (i.e., labels) from California housing data set.\n", + "\n", + " Args:\n", + " california_housing_dataframe: A Pandas DataFrame expected to contain data\n", + " from the California housing data set.\n", + " Returns:\n", + " A DataFrame that contains the target feature.\n", + " \"\"\"\n", + " output_targets = pd.DataFrame()\n", + " # Scale the target to be in units of thousands of dollars.\n", + " output_targets[\"median_house_value\"] = (\n", + " california_housing_dataframe[\"median_house_value\"] / 1000.0)\n", + " return output_targets" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "ufplEkjN8KUp", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# Choose the first 12000 (out of 17000) examples for training.\n", + "training_examples = preprocess_features(california_housing_dataframe.head(12000))\n", + "training_targets = preprocess_targets(california_housing_dataframe.head(12000))\n", + "\n", + "# Choose the last 5000 (out of 17000) examples for validation.\n", + "validation_examples = preprocess_features(california_housing_dataframe.tail(5000))\n", + "validation_targets = preprocess_targets(california_housing_dataframe.tail(5000))\n", + "\n", + "# Double-check that we've done the right thing.\n", + "print(\"Training examples summary:\")\n", + "display.display(training_examples.describe())\n", + "print(\"Validation examples summary:\")\n", + "display.display(validation_examples.describe())\n", + "\n", + "print(\"Training targets summary:\")\n", + "display.display(training_targets.describe())\n", + "print(\"Validation targets summary:\")\n", + "display.display(validation_targets.describe())" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "oJlrB4rJ_2Ma", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def construct_feature_columns(input_features):\n", + " \"\"\"Construct the TensorFlow Feature Columns.\n", + "\n", + " Args:\n", + " input_features: The names of the numerical input features to use.\n", + " Returns:\n", + " A set of feature columns\n", + " \"\"\"\n", + " return set([tf.feature_column.numeric_column(my_feature)\n", + " for my_feature in input_features])" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "NBxoAfp2AcB6", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def my_input_fn(features, targets, batch_size=1, shuffle=True, num_epochs=None):\n", + " \"\"\"Trains a linear regression model.\n", + " \n", + " Args:\n", + " features: pandas DataFrame of features\n", + " targets: pandas DataFrame of targets\n", + " batch_size: Size of batches to be passed to the model\n", + " shuffle: True or False. Whether to shuffle the data.\n", + " num_epochs: Number of epochs for which data should be repeated. None = repeat indefinitely\n", + " Returns:\n", + " Tuple of (features, labels) for next data batch\n", + " \"\"\"\n", + " \n", + " # Convert pandas data into a dict of np arrays.\n", + " features = {key:np.array(value) for key,value in dict(features).items()} \n", + " \n", + " # Construct a dataset, and configure batching/repeating.\n", + " ds = Dataset.from_tensor_slices((features,targets)) # warning: 2GB limit\n", + " ds = ds.batch(batch_size).repeat(num_epochs)\n", + " \n", + " # Shuffle the data, if specified.\n", + " if shuffle:\n", + " ds = ds.shuffle(10000)\n", + " \n", + " # Return the next batch of data.\n", + " features, labels = ds.make_one_shot_iterator().get_next()\n", + " return features, labels" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "hweDyy31LBsV", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## FTRL Optimization Algorithm\n", + "\n", + "High dimensional linear models benefit from using a variant of gradient-based optimization called FTRL. This algorithm has the benefit of scaling the learning rate differently for different coefficients, which can be useful if some features rarely take non-zero values (it also is well suited to support L1 regularization). We can apply FTRL using the [FtrlOptimizer](https://www.tensorflow.org/api_docs/python/tf/train/FtrlOptimizer)." + ] + }, + { + "metadata": { + "id": "S0SBf1X1IK_O", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def train_model(\n", + " learning_rate,\n", + " steps,\n", + " batch_size,\n", + " feature_columns,\n", + " training_examples,\n", + " training_targets,\n", + " validation_examples,\n", + " validation_targets):\n", + " \"\"\"Trains a linear regression model.\n", + " \n", + " In addition to training, this function also prints training progress information,\n", + " as well as a plot of the training and validation loss over time.\n", + " \n", + " Args:\n", + " learning_rate: A `float`, the learning rate.\n", + " steps: A non-zero `int`, the total number of training steps. A training step\n", + " consists of a forward and backward pass using a single batch.\n", + " feature_columns: A `set` specifying the input feature columns to use.\n", + " training_examples: A `DataFrame` containing one or more columns from\n", + " `california_housing_dataframe` to use as input features for training.\n", + " training_targets: A `DataFrame` containing exactly one column from\n", + " `california_housing_dataframe` to use as target for training.\n", + " validation_examples: A `DataFrame` containing one or more columns from\n", + " `california_housing_dataframe` to use as input features for validation.\n", + " validation_targets: A `DataFrame` containing exactly one column from\n", + " `california_housing_dataframe` to use as target for validation.\n", + " \n", + " Returns:\n", + " A `LinearRegressor` object trained on the training data.\n", + " \"\"\"\n", + "\n", + " periods = 10\n", + " steps_per_period = steps / periods\n", + "\n", + " # Create a linear regressor object.\n", + " my_optimizer = tf.train.FtrlOptimizer(learning_rate=learning_rate)\n", + " my_optimizer = tf.contrib.estimator.clip_gradients_by_norm(my_optimizer, 5.0)\n", + " linear_regressor = tf.estimator.LinearRegressor(\n", + " feature_columns=feature_columns,\n", + " optimizer=my_optimizer\n", + " )\n", + " \n", + " training_input_fn = lambda: my_input_fn(training_examples, \n", + " training_targets[\"median_house_value\"], \n", + " batch_size=batch_size)\n", + " predict_training_input_fn = lambda: my_input_fn(training_examples, \n", + " training_targets[\"median_house_value\"], \n", + " num_epochs=1, \n", + " shuffle=False)\n", + " predict_validation_input_fn = lambda: my_input_fn(validation_examples, \n", + " validation_targets[\"median_house_value\"], \n", + " num_epochs=1, \n", + " shuffle=False)\n", + "\n", + " # Train the model, but do so inside a loop so that we can periodically assess\n", + " # loss metrics.\n", + " print(\"Training model...\")\n", + " print(\"RMSE (on training data):\")\n", + " training_rmse = []\n", + " validation_rmse = []\n", + " for period in range (0, periods):\n", + " # Train the model, starting from the prior state.\n", + " linear_regressor.train(\n", + " input_fn=training_input_fn,\n", + " steps=steps_per_period\n", + " )\n", + " # Take a break and compute predictions.\n", + " training_predictions = linear_regressor.predict(input_fn=predict_training_input_fn)\n", + " training_predictions = np.array([item['predictions'][0] for item in training_predictions])\n", + " validation_predictions = linear_regressor.predict(input_fn=predict_validation_input_fn)\n", + " validation_predictions = np.array([item['predictions'][0] for item in validation_predictions])\n", + " \n", + " # Compute training and validation loss.\n", + " training_root_mean_squared_error = math.sqrt(\n", + " metrics.mean_squared_error(training_predictions, training_targets))\n", + " validation_root_mean_squared_error = math.sqrt(\n", + " metrics.mean_squared_error(validation_predictions, validation_targets))\n", + " # Occasionally print the current loss.\n", + " print(\" period %02d : %0.2f\" % (period, training_root_mean_squared_error))\n", + " # Add the loss metrics from this period to our list.\n", + " training_rmse.append(training_root_mean_squared_error)\n", + " validation_rmse.append(validation_root_mean_squared_error)\n", + " print(\"Model training finished.\")\n", + "\n", + " \n", + " # Output a graph of loss metrics over periods.\n", + " plt.ylabel(\"RMSE\")\n", + " plt.xlabel(\"Periods\")\n", + " plt.title(\"Root Mean Squared Error vs. Periods\")\n", + " plt.tight_layout()\n", + " plt.plot(training_rmse, label=\"training\")\n", + " plt.plot(validation_rmse, label=\"validation\")\n", + " plt.legend()\n", + "\n", + " return linear_regressor" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "1Cdr02tLIK_Q", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "_ = train_model(\n", + " learning_rate=1.0,\n", + " steps=500,\n", + " batch_size=100,\n", + " feature_columns=construct_feature_columns(training_examples),\n", + " training_examples=training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "i4lGvqajDWlw", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## One-Hot Encoding for Discrete Features\n", + "\n", + "Discrete (i.e. strings, enumerations, integers) features are usually converted into families of binary features before training a logistic regression model.\n", + "\n", + "For example, suppose we created a synthetic feature that can take any of the values `0`, `1` or `2`, and that we have a few training points:\n", + "\n", + "| # | feature_value |\n", + "|---|---------------|\n", + "| 0 | 2 |\n", + "| 1 | 0 |\n", + "| 2 | 1 |\n", + "\n", + "For each possible categorical value, we make a new **binary** feature of **real values** that can take one of just two possible values: 1.0 if the example has that value, and 0.0 if not. In the example above, the categorical feature would be converted into three features, and the training points now look like:\n", + "\n", + "| # | feature_value_0 | feature_value_1 | feature_value_2 |\n", + "|---|-----------------|-----------------|-----------------|\n", + "| 0 | 0.0 | 0.0 | 1.0 |\n", + "| 1 | 1.0 | 0.0 | 0.0 |\n", + "| 2 | 0.0 | 1.0 | 0.0 |" + ] + }, + { + "metadata": { + "id": "KnssXowblKm7", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Bucketized (Binned) Features\n", + "\n", + "Bucketization is also known as binning.\n", + "\n", + "We can bucketize `population` into the following 3 buckets (for instance):\n", + "- `bucket_0` (`< 5000`): corresponding to less populated blocks\n", + "- `bucket_1` (`5000 - 25000`): corresponding to mid populated blocks\n", + "- `bucket_2` (`> 25000`): corresponding to highly populated blocks\n", + "\n", + "Given the preceding bucket definitions, the following `population` vector:\n", + "\n", + " [[10001], [42004], [2500], [18000]]\n", + "\n", + "becomes the following bucketized feature vector:\n", + "\n", + " [[1], [2], [0], [1]]\n", + "\n", + "The feature values are now the bucket indices. Note that these indices are considered to be discrete features. Typically, these will be further converted in one-hot representations as above, but this is done transparently.\n", + "\n", + "To define feature columns for bucketized features, instead of using `numeric_column`, we can use [`bucketized_column`](https://www.tensorflow.org/api_docs/python/tf/feature_column/bucketized_column), which takes a numeric column as input and transforms it to a bucketized feature using the bucket boundaries specified in the `boundaries` argument. The following code defines bucketized feature columns for `households` and `longitude`; the `get_quantile_based_boundaries` function calculates boundaries based on quantiles, so that each bucket contains an equal number of elements." + ] + }, + { + "metadata": { + "id": "cc9qZrtRy-ED", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def get_quantile_based_boundaries(feature_values, num_buckets):\n", + " boundaries = np.arange(1.0, num_buckets) / num_buckets\n", + " quantiles = feature_values.quantile(boundaries)\n", + " return [quantiles[q] for q in quantiles.keys()]\n", + "\n", + "# Divide households into 7 buckets.\n", + "households = tf.feature_column.numeric_column(\"households\")\n", + "bucketized_households = tf.feature_column.bucketized_column(\n", + " households, boundaries=get_quantile_based_boundaries(\n", + " california_housing_dataframe[\"households\"], 7))\n", + "\n", + "# Divide longitude into 10 buckets.\n", + "longitude = tf.feature_column.numeric_column(\"longitude\")\n", + "bucketized_longitude = tf.feature_column.bucketized_column(\n", + " longitude, boundaries=get_quantile_based_boundaries(\n", + " california_housing_dataframe[\"longitude\"], 10))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "U-pQDAa0MeN3", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Task 1: Train the Model on Bucketized Feature Columns\n", + "**Bucketize all the real valued features in our example, train the model and see if the results improve.**\n", + "\n", + "In the preceding code block, two real valued columns (namely `households` and `longitude`) have been transformed into bucketized feature columns. Your task is to bucketize the rest of the columns, then run the code to train the model. There are various heuristics to find the range of the buckets. This exercise uses a quantile-based technique, which chooses the bucket boundaries in such a way that each bucket has the same number of examples." + ] + }, + { + "metadata": { + "id": "YFXV9lyMLedy", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def construct_feature_columns():\n", + " \"\"\"Construct the TensorFlow Feature Columns.\n", + "\n", + " Returns:\n", + " A set of feature columns\n", + " \"\"\" \n", + " households = tf.feature_column.numeric_column(\"households\")\n", + " longitude = tf.feature_column.numeric_column(\"longitude\")\n", + " latitude = tf.feature_column.numeric_column(\"latitude\")\n", + " housing_median_age = tf.feature_column.numeric_column(\"housing_median_age\")\n", + " median_income = tf.feature_column.numeric_column(\"median_income\")\n", + " rooms_per_person = tf.feature_column.numeric_column(\"rooms_per_person\")\n", + " \n", + " # Divide households into 7 buckets.\n", + " bucketized_households = tf.feature_column.bucketized_column(\n", + " households, boundaries=get_quantile_based_boundaries(\n", + " training_examples[\"households\"], 7))\n", + "\n", + " # Divide longitude into 10 buckets.\n", + " bucketized_longitude = tf.feature_column.bucketized_column(\n", + " longitude, boundaries=get_quantile_based_boundaries(\n", + " training_examples[\"longitude\"], 10))\n", + "\n", + " #\n", + " # YOUR CODE HERE: bucketize the following columns, following the example above:\n", + " #\n", + " # Divide latitude into 10 buckets.\n", + " bucketized_latitude = tf.feature_column.bucketized_column(\n", + " latitude, boundaries=get_quantile_based_boundaries(\n", + " training_examples[\"latitude\"], 10))\n", + "\n", + " # Divide housing_median_age into 7 buckets.\n", + " bucketized_housing_median_age = tf.feature_column.bucketized_column(\n", + " housing_median_age, boundaries=get_quantile_based_boundaries(\n", + " training_examples[\"housing_median_age\"], 7))\n", + " \n", + " # Divide median_income into 7 buckets.\n", + " bucketized_median_income = tf.feature_column.bucketized_column(\n", + " median_income, boundaries=get_quantile_based_boundaries(\n", + " training_examples[\"median_income\"], 7))\n", + " \n", + " # Divide rooms_per_person into 7 buckets.\n", + " bucketized_rooms_per_person = tf.feature_column.bucketized_column(\n", + " rooms_per_person, boundaries=get_quantile_based_boundaries(\n", + " training_examples[\"rooms_per_person\"], 7))\n", + " \n", + " \n", + " \n", + " feature_columns = set([\n", + " bucketized_longitude,\n", + " bucketized_latitude,\n", + " bucketized_housing_median_age,\n", + " bucketized_households,\n", + " bucketized_median_income,\n", + " bucketized_rooms_per_person])\n", + " \n", + " return feature_columns\n" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "0FfUytOTNJhL", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "_ = train_model(\n", + " learning_rate=1.0,\n", + " steps=500,\n", + " batch_size=100,\n", + " feature_columns=construct_feature_columns(),\n", + " training_examples=training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "ZTDHHM61NPTw", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Solution\n", + "\n", + "Click below for a solution." + ] + }, + { + "metadata": { + "id": "JQHnUhL_NRwA", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "You may be wondering how to determine how many buckets to use. That is of course data-dependent. Here, we just selected arbitrary values so as to obtain a not-too-large model." + ] + }, + { + "metadata": { + "id": "Ro5civQ3Ngh_", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def construct_feature_columns():\n", + " \"\"\"Construct the TensorFlow Feature Columns.\n", + "\n", + " Returns:\n", + " A set of feature columns\n", + " \"\"\" \n", + " households = tf.feature_column.numeric_column(\"households\")\n", + " longitude = tf.feature_column.numeric_column(\"longitude\")\n", + " latitude = tf.feature_column.numeric_column(\"latitude\")\n", + " housing_median_age = tf.feature_column.numeric_column(\"housing_median_age\")\n", + " median_income = tf.feature_column.numeric_column(\"median_income\")\n", + " rooms_per_person = tf.feature_column.numeric_column(\"rooms_per_person\")\n", + " \n", + " # Divide households into 7 buckets.\n", + " bucketized_households = tf.feature_column.bucketized_column(\n", + " households, boundaries=get_quantile_based_boundaries(\n", + " training_examples[\"households\"], 7))\n", + "\n", + " # Divide longitude into 10 buckets.\n", + " bucketized_longitude = tf.feature_column.bucketized_column(\n", + " longitude, boundaries=get_quantile_based_boundaries(\n", + " training_examples[\"longitude\"], 10))\n", + " \n", + " # Divide latitude into 10 buckets.\n", + " bucketized_latitude = tf.feature_column.bucketized_column(\n", + " latitude, boundaries=get_quantile_based_boundaries(\n", + " training_examples[\"latitude\"], 10))\n", + "\n", + " # Divide housing_median_age into 7 buckets.\n", + " bucketized_housing_median_age = tf.feature_column.bucketized_column(\n", + " housing_median_age, boundaries=get_quantile_based_boundaries(\n", + " training_examples[\"housing_median_age\"], 7))\n", + " \n", + " # Divide median_income into 7 buckets.\n", + " bucketized_median_income = tf.feature_column.bucketized_column(\n", + " median_income, boundaries=get_quantile_based_boundaries(\n", + " training_examples[\"median_income\"], 7))\n", + " \n", + " # Divide rooms_per_person into 7 buckets.\n", + " bucketized_rooms_per_person = tf.feature_column.bucketized_column(\n", + " rooms_per_person, boundaries=get_quantile_based_boundaries(\n", + " training_examples[\"rooms_per_person\"], 7))\n", + " \n", + " feature_columns = set([\n", + " bucketized_longitude,\n", + " bucketized_latitude,\n", + " bucketized_housing_median_age,\n", + " bucketized_households,\n", + " bucketized_median_income,\n", + " bucketized_rooms_per_person])\n", + " \n", + " return feature_columns" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "RNgfYk6OO8Sy", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "_ = train_model(\n", + " learning_rate=1.0,\n", + " steps=500,\n", + " batch_size=100,\n", + " feature_columns=construct_feature_columns(),\n", + " training_examples=training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "AFJ1qoZPlQcs", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Feature Crosses\n", + "\n", + "Crossing two (or more) features is a clever way to learn non-linear relations using a linear model. In our problem, if we just use the feature `latitude` for learning, the model might learn that city blocks at a particular latitude (or within a particular range of latitudes since we have bucketized it) are more likely to be expensive than others. Similarly for the feature `longitude`. However, if we cross `longitude` by `latitude`, the crossed feature represents a well defined city block. If the model learns that certain city blocks (within range of latitudes and longitudes) are more likely to be more expensive than others, it is a stronger signal than two features considered individually.\n", + "\n", + "Currently, the feature columns API only supports discrete features for crosses. To cross two continuous values, like `latitude` or `longitude`, we can bucketize them.\n", + "\n", + "If we cross the `latitude` and `longitude` features (supposing, for example, that `longitude` was bucketized into `2` buckets, while `latitude` has `3` buckets), we actually get six crossed binary features. Each of these features will get its own separate weight when we train the model." + ] + }, + { + "metadata": { + "id": "-Rk0c1oTYaVH", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Task 2: Train the Model Using Feature Crosses\n", + "\n", + "**Add a feature cross of `longitude` and `latitude` to your model, train it, and determine whether the results improve.**\n", + "\n", + "Refer to the TensorFlow API docs for [`crossed_column()`](https://www.tensorflow.org/api_docs/python/tf/feature_column/crossed_column) to build the feature column for your cross. Use a `hash_bucket_size` of `1000`." + ] + }, + { + "metadata": { + "id": "-eYiVEGeYhUi", + "colab_type": "code", + "cellView": "both", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def construct_feature_columns():\n", + " \"\"\"Construct the TensorFlow Feature Columns.\n", + "\n", + " Returns:\n", + " A set of feature columns\n", + " \"\"\" \n", + " households = tf.feature_column.numeric_column(\"households\")\n", + " longitude = tf.feature_column.numeric_column(\"longitude\")\n", + " latitude = tf.feature_column.numeric_column(\"latitude\")\n", + " housing_median_age = tf.feature_column.numeric_column(\"housing_median_age\")\n", + " median_income = tf.feature_column.numeric_column(\"median_income\")\n", + " rooms_per_person = tf.feature_column.numeric_column(\"rooms_per_person\")\n", + " \n", + " # Divide households into 7 buckets.\n", + " bucketized_households = tf.feature_column.bucketized_column(\n", + " households, boundaries=get_quantile_based_boundaries(\n", + " training_examples[\"households\"], 7))\n", + "\n", + " # Divide longitude into 10 buckets.\n", + " bucketized_longitude = tf.feature_column.bucketized_column(\n", + " longitude, boundaries=get_quantile_based_boundaries(\n", + " training_examples[\"longitude\"], 10))\n", + " \n", + " # Divide latitude into 10 buckets.\n", + " bucketized_latitude = tf.feature_column.bucketized_column(\n", + " latitude, boundaries=get_quantile_based_boundaries(\n", + " training_examples[\"latitude\"], 10))\n", + "\n", + " # Divide housing_median_age into 7 buckets.\n", + " bucketized_housing_median_age = tf.feature_column.bucketized_column(\n", + " housing_median_age, boundaries=get_quantile_based_boundaries(\n", + " training_examples[\"housing_median_age\"], 7))\n", + " \n", + " # Divide median_income into 7 buckets.\n", + " bucketized_median_income = tf.feature_column.bucketized_column(\n", + " median_income, boundaries=get_quantile_based_boundaries(\n", + " training_examples[\"median_income\"], 7))\n", + " \n", + " # Divide rooms_per_person into 7 buckets.\n", + " bucketized_rooms_per_person = tf.feature_column.bucketized_column(\n", + " rooms_per_person, boundaries=get_quantile_based_boundaries(\n", + " training_examples[\"rooms_per_person\"], 7))\n", + " \n", + " # YOUR CODE HERE: Make a feature column for the long_x_lat feature cross\n", + " long_x_lat = tf.feature_column.crossed_column(\n", + " set([bucketized_longitude, bucketized_latitude]), hash_bucket_size=1000) \n", + " \n", + " feature_columns = set([\n", + " bucketized_longitude,\n", + " bucketized_latitude,\n", + " bucketized_housing_median_age,\n", + " bucketized_households,\n", + " bucketized_median_income,\n", + " bucketized_rooms_per_person,\n", + " long_x_lat])\n", + " \n", + " return feature_columns" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "xZuZMp3EShkM", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "_ = train_model(\n", + " learning_rate=1.0,\n", + " steps=500,\n", + " batch_size=100,\n", + " feature_columns=construct_feature_columns(),\n", + " training_examples=training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "0i7vGo9PTaZl", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Solution\n", + "\n", + "Click below for the solution." + ] + }, + { + "metadata": { + "id": "3tAWu8qSTe2v", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def construct_feature_columns():\n", + " \"\"\"Construct the TensorFlow Feature Columns.\n", + "\n", + " Returns:\n", + " A set of feature columns\n", + " \"\"\" \n", + " households = tf.feature_column.numeric_column(\"households\")\n", + " longitude = tf.feature_column.numeric_column(\"longitude\")\n", + " latitude = tf.feature_column.numeric_column(\"latitude\")\n", + " housing_median_age = tf.feature_column.numeric_column(\"housing_median_age\")\n", + " median_income = tf.feature_column.numeric_column(\"median_income\")\n", + " rooms_per_person = tf.feature_column.numeric_column(\"rooms_per_person\")\n", + " \n", + " # Divide households into 7 buckets.\n", + " bucketized_households = tf.feature_column.bucketized_column(\n", + " households, boundaries=get_quantile_based_boundaries(\n", + " training_examples[\"households\"], 7))\n", + "\n", + " # Divide longitude into 10 buckets.\n", + " bucketized_longitude = tf.feature_column.bucketized_column(\n", + " longitude, boundaries=get_quantile_based_boundaries(\n", + " training_examples[\"longitude\"], 10))\n", + " \n", + " # Divide latitude into 10 buckets.\n", + " bucketized_latitude = tf.feature_column.bucketized_column(\n", + " latitude, boundaries=get_quantile_based_boundaries(\n", + " training_examples[\"latitude\"], 10))\n", + "\n", + " # Divide housing_median_age into 7 buckets.\n", + " bucketized_housing_median_age = tf.feature_column.bucketized_column(\n", + " housing_median_age, boundaries=get_quantile_based_boundaries(\n", + " training_examples[\"housing_median_age\"], 7))\n", + " \n", + " # Divide median_income into 7 buckets.\n", + " bucketized_median_income = tf.feature_column.bucketized_column(\n", + " median_income, boundaries=get_quantile_based_boundaries(\n", + " training_examples[\"median_income\"], 7))\n", + " \n", + " # Divide rooms_per_person into 7 buckets.\n", + " bucketized_rooms_per_person = tf.feature_column.bucketized_column(\n", + " rooms_per_person, boundaries=get_quantile_based_boundaries(\n", + " training_examples[\"rooms_per_person\"], 7))\n", + " \n", + " # YOUR CODE HERE: Make a feature column for the long_x_lat feature cross\n", + " long_x_lat = tf.feature_column.crossed_column(\n", + " set([bucketized_longitude, bucketized_latitude]), hash_bucket_size=1000) \n", + " \n", + " feature_columns = set([\n", + " bucketized_longitude,\n", + " bucketized_latitude,\n", + " bucketized_housing_median_age,\n", + " bucketized_households,\n", + " bucketized_median_income,\n", + " bucketized_rooms_per_person,\n", + " long_x_lat])\n", + " \n", + " return feature_columns" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "-_vvNYIyTtPC", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "_ = train_model(\n", + " learning_rate=1.0,\n", + " steps=500,\n", + " batch_size=100,\n", + " feature_columns=construct_feature_columns(),\n", + " training_examples=training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "ymlHJ-vrhLZw", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Optional Challenge: Try Out More Synthetic Features\n", + "\n", + "So far, we've tried simple bucketized columns and feature crosses, but there are many more combinations that could potentially improve the results. For example, you could cross multiple columns. What happens if you vary the number of buckets? What other synthetic features can you think of? Do they improve the model?" + ] + } + ] +} \ No newline at end of file diff --git a/feature_sets.ipynb b/feature_sets.ipynb new file mode 100644 index 0000000..81b9fc6 --- /dev/null +++ b/feature_sets.ipynb @@ -0,0 +1,1193 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "feature_sets.ipynb", + "version": "0.3.2", + "provenance": [], + "collapsed_sections": [ + "JndnmDMp66FL", + "IGINhMIJ5Wyt", + "pZa8miwu6_tQ" + ], + "include_colab_link": true + }, + "kernelspec": { + "name": "python2", + "display_name": "Python 2" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "metadata": { + "id": "JndnmDMp66FL", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "#### Copyright 2017 Google LLC." + ] + }, + { + "metadata": { + "id": "hMqWDc_m6rUC", + "colab_type": "code", + "cellView": "both", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "zbIgBK-oXHO7", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Feature Sets" + ] + }, + { + "metadata": { + "id": "bL04rAQwH3pH", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "**Learning Objective:** Create a minimal set of features that performs just as well as a more complex feature set" + ] + }, + { + "metadata": { + "id": "F8Hci6tAH3pH", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "So far, we've thrown all of our features into the model. Models with fewer features use fewer resources and are easier to maintain. Let's see if we can build a model on a minimal set of housing features that will perform equally as well as one that uses all the features in the data set." + ] + }, + { + "metadata": { + "id": "F5ZjVwK_qOyR", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Setup\n", + "\n", + "As before, let's load and prepare the California housing data." + ] + }, + { + "metadata": { + "id": "SrOYRILAH3pJ", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "from __future__ import print_function\n", + "\n", + "import math\n", + "\n", + "from IPython import display\n", + "from matplotlib import cm\n", + "from matplotlib import gridspec\n", + "from matplotlib import pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "from sklearn import metrics\n", + "import tensorflow as tf\n", + "from tensorflow.python.data import Dataset\n", + "\n", + "tf.logging.set_verbosity(tf.logging.ERROR)\n", + "pd.options.display.max_rows = 10\n", + "pd.options.display.float_format = '{:.1f}'.format\n", + "\n", + "california_housing_dataframe = pd.read_csv(\"https://download.mlcc.google.com/mledu-datasets/california_housing_train.csv\", sep=\",\")\n", + "\n", + "california_housing_dataframe = california_housing_dataframe.reindex(\n", + " np.random.permutation(california_housing_dataframe.index))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "dGnXo7flH3pM", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def preprocess_features(california_housing_dataframe):\n", + " \"\"\"Prepares input features from California housing data set.\n", + "\n", + " Args:\n", + " california_housing_dataframe: A Pandas DataFrame expected to contain data\n", + " from the California housing data set.\n", + " Returns:\n", + " A DataFrame that contains the features to be used for the model, including\n", + " synthetic features.\n", + " \"\"\"\n", + " selected_features = california_housing_dataframe[\n", + " [\"latitude\",\n", + " \"longitude\",\n", + " \"housing_median_age\",\n", + " \"total_rooms\",\n", + " \"total_bedrooms\",\n", + " \"population\",\n", + " \"households\",\n", + " \"median_income\"]]\n", + " processed_features = selected_features.copy()\n", + " # Create a synthetic feature.\n", + " processed_features[\"rooms_per_person\"] = (\n", + " california_housing_dataframe[\"total_rooms\"] /\n", + " california_housing_dataframe[\"population\"])\n", + " return processed_features\n", + "\n", + "def preprocess_targets(california_housing_dataframe):\n", + " \"\"\"Prepares target features (i.e., labels) from California housing data set.\n", + "\n", + " Args:\n", + " california_housing_dataframe: A Pandas DataFrame expected to contain data\n", + " from the California housing data set.\n", + " Returns:\n", + " A DataFrame that contains the target feature.\n", + " \"\"\"\n", + " output_targets = pd.DataFrame()\n", + " # Scale the target to be in units of thousands of dollars.\n", + " output_targets[\"median_house_value\"] = (\n", + " california_housing_dataframe[\"median_house_value\"] / 1000.0)\n", + " return output_targets" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "jLXC8y4AqsIy", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1153 + }, + "outputId": "ae37131b-fc06-405d-d942-0d84fbd498d6" + }, + "cell_type": "code", + "source": [ + "# Choose the first 12000 (out of 17000) examples for training.\n", + "training_examples = preprocess_features(california_housing_dataframe.head(12000))\n", + "training_targets = preprocess_targets(california_housing_dataframe.head(12000))\n", + "\n", + "# Choose the last 5000 (out of 17000) examples for validation.\n", + "validation_examples = preprocess_features(california_housing_dataframe.tail(5000))\n", + "validation_targets = preprocess_targets(california_housing_dataframe.tail(5000))\n", + "\n", + "# Double-check that we've done the right thing.\n", + "print(\"Training examples summary:\")\n", + "display.display(training_examples.describe())\n", + "print(\"Validation examples summary:\")\n", + "display.display(validation_examples.describe())\n", + "\n", + "print(\"Training targets summary:\")\n", + "display.display(training_targets.describe())\n", + "print(\"Validation targets summary:\")\n", + "display.display(validation_targets.describe())" + ], + "execution_count": 3, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Training examples summary:\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + " latitude longitude housing_median_age total_rooms total_bedrooms \\\n", + "count 12000.0 12000.0 12000.0 12000.0 12000.0 \n", + "mean 35.6 -119.6 28.6 2641.0 540.0 \n", + "std 2.1 2.0 12.5 2226.2 429.6 \n", + "min 32.5 -124.3 1.0 2.0 2.0 \n", + "25% 33.9 -121.8 18.0 1453.0 295.0 \n", + "50% 34.2 -118.5 29.0 2119.0 432.0 \n", + "75% 37.7 -118.0 37.0 3146.2 646.0 \n", + "max 42.0 -114.5 52.0 37937.0 6445.0 \n", + "\n", + " population households median_income rooms_per_person \n", + "count 12000.0 12000.0 12000.0 12000.0 \n", + "mean 1437.1 502.4 3.9 2.0 \n", + "std 1188.8 394.0 1.9 1.1 \n", + "min 3.0 2.0 0.5 0.0 \n", + "25% 786.0 280.0 2.6 1.5 \n", + "50% 1167.5 407.0 3.5 1.9 \n", + "75% 1720.0 604.0 4.7 2.3 \n", + "max 35682.0 6082.0 15.0 55.2 " + ], + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
latitudelongitudehousing_median_agetotal_roomstotal_bedroomspopulationhouseholdsmedian_incomerooms_per_person
count12000.012000.012000.012000.012000.012000.012000.012000.012000.0
mean35.6-119.628.62641.0540.01437.1502.43.92.0
std2.12.012.52226.2429.61188.8394.01.91.1
min32.5-124.31.02.02.03.02.00.50.0
25%33.9-121.818.01453.0295.0786.0280.02.61.5
50%34.2-118.529.02119.0432.01167.5407.03.51.9
75%37.7-118.037.03146.2646.01720.0604.04.72.3
max42.0-114.552.037937.06445.035682.06082.015.055.2
\n", + "
" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "Validation examples summary:\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + " latitude longitude housing_median_age total_rooms total_bedrooms \\\n", + "count 5000.0 5000.0 5000.0 5000.0 5000.0 \n", + "mean 35.6 -119.6 28.5 2650.0 537.9 \n", + "std 2.1 2.0 12.7 2065.0 401.4 \n", + "min 32.5 -124.3 2.0 8.0 1.0 \n", + "25% 33.9 -121.8 18.0 1476.0 301.0 \n", + "50% 34.3 -118.5 28.0 2154.0 438.0 \n", + "75% 37.7 -118.0 37.0 3165.2 653.0 \n", + "max 42.0 -114.3 52.0 27700.0 4457.0 \n", + "\n", + " population households median_income rooms_per_person \n", + "count 5000.0 5000.0 5000.0 5000.0 \n", + "mean 1411.5 498.3 3.9 2.0 \n", + "std 1042.8 360.7 1.9 1.3 \n", + "min 8.0 1.0 0.5 0.1 \n", + "25% 797.0 284.0 2.6 1.5 \n", + "50% 1166.0 413.0 3.6 1.9 \n", + "75% 1723.0 607.2 4.8 2.3 \n", + "max 15037.0 4204.0 15.0 52.0 " + ], + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
latitudelongitudehousing_median_agetotal_roomstotal_bedroomspopulationhouseholdsmedian_incomerooms_per_person
count5000.05000.05000.05000.05000.05000.05000.05000.05000.0
mean35.6-119.628.52650.0537.91411.5498.33.92.0
std2.12.012.72065.0401.41042.8360.71.91.3
min32.5-124.32.08.01.08.01.00.50.1
25%33.9-121.818.01476.0301.0797.0284.02.61.5
50%34.3-118.528.02154.0438.01166.0413.03.61.9
75%37.7-118.037.03165.2653.01723.0607.24.82.3
max42.0-114.352.027700.04457.015037.04204.015.052.0
\n", + "
" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "Training targets summary:\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + " median_house_value\n", + "count 12000.0\n", + "mean 207.0\n", + "std 115.6\n", + "min 15.0\n", + "25% 119.4\n", + "50% 180.4\n", + "75% 264.1\n", + "max 500.0" + ], + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
median_house_value
count12000.0
mean207.0
std115.6
min15.0
25%119.4
50%180.4
75%264.1
max500.0
\n", + "
" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "Validation targets summary:\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + " median_house_value\n", + "count 5000.0\n", + "mean 208.1\n", + "std 117.0\n", + "min 15.0\n", + "25% 119.4\n", + "50% 179.8\n", + "75% 266.7\n", + "max 500.0" + ], + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
median_house_value
count5000.0
mean208.1
std117.0
min15.0
25%119.4
50%179.8
75%266.7
max500.0
\n", + "
" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "metadata": { + "id": "hLvmkugKLany", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Task 1: Develop a Good Feature Set\n", + "\n", + "**What's the best performance you can get with just 2 or 3 features?**\n", + "\n", + "A **correlation matrix** shows pairwise correlations, both for each feature compared to the target and for each feature compared to other features.\n", + "\n", + "Here, correlation is defined as the [Pearson correlation coefficient](https://en.wikipedia.org/wiki/Pearson_product-moment_correlation_coefficient). You don't have to understand the mathematical details for this exercise.\n", + "\n", + "Correlation values have the following meanings:\n", + "\n", + " * `-1.0`: perfect negative correlation\n", + " * `0.0`: no correlation\n", + " * `1.0`: perfect positive correlation" + ] + }, + { + "metadata": { + "id": "UzoZUSdLIolF", + "colab_type": "code", + "cellView": "both", + "colab": {} + }, + "cell_type": "code", + "source": [ + "correlation_dataframe = training_examples.copy()\n", + "correlation_dataframe[\"target\"] = training_targets[\"median_house_value\"]\n", + "\n", + "correlation_dataframe.corr()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "RQpktkNpia2P", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Features that have strong positive or negative correlations with the target will add information to our model. We can use the correlation matrix to find such strongly correlated features.\n", + "\n", + "We'd also like to have features that aren't so strongly correlated with each other, so that they add independent information.\n", + "\n", + "Use this information to try removing features. You can also try developing additional synthetic features, such as ratios of two raw features.\n", + "\n", + "For convenience, we've included the training code from the previous exercise." + ] + }, + { + "metadata": { + "id": "bjR5jWpFr2xs", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def construct_feature_columns(input_features):\n", + " \"\"\"Construct the TensorFlow Feature Columns.\n", + "\n", + " Args:\n", + " input_features: The names of the numerical input features to use.\n", + " Returns:\n", + " A set of feature columns\n", + " \"\"\" \n", + " return set([tf.feature_column.numeric_column(my_feature)\n", + " for my_feature in input_features])" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "jsvKHzRciH9T", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def my_input_fn(features, targets, batch_size=1, shuffle=True, num_epochs=None):\n", + " \"\"\"Trains a linear regression model.\n", + " \n", + " Args:\n", + " features: pandas DataFrame of features\n", + " targets: pandas DataFrame of targets\n", + " batch_size: Size of batches to be passed to the model\n", + " shuffle: True or False. Whether to shuffle the data.\n", + " num_epochs: Number of epochs for which data should be repeated. None = repeat indefinitely\n", + " Returns:\n", + " Tuple of (features, labels) for next data batch\n", + " \"\"\"\n", + " \n", + " # Convert pandas data into a dict of np arrays.\n", + " features = {key:np.array(value) for key,value in dict(features).items()} \n", + " \n", + " # Construct a dataset, and configure batching/repeating.\n", + " ds = Dataset.from_tensor_slices((features,targets)) # warning: 2GB limit\n", + " ds = ds.batch(batch_size).repeat(num_epochs)\n", + "\n", + " # Shuffle the data, if specified.\n", + " if shuffle:\n", + " ds = ds.shuffle(10000)\n", + " \n", + " # Return the next batch of data.\n", + " features, labels = ds.make_one_shot_iterator().get_next()\n", + " return features, labels" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "g3kjQV9WH3pb", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def train_model(\n", + " learning_rate,\n", + " steps,\n", + " batch_size,\n", + " training_examples,\n", + " training_targets,\n", + " validation_examples,\n", + " validation_targets):\n", + " \"\"\"Trains a linear regression model.\n", + " \n", + " In addition to training, this function also prints training progress information,\n", + " as well as a plot of the training and validation loss over time.\n", + " \n", + " Args:\n", + " learning_rate: A `float`, the learning rate.\n", + " steps: A non-zero `int`, the total number of training steps. A training step\n", + " consists of a forward and backward pass using a single batch.\n", + " batch_size: A non-zero `int`, the batch size.\n", + " training_examples: A `DataFrame` containing one or more columns from\n", + " `california_housing_dataframe` to use as input features for training.\n", + " training_targets: A `DataFrame` containing exactly one column from\n", + " `california_housing_dataframe` to use as target for training.\n", + " validation_examples: A `DataFrame` containing one or more columns from\n", + " `california_housing_dataframe` to use as input features for validation.\n", + " validation_targets: A `DataFrame` containing exactly one column from\n", + " `california_housing_dataframe` to use as target for validation.\n", + " \n", + " Returns:\n", + " A `LinearRegressor` object trained on the training data.\n", + " \"\"\"\n", + "\n", + " periods = 10\n", + " steps_per_period = steps / periods\n", + "\n", + " # Create a linear regressor object.\n", + " my_optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n", + " my_optimizer = tf.contrib.estimator.clip_gradients_by_norm(my_optimizer, 5.0)\n", + " linear_regressor = tf.estimator.LinearRegressor(\n", + " feature_columns=construct_feature_columns(training_examples),\n", + " optimizer=my_optimizer\n", + " )\n", + " \n", + " # Create input functions.\n", + " training_input_fn = lambda: my_input_fn(training_examples, \n", + " training_targets[\"median_house_value\"], \n", + " batch_size=batch_size)\n", + " predict_training_input_fn = lambda: my_input_fn(training_examples, \n", + " training_targets[\"median_house_value\"], \n", + " num_epochs=1, \n", + " shuffle=False)\n", + " predict_validation_input_fn = lambda: my_input_fn(validation_examples, \n", + " validation_targets[\"median_house_value\"], \n", + " num_epochs=1, \n", + " shuffle=False)\n", + "\n", + " # Train the model, but do so inside a loop so that we can periodically assess\n", + " # loss metrics.\n", + " print(\"Training model...\")\n", + " print(\"RMSE (on training data):\")\n", + " training_rmse = []\n", + " validation_rmse = []\n", + " for period in range (0, periods):\n", + " # Train the model, starting from the prior state.\n", + " linear_regressor.train(\n", + " input_fn=training_input_fn,\n", + " steps=steps_per_period,\n", + " )\n", + " # Take a break and compute predictions.\n", + " training_predictions = linear_regressor.predict(input_fn=predict_training_input_fn)\n", + " training_predictions = np.array([item['predictions'][0] for item in training_predictions])\n", + " \n", + " validation_predictions = linear_regressor.predict(input_fn=predict_validation_input_fn)\n", + " validation_predictions = np.array([item['predictions'][0] for item in validation_predictions])\n", + " \n", + " # Compute training and validation loss.\n", + " training_root_mean_squared_error = math.sqrt(\n", + " metrics.mean_squared_error(training_predictions, training_targets))\n", + " validation_root_mean_squared_error = math.sqrt(\n", + " metrics.mean_squared_error(validation_predictions, validation_targets))\n", + " # Occasionally print the current loss.\n", + " print(\" period %02d : %0.2f\" % (period, training_root_mean_squared_error))\n", + " # Add the loss metrics from this period to our list.\n", + " training_rmse.append(training_root_mean_squared_error)\n", + " validation_rmse.append(validation_root_mean_squared_error)\n", + " print(\"Model training finished.\")\n", + "\n", + " \n", + " # Output a graph of loss metrics over periods.\n", + " plt.ylabel(\"RMSE\")\n", + " plt.xlabel(\"Periods\")\n", + " plt.title(\"Root Mean Squared Error vs. Periods\")\n", + " plt.tight_layout()\n", + " plt.plot(training_rmse, label=\"training\")\n", + " plt.plot(validation_rmse, label=\"validation\")\n", + " plt.legend()\n", + "\n", + " return linear_regressor" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "varLu7RNH3pf", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Spend 5 minutes searching for a good set of features and training parameters. Then check the solution to see what we chose. Don't forget that different features may require different learning parameters." + ] + }, + { + "metadata": { + "id": "DSgUxRIlH3pg", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#\n", + "# Your code here: add your features of choice as a list of quoted strings.\n", + "#\n", + "minimal_features = [\n", + "]\n", + "\n", + "assert minimal_features, \"You must select at least one feature!\"\n", + "\n", + "minimal_training_examples = training_examples[minimal_features]\n", + "minimal_validation_examples = validation_examples[minimal_features]\n", + "\n", + "#\n", + "# Don't forget to adjust these parameters.\n", + "#\n", + "train_model(\n", + " learning_rate=0.001,\n", + " steps=500,\n", + " batch_size=5,\n", + " training_examples=minimal_training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=minimal_validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "IGINhMIJ5Wyt", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Solution\n", + "\n", + "Click below for a solution." + ] + }, + { + "metadata": { + "id": "BAGoXFPZ5ZE3", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "minimal_features = [\n", + " \"median_income\",\n", + " \"latitude\",\n", + "]\n", + "\n", + "minimal_training_examples = training_examples[minimal_features]\n", + "minimal_validation_examples = validation_examples[minimal_features]\n", + "\n", + "_ = train_model(\n", + " learning_rate=0.01,\n", + " steps=500,\n", + " batch_size=5,\n", + " training_examples=minimal_training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=minimal_validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "RidI9YhKOiY2", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Task 2: Make Better Use of Latitude\n", + "\n", + "Plotting `latitude` vs. `median_house_value` shows that there really isn't a linear relationship there.\n", + "\n", + "Instead, there are a couple of peaks, which roughly correspond to Los Angeles and San Francisco." + ] + }, + { + "metadata": { + "id": "hfGUKj2IR_F1", + "colab_type": "code", + "cellView": "both", + "colab": {} + }, + "cell_type": "code", + "source": [ + "plt.scatter(training_examples[\"latitude\"], training_targets[\"median_house_value\"])" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "6N0p91k2iFCP", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "**Try creating some synthetic features that do a better job with latitude.**\n", + "\n", + "For example, you could have a feature that maps `latitude` to a value of `|latitude - 38|`, and call this `distance_from_san_francisco`.\n", + "\n", + "Or you could break the space into 10 different buckets. `latitude_32_to_33`, `latitude_33_to_34`, etc., each showing a value of `1.0` if `latitude` is within that bucket range and a value of `0.0` otherwise.\n", + "\n", + "Use the correlation matrix to help guide development, and then add them to your model if you find something that looks good.\n", + "\n", + "What's the best validation performance you can get?" + ] + }, + { + "metadata": { + "id": "wduJ2B28yMFl", + "colab_type": "code", + "cellView": "form", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#\n", + "# YOUR CODE HERE: Train on a new data set that includes synthetic features based on latitude.\n", + "#\n", + "def select_and_transform_features(source_df):\n", + " LATITUDE_RANGES = zip(range(32, 44), range(33, 45))\n", + " selected_examples = pd.DataFrame()\n", + " selected_examples[\"median_income\"] = source_df[\"median_income\"]\n", + " for r in LATITUDE_RANGES:\n", + " selected_examples[\"latitude_%d_to_%d\" % r] = source_df[\"latitude\"].apply(\n", + " lambda l: 1.0 if l >= r[0] and l < r[1] else 0.0)\n", + " return selected_examples\n", + "\n", + "selected_training_examples = select_and_transform_features(training_examples)\n", + "selected_validation_examples = select_and_transform_features(validation_examples)\n", + "\n", + "\n", + "_ = train_model(\n", + " learning_rate=0.01,\n", + " steps=500,\n", + " batch_size=5,\n", + " training_examples=selected_training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=selected_validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "pZa8miwu6_tQ", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Solution\n", + "\n", + "Click below for a solution." + ] + }, + { + "metadata": { + "id": "PzABdyjq7IZU", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Aside from `latitude`, we'll also keep `median_income`, to compare with the previous results.\n", + "\n", + "We decided to bucketize the latitude. This is fairly straightforward in Pandas using `Series.apply`." + ] + }, + { + "metadata": { + "id": "xdVF8siZ7Lup", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def select_and_transform_features(source_df):\n", + " LATITUDE_RANGES = zip(range(32, 44), range(33, 45))\n", + " selected_examples = pd.DataFrame()\n", + " selected_examples[\"median_income\"] = source_df[\"median_income\"]\n", + " for r in LATITUDE_RANGES:\n", + " selected_examples[\"latitude_%d_to_%d\" % r] = source_df[\"latitude\"].apply(\n", + " lambda l: 1.0 if l >= r[0] and l < r[1] else 0.0)\n", + " return selected_examples\n", + "\n", + "selected_training_examples = select_and_transform_features(training_examples)\n", + "selected_validation_examples = select_and_transform_features(validation_examples)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "U4iAdY6t7Pkh", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "_ = train_model(\n", + " learning_rate=0.01,\n", + " steps=500,\n", + " batch_size=5,\n", + " training_examples=selected_training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=selected_validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 0, + "outputs": [] + } + ] +} \ No newline at end of file diff --git a/first_steps_with_tensor_flow.ipynb b/first_steps_with_tensor_flow.ipynb new file mode 100644 index 0000000..0ac8548 --- /dev/null +++ b/first_steps_with_tensor_flow.ipynb @@ -0,0 +1,971 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "first_steps_with_tensor_flow.ipynb", + "version": "0.3.2", + "provenance": [], + "collapsed_sections": [ + "JndnmDMp66FL", + "ajVM7rkoYXeL", + "ci1ISxxrZ7v0" + ], + "include_colab_link": true + }, + "kernelspec": { + "name": "python2", + "display_name": "Python 2" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "metadata": { + "id": "JndnmDMp66FL", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "#### Copyright 2017 Google LLC." + ] + }, + { + "metadata": { + "id": "hMqWDc_m6rUC", + "colab_type": "code", + "cellView": "both", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "4f3CKqFUqL2-", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# First Steps with TensorFlow" + ] + }, + { + "metadata": { + "id": "Bd2Zkk1LE2Zr", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "**Learning Objectives:**\n", + " * Learn fundamental TensorFlow concepts\n", + " * Use the `LinearRegressor` class in TensorFlow to predict median housing price, at the granularity of city blocks, based on one input feature\n", + " * Evaluate the accuracy of a model's predictions using Root Mean Squared Error (RMSE)\n", + " * Improve the accuracy of a model by tuning its hyperparameters" + ] + }, + { + "metadata": { + "id": "MxiIKhP4E2Zr", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "The [data](https://developers.google.com/machine-learning/crash-course/california-housing-data-description) is based on 1990 census data from California." + ] + }, + { + "metadata": { + "id": "6TjLjL9IU80G", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Setup\n", + "In this first cell, we'll load the necessary libraries." + ] + }, + { + "metadata": { + "id": "rVFf5asKE2Zt", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "from __future__ import print_function\n", + "\n", + "import math\n", + "\n", + "from IPython import display\n", + "from matplotlib import cm\n", + "from matplotlib import gridspec\n", + "from matplotlib import pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "from sklearn import metrics\n", + "import tensorflow as tf\n", + "from tensorflow.python.data import Dataset\n", + "\n", + "tf.logging.set_verbosity(tf.logging.ERROR)\n", + "pd.options.display.max_rows = 10\n", + "pd.options.display.float_format = '{:.1f}'.format" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "ipRyUHjhU80Q", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Next, we'll load our data set." + ] + }, + { + "metadata": { + "id": "9ivCDWnwE2Zx", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "california_housing_dataframe = pd.read_csv(\"https://download.mlcc.google.com/mledu-datasets/california_housing_train.csv\", sep=\",\")" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "vVk_qlG6U80j", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "We'll randomize the data, just to be sure not to get any pathological ordering effects that might harm the performance of Stochastic Gradient Descent. Additionally, we'll scale `median_house_value` to be in units of thousands, so it can be learned a little more easily with learning rates in a range that we usually use." + ] + }, + { + "metadata": { + "id": "r0eVyguIU80m", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "california_housing_dataframe = california_housing_dataframe.reindex(\n", + " np.random.permutation(california_housing_dataframe.index))\n", + "california_housing_dataframe[\"median_house_value\"] /= 1000.0\n", + "california_housing_dataframe" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "HzzlSs3PtTmt", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Examine the Data\n", + "\n", + "It's a good idea to get to know your data a little bit before you work with it.\n", + "\n", + "We'll print out a quick summary of a few useful statistics on each column: count of examples, mean, standard deviation, max, min, and various quantiles." + ] + }, + { + "metadata": { + "id": "gzb10yoVrydW", + "colab_type": "code", + "cellView": "both", + "colab": {} + }, + "cell_type": "code", + "source": [ + "california_housing_dataframe.describe()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "Lr6wYl2bt2Ep", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Build the First Model\n", + "\n", + "In this exercise, we'll try to predict `median_house_value`, which will be our label (sometimes also called a target). We'll use `total_rooms` as our input feature.\n", + "\n", + "**NOTE:** Our data is at the city block level, so this feature represents the total number of rooms in that block.\n", + "\n", + "To train our model, we'll use the [LinearRegressor](https://www.tensorflow.org/api_docs/python/tf/estimator/LinearRegressor) interface provided by the TensorFlow [Estimator](https://www.tensorflow.org/get_started/estimator) API. This API takes care of a lot of the low-level model plumbing, and exposes convenient methods for performing model training, evaluation, and inference." + ] + }, + { + "metadata": { + "id": "0cpcsieFhsNI", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Step 1: Define Features and Configure Feature Columns" + ] + }, + { + "metadata": { + "id": "EL8-9d4ZJNR7", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "In order to import our training data into TensorFlow, we need to specify what type of data each feature contains. There are two main types of data we'll use in this and future exercises:\n", + "\n", + "* **Categorical Data**: Data that is textual. In this exercise, our housing data set does not contain any categorical features, but examples you might see would be the home style, the words in a real-estate ad.\n", + "\n", + "* **Numerical Data**: Data that is a number (integer or float) and that you want to treat as a number. As we will discuss more later sometimes you might want to treat numerical data (e.g., a postal code) as if it were categorical.\n", + "\n", + "In TensorFlow, we indicate a feature's data type using a construct called a **feature column**. Feature columns store only a description of the feature data; they do not contain the feature data itself.\n", + "\n", + "To start, we're going to use just one numeric input feature, `total_rooms`. The following code pulls the `total_rooms` data from our `california_housing_dataframe` and defines the feature column using `numeric_column`, which specifies its data is numeric:" + ] + }, + { + "metadata": { + "id": "rhEbFCZ86cDZ", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# Define the input feature: total_rooms.\n", + "my_feature = california_housing_dataframe[[\"total_rooms\"]]\n", + "\n", + "# Configure a numeric feature column for total_rooms.\n", + "feature_columns = [tf.feature_column.numeric_column(\"total_rooms\")]" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "K_3S8teX7Rd2", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "**NOTE:** The shape of our `total_rooms` data is a one-dimensional array (a list of the total number of rooms for each block). This is the default shape for `numeric_column`, so we don't have to pass it as an argument." + ] + }, + { + "metadata": { + "id": "UMl3qrU5MGV6", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Step 2: Define the Target" + ] + }, + { + "metadata": { + "id": "cw4nrfcB7kyk", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Next, we'll define our target, which is `median_house_value`. Again, we can pull it from our `california_housing_dataframe`:" + ] + }, + { + "metadata": { + "id": "l1NvvNkH8Kbt", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# Define the label.\n", + "targets = california_housing_dataframe[\"median_house_value\"]" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "4M-rTFHL2UkA", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Step 3: Configure the LinearRegressor" + ] + }, + { + "metadata": { + "id": "fUfGQUNp7jdL", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Next, we'll configure a linear regression model using LinearRegressor. We'll train this model using the `GradientDescentOptimizer`, which implements Mini-Batch Stochastic Gradient Descent (SGD). The `learning_rate` argument controls the size of the gradient step.\n", + "\n", + "**NOTE:** To be safe, we also apply [gradient clipping](https://developers.google.com/machine-learning/glossary/#gradient_clipping) to our optimizer via `clip_gradients_by_norm`. Gradient clipping ensures the magnitude of the gradients do not become too large during training, which can cause gradient descent to fail. " + ] + }, + { + "metadata": { + "id": "ubhtW-NGU802", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# Use gradient descent as the optimizer for training the model.\n", + "my_optimizer=tf.train.GradientDescentOptimizer(learning_rate=0.0000001)\n", + "my_optimizer = tf.contrib.estimator.clip_gradients_by_norm(my_optimizer, 5.0)\n", + "\n", + "# Configure the linear regression model with our feature columns and optimizer.\n", + "# Set a learning rate of 0.0000001 for Gradient Descent.\n", + "linear_regressor = tf.estimator.LinearRegressor(\n", + " feature_columns=feature_columns,\n", + " optimizer=my_optimizer\n", + ")" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "-0IztwdK2f3F", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Step 4: Define the Input Function" + ] + }, + { + "metadata": { + "id": "S5M5j6xSCHxx", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "To import our California housing data into our `LinearRegressor`, we need to define an input function, which instructs TensorFlow how to preprocess\n", + "the data, as well as how to batch, shuffle, and repeat it during model training.\n", + "\n", + "First, we'll convert our *pandas* feature data into a dict of NumPy arrays. We can then use the TensorFlow [Dataset API](https://www.tensorflow.org/programmers_guide/datasets) to construct a dataset object from our data, and then break\n", + "our data into batches of `batch_size`, to be repeated for the specified number of epochs (num_epochs). \n", + "\n", + "**NOTE:** When the default value of `num_epochs=None` is passed to `repeat()`, the input data will be repeated indefinitely.\n", + "\n", + "Next, if `shuffle` is set to `True`, we'll shuffle the data so that it's passed to the model randomly during training. The `buffer_size` argument specifies\n", + "the size of the dataset from which `shuffle` will randomly sample.\n", + "\n", + "Finally, our input function constructs an iterator for the dataset and returns the next batch of data to the LinearRegressor." + ] + }, + { + "metadata": { + "id": "RKZ9zNcHJtwc", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def my_input_fn(features, targets, batch_size=1, shuffle=True, num_epochs=None):\n", + " \"\"\"Trains a linear regression model of one feature.\n", + " \n", + " Args:\n", + " features: pandas DataFrame of features\n", + " targets: pandas DataFrame of targets\n", + " batch_size: Size of batches to be passed to the model\n", + " shuffle: True or False. Whether to shuffle the data.\n", + " num_epochs: Number of epochs for which data should be repeated. None = repeat indefinitely\n", + " Returns:\n", + " Tuple of (features, labels) for next data batch\n", + " \"\"\"\n", + " \n", + " # Convert pandas data into a dict of np arrays.\n", + " features = {key:np.array(value) for key,value in dict(features).items()} \n", + " \n", + " # Construct a dataset, and configure batching/repeating.\n", + " ds = Dataset.from_tensor_slices((features,targets)) # warning: 2GB limit\n", + " ds = ds.batch(batch_size).repeat(num_epochs)\n", + " \n", + " # Shuffle the data, if specified.\n", + " if shuffle:\n", + " ds = ds.shuffle(buffer_size=10000)\n", + " \n", + " # Return the next batch of data.\n", + " features, labels = ds.make_one_shot_iterator().get_next()\n", + " return features, labels" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "wwa6UeA1V5F_", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "**NOTE:** We'll continue to use this same input function in later exercises. For more\n", + "detailed documentation of input functions and the `Dataset` API, see the [TensorFlow Programmer's Guide](https://www.tensorflow.org/programmers_guide/datasets)." + ] + }, + { + "metadata": { + "id": "4YS50CQb2ooO", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Step 5: Train the Model" + ] + }, + { + "metadata": { + "id": "yP92XkzhU803", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "We can now call `train()` on our `linear_regressor` to train the model. We'll wrap `my_input_fn` in a `lambda`\n", + "so we can pass in `my_feature` and `target` as arguments (see this [TensorFlow input function tutorial](https://www.tensorflow.org/get_started/input_fn#passing_input_fn_data_to_your_model) for more details), and to start, we'll\n", + "train for 100 steps." + ] + }, + { + "metadata": { + "id": "5M-Kt6w8U803", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "_ = linear_regressor.train(\n", + " input_fn = lambda:my_input_fn(my_feature, targets),\n", + " steps=100\n", + ")" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "7Nwxqxlx2sOv", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Step 6: Evaluate the Model" + ] + }, + { + "metadata": { + "id": "KoDaF2dlJQG5", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Let's make predictions on that training data, to see how well our model fit it during training.\n", + "\n", + "**NOTE:** Training error measures how well your model fits the training data, but it **_does not_** measure how well your model **_generalizes to new data_**. In later exercises, you'll explore how to split your data to evaluate your model's ability to generalize.\n" + ] + }, + { + "metadata": { + "id": "pDIxp6vcU809", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# Create an input function for predictions.\n", + "# Note: Since we're making just one prediction for each example, we don't \n", + "# need to repeat or shuffle the data here.\n", + "prediction_input_fn =lambda: my_input_fn(my_feature, targets, num_epochs=1, shuffle=False)\n", + "\n", + "# Call predict() on the linear_regressor to make predictions.\n", + "predictions = linear_regressor.predict(input_fn=prediction_input_fn)\n", + "\n", + "# Format predictions as a NumPy array, so we can calculate error metrics.\n", + "predictions = np.array([item['predictions'][0] for item in predictions])\n", + "\n", + "# Print Mean Squared Error and Root Mean Squared Error.\n", + "mean_squared_error = metrics.mean_squared_error(predictions, targets)\n", + "root_mean_squared_error = math.sqrt(mean_squared_error)\n", + "print(\"Mean Squared Error (on training data): %0.3f\" % mean_squared_error)\n", + "print(\"Root Mean Squared Error (on training data): %0.3f\" % root_mean_squared_error)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "AKWstXXPzOVz", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Is this a good model? How would you judge how large this error is?\n", + "\n", + "Mean Squared Error (MSE) can be hard to interpret, so we often look at Root Mean Squared Error (RMSE)\n", + "instead. A nice property of RMSE is that it can be interpreted on the same scale as the original targets.\n", + "\n", + "Let's compare the RMSE to the difference of the min and max of our targets:" + ] + }, + { + "metadata": { + "id": "7UwqGbbxP53O", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "min_house_value = california_housing_dataframe[\"median_house_value\"].min()\n", + "max_house_value = california_housing_dataframe[\"median_house_value\"].max()\n", + "min_max_difference = max_house_value - min_house_value\n", + "\n", + "print(\"Min. Median House Value: %0.3f\" % min_house_value)\n", + "print(\"Max. Median House Value: %0.3f\" % max_house_value)\n", + "print(\"Difference between Min. and Max.: %0.3f\" % min_max_difference)\n", + "print(\"Root Mean Squared Error: %0.3f\" % root_mean_squared_error)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "JigJr0C7Pzit", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Our error spans nearly half the range of the target values. Can we do better?\n", + "\n", + "This is the question that nags at every model developer. Let's develop some basic strategies to reduce model error.\n", + "\n", + "The first thing we can do is take a look at how well our predictions match our targets, in terms of overall summary statistics." + ] + }, + { + "metadata": { + "id": "941nclxbzqGH", + "colab_type": "code", + "cellView": "both", + "colab": {} + }, + "cell_type": "code", + "source": [ + "calibration_data = pd.DataFrame()\n", + "calibration_data[\"predictions\"] = pd.Series(predictions)\n", + "calibration_data[\"targets\"] = pd.Series(targets)\n", + "calibration_data.describe()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "E2-bf8Hq36y8", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Okay, maybe this information is helpful. How does the mean value compare to the model's RMSE? How about the various quantiles?\n", + "\n", + "We can also visualize the data and the line we've learned. Recall that linear regression on a single feature can be drawn as a line mapping input *x* to output *y*.\n", + "\n", + "First, we'll get a uniform random sample of the data so we can make a readable scatter plot." + ] + }, + { + "metadata": { + "id": "SGRIi3mAU81H", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "sample = california_housing_dataframe.sample(n=300)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "N-JwuJBKU81J", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Next, we'll plot the line we've learned, drawing from the model's bias term and feature weight, together with the scatter plot. The line will show up red." + ] + }, + { + "metadata": { + "id": "7G12E76-339G", + "colab_type": "code", + "cellView": "both", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# Get the min and max total_rooms values.\n", + "x_0 = sample[\"total_rooms\"].min()\n", + "x_1 = sample[\"total_rooms\"].max()\n", + "\n", + "# Retrieve the final weight and bias generated during training.\n", + "weight = linear_regressor.get_variable_value('linear/linear_model/total_rooms/weights')[0]\n", + "bias = linear_regressor.get_variable_value('linear/linear_model/bias_weights')\n", + "\n", + "# Get the predicted median_house_values for the min and max total_rooms values.\n", + "y_0 = weight * x_0 + bias \n", + "y_1 = weight * x_1 + bias\n", + "\n", + "# Plot our regression line from (x_0, y_0) to (x_1, y_1).\n", + "plt.plot([x_0, x_1], [y_0, y_1], c='r')\n", + "\n", + "# Label the graph axes.\n", + "plt.ylabel(\"median_house_value\")\n", + "plt.xlabel(\"total_rooms\")\n", + "\n", + "# Plot a scatter plot from our data sample.\n", + "plt.scatter(sample[\"total_rooms\"], sample[\"median_house_value\"])\n", + "\n", + "# Display graph.\n", + "plt.show()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "t0lRt4USU81L", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "This initial line looks way off. See if you can look back at the summary stats and see the same information encoded there.\n", + "\n", + "Together, these initial sanity checks suggest we may be able to find a much better line." + ] + }, + { + "metadata": { + "id": "AZWF67uv0HTG", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Tweak the Model Hyperparameters\n", + "For this exercise, we've put all the above code in a single function for convenience. You can call the function with different parameters to see the effect.\n", + "\n", + "In this function, we'll proceed in 10 evenly divided periods so that we can observe the model improvement at each period.\n", + "\n", + "For each period, we'll compute and graph training loss. This may help you judge when a model is converged, or if it needs more iterations.\n", + "\n", + "We'll also plot the feature weight and bias term values learned by the model over time. This is another way to see how things converge." + ] + }, + { + "metadata": { + "id": "wgSMeD5UU81N", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def train_model(learning_rate, steps, batch_size, input_feature=\"total_rooms\"):\n", + " \"\"\"Trains a linear regression model of one feature.\n", + " \n", + " Args:\n", + " learning_rate: A `float`, the learning rate.\n", + " steps: A non-zero `int`, the total number of training steps. A training step\n", + " consists of a forward and backward pass using a single batch.\n", + " batch_size: A non-zero `int`, the batch size.\n", + " input_feature: A `string` specifying a column from `california_housing_dataframe`\n", + " to use as input feature.\n", + " \"\"\"\n", + " \n", + " periods = 10\n", + " steps_per_period = steps / periods\n", + "\n", + " my_feature = input_feature\n", + " my_feature_data = california_housing_dataframe[[my_feature]]\n", + " my_label = \"median_house_value\"\n", + " targets = california_housing_dataframe[my_label]\n", + "\n", + " # Create feature columns.\n", + " feature_columns = [tf.feature_column.numeric_column(my_feature)]\n", + " \n", + " # Create input functions.\n", + " training_input_fn = lambda:my_input_fn(my_feature_data, targets, batch_size=batch_size)\n", + " prediction_input_fn = lambda: my_input_fn(my_feature_data, targets, num_epochs=1, shuffle=False)\n", + " \n", + " # Create a linear regressor object.\n", + " my_optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n", + " my_optimizer = tf.contrib.estimator.clip_gradients_by_norm(my_optimizer, 5.0)\n", + " linear_regressor = tf.estimator.LinearRegressor(\n", + " feature_columns=feature_columns,\n", + " optimizer=my_optimizer\n", + " )\n", + "\n", + " # Set up to plot the state of our model's line each period.\n", + " plt.figure(figsize=(15, 6))\n", + " plt.subplot(1, 2, 1)\n", + " plt.title(\"Learned Line by Period\")\n", + " plt.ylabel(my_label)\n", + " plt.xlabel(my_feature)\n", + " sample = california_housing_dataframe.sample(n=300)\n", + " plt.scatter(sample[my_feature], sample[my_label])\n", + " colors = [cm.coolwarm(x) for x in np.linspace(-1, 1, periods)]\n", + "\n", + " # Train the model, but do so inside a loop so that we can periodically assess\n", + " # loss metrics.\n", + " print(\"Training model...\")\n", + " print(\"RMSE (on training data):\")\n", + " root_mean_squared_errors = []\n", + " for period in range (0, periods):\n", + " # Train the model, starting from the prior state.\n", + " linear_regressor.train(\n", + " input_fn=training_input_fn,\n", + " steps=steps_per_period\n", + " )\n", + " # Take a break and compute predictions.\n", + " predictions = linear_regressor.predict(input_fn=prediction_input_fn)\n", + " predictions = np.array([item['predictions'][0] for item in predictions])\n", + " \n", + " # Compute loss.\n", + " root_mean_squared_error = math.sqrt(\n", + " metrics.mean_squared_error(predictions, targets))\n", + " # Occasionally print the current loss.\n", + " print(\" period %02d : %0.2f\" % (period, root_mean_squared_error))\n", + " # Add the loss metrics from this period to our list.\n", + " root_mean_squared_errors.append(root_mean_squared_error)\n", + " # Finally, track the weights and biases over time.\n", + " # Apply some math to ensure that the data and line are plotted neatly.\n", + " y_extents = np.array([0, sample[my_label].max()])\n", + " \n", + " weight = linear_regressor.get_variable_value('linear/linear_model/%s/weights' % input_feature)[0]\n", + " bias = linear_regressor.get_variable_value('linear/linear_model/bias_weights')\n", + "\n", + " x_extents = (y_extents - bias) / weight\n", + " x_extents = np.maximum(np.minimum(x_extents,\n", + " sample[my_feature].max()),\n", + " sample[my_feature].min())\n", + " y_extents = weight * x_extents + bias\n", + " plt.plot(x_extents, y_extents, color=colors[period]) \n", + " print(\"Model training finished.\")\n", + "\n", + " # Output a graph of loss metrics over periods.\n", + " plt.subplot(1, 2, 2)\n", + " plt.ylabel('RMSE')\n", + " plt.xlabel('Periods')\n", + " plt.title(\"Root Mean Squared Error vs. Periods\")\n", + " plt.tight_layout()\n", + " plt.plot(root_mean_squared_errors)\n", + "\n", + " # Output a table with calibration data.\n", + " calibration_data = pd.DataFrame()\n", + " calibration_data[\"predictions\"] = pd.Series(predictions)\n", + " calibration_data[\"targets\"] = pd.Series(targets)\n", + " display.display(calibration_data.describe())\n", + "\n", + " print(\"Final RMSE (on training data): %0.2f\" % root_mean_squared_error)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "kg8A4ArBU81Q", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Task 1: Achieve an RMSE of 180 or Below\n", + "\n", + "Tweak the model hyperparameters to improve loss and better match the target distribution.\n", + "If, after 5 minutes or so, you're having trouble beating a RMSE of 180, check the solution for a possible combination." + ] + }, + { + "metadata": { + "id": "UzoZUSdLIolF", + "colab_type": "code", + "cellView": "both", + "colab": {} + }, + "cell_type": "code", + "source": [ + "train_model(\n", + " learning_rate=0.00002,\n", + " steps=600,\n", + " batch_size=5\n", + ")" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "ajVM7rkoYXeL", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Solution\n", + "\n", + "Click below for one possible solution." + ] + }, + { + "metadata": { + "id": "T3zmldDwYy5c", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "train_model(\n", + " learning_rate=0.00002,\n", + " steps=500,\n", + " batch_size=5\n", + ")" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "M8H0_D4vYa49", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "This is just one possible configuration; there may be other combinations of settings that also give good results. Note that in general, this exercise isn't about finding the *one best* setting, but to help build your intutions about how tweaking the model configuration affects prediction quality." + ] + }, + { + "metadata": { + "id": "QU5sLyYTqzqL", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Is There a Standard Heuristic for Model Tuning?\n", + "\n", + "This is a commonly asked question. The short answer is that the effects of different hyperparameters are data dependent. So there are no hard-and-fast rules; you'll need to test on your data.\n", + "\n", + "That said, here are a few rules of thumb that may help guide you:\n", + "\n", + " * Training error should steadily decrease, steeply at first, and should eventually plateau as training converges.\n", + " * If the training has not converged, try running it for longer.\n", + " * If the training error decreases too slowly, increasing the learning rate may help it decrease faster.\n", + " * But sometimes the exact opposite may happen if the learning rate is too high.\n", + " * If the training error varies wildly, try decreasing the learning rate.\n", + " * Lower learning rate plus larger number of steps or larger batch size is often a good combination.\n", + " * Very small batch sizes can also cause instability. First try larger values like 100 or 1000, and decrease until you see degradation.\n", + "\n", + "Again, never go strictly by these rules of thumb, because the effects are data dependent. Always experiment and verify." + ] + }, + { + "metadata": { + "id": "GpV-uF_cBCBU", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Task 2: Try a Different Feature\n", + "\n", + "See if you can do any better by replacing the `total_rooms` feature with the `population` feature.\n", + "\n", + "Don't take more than 5 minutes on this portion." + ] + }, + { + "metadata": { + "id": "YMyOxzb0ZlAH", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# YOUR CODE HERE\n", + "train_model(\n", + " learning_rate=0.00002,\n", + " steps=1000,\n", + " batch_size=5,\n", + " input_feature=\"population\"\n", + ")" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "ci1ISxxrZ7v0", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Solution\n", + "\n", + "Click below for one possible solution." + ] + }, + { + "metadata": { + "id": "SjdQQCduZ7BV", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "train_model(\n", + " learning_rate=0.00002,\n", + " steps=1000,\n", + " batch_size=5,\n", + " input_feature=\"population\"\n", + ")" + ], + "execution_count": 0, + "outputs": [] + } + ] +} \ No newline at end of file diff --git a/improving_neural_net_performance.ipynb b/improving_neural_net_performance.ipynb new file mode 100644 index 0000000..7f8d970 --- /dev/null +++ b/improving_neural_net_performance.ipynb @@ -0,0 +1,1058 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "improving_neural_net_performance.ipynb", + "version": "0.3.2", + "provenance": [], + "collapsed_sections": [ + "JndnmDMp66FL", + "jFfc3saSxg6t", + "FSPZIiYgyh93", + "GhFtWjQRzD2l", + "P8BLQ7T71JWd" + ], + "include_colab_link": true + }, + "kernelspec": { + "name": "python2", + "display_name": "Python 2" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "JndnmDMp66FL" + }, + "cell_type": "markdown", + "source": [ + "#### Copyright 2017 Google LLC." + ] + }, + { + "metadata": { + "cellView": "both", + "colab_type": "code", + "id": "hMqWDc_m6rUC", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "eV16J6oUY-HN" + }, + "cell_type": "markdown", + "source": [ + "# Improving Neural Net Performance" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "0Rwl1iXIKxkm" + }, + "cell_type": "markdown", + "source": [ + "**Learning Objective:** Improve the performance of a neural network by normalizing features and applying various optimization algorithms\n", + "\n", + "**NOTE:** The optimization methods described in this exercise are not specific to neural networks; they are effective means to improve most types of models." + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "lBPTONWzKxkn" + }, + "cell_type": "markdown", + "source": [ + "## Setup\n", + "\n", + "First, we'll load the data." + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "VtYVuONUKxko", + "colab": {} + }, + "cell_type": "code", + "source": [ + "from __future__ import print_function\n", + "\n", + "import math\n", + "\n", + "from IPython import display\n", + "from matplotlib import cm\n", + "from matplotlib import gridspec\n", + "from matplotlib import pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "from sklearn import metrics\n", + "import tensorflow as tf\n", + "from tensorflow.python.data import Dataset\n", + "\n", + "tf.logging.set_verbosity(tf.logging.ERROR)\n", + "pd.options.display.max_rows = 10\n", + "pd.options.display.float_format = '{:.1f}'.format\n", + "\n", + "california_housing_dataframe = pd.read_csv(\"https://download.mlcc.google.com/mledu-datasets/california_housing_train.csv\", sep=\",\")\n", + "\n", + "california_housing_dataframe = california_housing_dataframe.reindex(\n", + " np.random.permutation(california_housing_dataframe.index))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "code", + "id": "B8qC-jTIKxkr", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def preprocess_features(california_housing_dataframe):\n", + " \"\"\"Prepares input features from California housing data set.\n", + "\n", + " Args:\n", + " california_housing_dataframe: A Pandas DataFrame expected to contain data\n", + " from the California housing data set.\n", + " Returns:\n", + " A DataFrame that contains the features to be used for the model, including\n", + " synthetic features.\n", + " \"\"\"\n", + " selected_features = california_housing_dataframe[\n", + " [\"latitude\",\n", + " \"longitude\",\n", + " \"housing_median_age\",\n", + " \"total_rooms\",\n", + " \"total_bedrooms\",\n", + " \"population\",\n", + " \"households\",\n", + " \"median_income\"]]\n", + " processed_features = selected_features.copy()\n", + " # Create a synthetic feature.\n", + " processed_features[\"rooms_per_person\"] = (\n", + " california_housing_dataframe[\"total_rooms\"] /\n", + " california_housing_dataframe[\"population\"])\n", + " return processed_features\n", + "\n", + "def preprocess_targets(california_housing_dataframe):\n", + " \"\"\"Prepares target features (i.e., labels) from California housing data set.\n", + "\n", + " Args:\n", + " california_housing_dataframe: A Pandas DataFrame expected to contain data\n", + " from the California housing data set.\n", + " Returns:\n", + " A DataFrame that contains the target feature.\n", + " \"\"\"\n", + " output_targets = pd.DataFrame()\n", + " # Scale the target to be in units of thousands of dollars.\n", + " output_targets[\"median_house_value\"] = (\n", + " california_housing_dataframe[\"median_house_value\"] / 1000.0)\n", + " return output_targets" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "code", + "id": "Ah6LjMIJ2spZ", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# Choose the first 12000 (out of 17000) examples for training.\n", + "training_examples = preprocess_features(california_housing_dataframe.head(12000))\n", + "training_targets = preprocess_targets(california_housing_dataframe.head(12000))\n", + "\n", + "# Choose the last 5000 (out of 17000) examples for validation.\n", + "validation_examples = preprocess_features(california_housing_dataframe.tail(5000))\n", + "validation_targets = preprocess_targets(california_housing_dataframe.tail(5000))\n", + "\n", + "# Double-check that we've done the right thing.\n", + "print(\"Training examples summary:\")\n", + "display.display(training_examples.describe())\n", + "print(\"Validation examples summary:\")\n", + "display.display(validation_examples.describe())\n", + "\n", + "print(\"Training targets summary:\")\n", + "display.display(training_targets.describe())\n", + "print(\"Validation targets summary:\")\n", + "display.display(validation_targets.describe())" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "NqIbXxx222ea" + }, + "cell_type": "markdown", + "source": [ + "## Train the Neural Network\n", + "\n", + "Next, we'll train the neural network." + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "6k3xYlSg27VB", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def construct_feature_columns(input_features):\n", + " \"\"\"Construct the TensorFlow Feature Columns.\n", + "\n", + " Args:\n", + " input_features: The names of the numerical input features to use.\n", + " Returns:\n", + " A set of feature columns\n", + " \"\"\" \n", + " return set([tf.feature_column.numeric_column(my_feature)\n", + " for my_feature in input_features])" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "code", + "id": "De9jwyy4wTUT", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def my_input_fn(features, targets, batch_size=1, shuffle=True, num_epochs=None):\n", + " \"\"\"Trains a neural network model.\n", + " \n", + " Args:\n", + " features: pandas DataFrame of features\n", + " targets: pandas DataFrame of targets\n", + " batch_size: Size of batches to be passed to the model\n", + " shuffle: True or False. Whether to shuffle the data.\n", + " num_epochs: Number of epochs for which data should be repeated. None = repeat indefinitely\n", + " Returns:\n", + " Tuple of (features, labels) for next data batch\n", + " \"\"\"\n", + " \n", + " # Convert pandas data into a dict of np arrays.\n", + " features = {key:np.array(value) for key,value in dict(features).items()} \n", + " \n", + " # Construct a dataset, and configure batching/repeating.\n", + " ds = Dataset.from_tensor_slices((features,targets)) # warning: 2GB limit\n", + " ds = ds.batch(batch_size).repeat(num_epochs)\n", + " \n", + " # Shuffle the data, if specified.\n", + " if shuffle:\n", + " ds = ds.shuffle(10000)\n", + " \n", + " # Return the next batch of data.\n", + " features, labels = ds.make_one_shot_iterator().get_next()\n", + " return features, labels" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "code", + "id": "W-51R3yIKxk4", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def train_nn_regression_model(\n", + " my_optimizer,\n", + " steps,\n", + " batch_size,\n", + " hidden_units,\n", + " training_examples,\n", + " training_targets,\n", + " validation_examples,\n", + " validation_targets):\n", + " \"\"\"Trains a neural network regression model.\n", + " \n", + " In addition to training, this function also prints training progress information,\n", + " as well as a plot of the training and validation loss over time.\n", + " \n", + " Args:\n", + " my_optimizer: An instance of `tf.train.Optimizer`, the optimizer to use.\n", + " steps: A non-zero `int`, the total number of training steps. A training step\n", + " consists of a forward and backward pass using a single batch.\n", + " batch_size: A non-zero `int`, the batch size.\n", + " hidden_units: A `list` of int values, specifying the number of neurons in each layer.\n", + " training_examples: A `DataFrame` containing one or more columns from\n", + " `california_housing_dataframe` to use as input features for training.\n", + " training_targets: A `DataFrame` containing exactly one column from\n", + " `california_housing_dataframe` to use as target for training.\n", + " validation_examples: A `DataFrame` containing one or more columns from\n", + " `california_housing_dataframe` to use as input features for validation.\n", + " validation_targets: A `DataFrame` containing exactly one column from\n", + " `california_housing_dataframe` to use as target for validation.\n", + " \n", + " Returns:\n", + " A tuple `(estimator, training_losses, validation_losses)`:\n", + " estimator: the trained `DNNRegressor` object.\n", + " training_losses: a `list` containing the training loss values taken during training.\n", + " validation_losses: a `list` containing the validation loss values taken during training.\n", + " \"\"\"\n", + "\n", + " periods = 10\n", + " steps_per_period = steps / periods\n", + " \n", + " # Create a DNNRegressor object.\n", + " my_optimizer = tf.contrib.estimator.clip_gradients_by_norm(my_optimizer, 5.0)\n", + " dnn_regressor = tf.estimator.DNNRegressor(\n", + " feature_columns=construct_feature_columns(training_examples),\n", + " hidden_units=hidden_units,\n", + " optimizer=my_optimizer\n", + " )\n", + " \n", + " # Create input functions.\n", + " training_input_fn = lambda: my_input_fn(training_examples, \n", + " training_targets[\"median_house_value\"], \n", + " batch_size=batch_size)\n", + " predict_training_input_fn = lambda: my_input_fn(training_examples, \n", + " training_targets[\"median_house_value\"], \n", + " num_epochs=1, \n", + " shuffle=False)\n", + " predict_validation_input_fn = lambda: my_input_fn(validation_examples, \n", + " validation_targets[\"median_house_value\"], \n", + " num_epochs=1, \n", + " shuffle=False)\n", + "\n", + " # Train the model, but do so inside a loop so that we can periodically assess\n", + " # loss metrics.\n", + " print(\"Training model...\")\n", + " print(\"RMSE (on training data):\")\n", + " training_rmse = []\n", + " validation_rmse = []\n", + " for period in range (0, periods):\n", + " # Train the model, starting from the prior state.\n", + " dnn_regressor.train(\n", + " input_fn=training_input_fn,\n", + " steps=steps_per_period\n", + " )\n", + " # Take a break and compute predictions.\n", + " training_predictions = dnn_regressor.predict(input_fn=predict_training_input_fn)\n", + " training_predictions = np.array([item['predictions'][0] for item in training_predictions])\n", + " \n", + " validation_predictions = dnn_regressor.predict(input_fn=predict_validation_input_fn)\n", + " validation_predictions = np.array([item['predictions'][0] for item in validation_predictions])\n", + " \n", + " # Compute training and validation loss.\n", + " training_root_mean_squared_error = math.sqrt(\n", + " metrics.mean_squared_error(training_predictions, training_targets))\n", + " validation_root_mean_squared_error = math.sqrt(\n", + " metrics.mean_squared_error(validation_predictions, validation_targets))\n", + " # Occasionally print the current loss.\n", + " print(\" period %02d : %0.2f\" % (period, training_root_mean_squared_error))\n", + " # Add the loss metrics from this period to our list.\n", + " training_rmse.append(training_root_mean_squared_error)\n", + " validation_rmse.append(validation_root_mean_squared_error)\n", + " print(\"Model training finished.\")\n", + "\n", + " # Output a graph of loss metrics over periods.\n", + " plt.ylabel(\"RMSE\")\n", + " plt.xlabel(\"Periods\")\n", + " plt.title(\"Root Mean Squared Error vs. Periods\")\n", + " plt.tight_layout()\n", + " plt.plot(training_rmse, label=\"training\")\n", + " plt.plot(validation_rmse, label=\"validation\")\n", + " plt.legend()\n", + "\n", + " print(\"Final RMSE (on training data): %0.2f\" % training_root_mean_squared_error)\n", + " print(\"Final RMSE (on validation data): %0.2f\" % validation_root_mean_squared_error)\n", + "\n", + " return dnn_regressor, training_rmse, validation_rmse" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "code", + "id": "KueReMZ9Kxk7", + "colab": {} + }, + "cell_type": "code", + "source": [ + "_ = train_nn_regression_model(\n", + " my_optimizer=tf.train.GradientDescentOptimizer(learning_rate=0.0007),\n", + " steps=5000,\n", + " batch_size=70,\n", + " hidden_units=[10, 10],\n", + " training_examples=training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "flxmFt0KKxk9" + }, + "cell_type": "markdown", + "source": [ + "## Linear Scaling\n", + "It can be a good standard practice to normalize the inputs to fall within the range -1, 1. This helps SGD not get stuck taking steps that are too large in one dimension, or too small in another. Fans of numerical optimization may note that there's a connection to the idea of using a preconditioner here." + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "Dws5rIQjKxk-", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def linear_scale(series):\n", + " min_val = series.min()\n", + " max_val = series.max()\n", + " scale = (max_val - min_val) / 2.0\n", + " return series.apply(lambda x:((x - min_val) / scale) - 1.0)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "MVmuHI76N2Sz" + }, + "cell_type": "markdown", + "source": [ + "## Task 1: Normalize the Features Using Linear Scaling\n", + "\n", + "**Normalize the inputs to the scale -1, 1.**\n", + "\n", + "**Spend about 5 minutes training and evaluating on the newly normalized data. How well can you do?**\n", + "\n", + "As a rule of thumb, NN's train best when the input features are roughly on the same scale.\n", + "\n", + "Sanity check your normalized data. (What would happen if you forgot to normalize one feature?)\n" + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "yD948ZgAM6Cx", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def normalize_linear_scale(examples_dataframe):\n", + " \"\"\"Returns a version of the input `DataFrame` that has all its features normalized linearly.\"\"\"\n", + " processed_features = pd.DataFrame()\n", + " processed_features[\"latitude\"] = linear_scale(examples_dataframe[\"latitude\"])\n", + " processed_features[\"longitude\"] = linear_scale(examples_dataframe[\"longitude\"])\n", + " processed_features[\"housing_median_age\"] = linear_scale(examples_dataframe[\"housing_median_age\"])\n", + " processed_features[\"total_rooms\"] = linear_scale(examples_dataframe[\"total_rooms\"])\n", + " processed_features[\"total_bedrooms\"] = linear_scale(examples_dataframe[\"total_bedrooms\"])\n", + " processed_features[\"population\"] = linear_scale(examples_dataframe[\"population\"])\n", + " processed_features[\"households\"] = linear_scale(examples_dataframe[\"households\"])\n", + " processed_features[\"median_income\"] = linear_scale(examples_dataframe[\"median_income\"])\n", + " processed_features[\"rooms_per_person\"] = linear_scale(examples_dataframe[\"rooms_per_person\"])\n", + " return processed_features\n", + "\n", + "normalized_dataframe = normalize_linear_scale(preprocess_features(california_housing_dataframe))\n", + "normalized_training_examples = normalized_dataframe.head(12000)\n", + "normalized_validation_examples = normalized_dataframe.tail(5000)\n", + "\n", + "_ = train_nn_regression_model(\n", + " my_optimizer=tf.train.GradientDescentOptimizer(learning_rate=0.005),\n", + " steps=2000,\n", + " batch_size=50,\n", + " hidden_units=[10, 10],\n", + " training_examples=normalized_training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=normalized_validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "jFfc3saSxg6t" + }, + "cell_type": "markdown", + "source": [ + "### Solution\n", + "\n", + "Click below for one possible solution." + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "Ax_IIQVRx4gr" + }, + "cell_type": "markdown", + "source": [ + "Since normalization uses min and max, we have to ensure it's done on the entire dataset at once. \n", + "\n", + "We can do that here because all our data is in a single DataFrame. If we had multiple data sets, a good practice would be to derive the normalization parameters from the training set and apply those identically to the test set." + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "D-bJBXrJx-U_", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def normalize_linear_scale(examples_dataframe):\n", + " \"\"\"Returns a version of the input `DataFrame` that has all its features normalized linearly.\"\"\"\n", + " processed_features = pd.DataFrame()\n", + " processed_features[\"latitude\"] = linear_scale(examples_dataframe[\"latitude\"])\n", + " processed_features[\"longitude\"] = linear_scale(examples_dataframe[\"longitude\"])\n", + " processed_features[\"housing_median_age\"] = linear_scale(examples_dataframe[\"housing_median_age\"])\n", + " processed_features[\"total_rooms\"] = linear_scale(examples_dataframe[\"total_rooms\"])\n", + " processed_features[\"total_bedrooms\"] = linear_scale(examples_dataframe[\"total_bedrooms\"])\n", + " processed_features[\"population\"] = linear_scale(examples_dataframe[\"population\"])\n", + " processed_features[\"households\"] = linear_scale(examples_dataframe[\"households\"])\n", + " processed_features[\"median_income\"] = linear_scale(examples_dataframe[\"median_income\"])\n", + " processed_features[\"rooms_per_person\"] = linear_scale(examples_dataframe[\"rooms_per_person\"])\n", + " return processed_features\n", + "\n", + "normalized_dataframe = normalize_linear_scale(preprocess_features(california_housing_dataframe))\n", + "normalized_training_examples = normalized_dataframe.head(12000)\n", + "normalized_validation_examples = normalized_dataframe.tail(5000)\n", + "\n", + "_ = train_nn_regression_model(\n", + " my_optimizer=tf.train.GradientDescentOptimizer(learning_rate=0.005),\n", + " steps=2000,\n", + " batch_size=50,\n", + " hidden_units=[10, 10],\n", + " training_examples=normalized_training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=normalized_validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "MrwtdStNJ6ZQ" + }, + "cell_type": "markdown", + "source": [ + "## Task 2: Try a Different Optimizer\n", + "\n", + "** Use the Adagrad and Adam optimizers and compare performance.**\n", + "\n", + "The Adagrad optimizer is one alternative. The key insight of Adagrad is that it modifies the learning rate adaptively for each coefficient in a model, monotonically lowering the effective learning rate. This works great for convex problems, but isn't always ideal for the non-convex problem Neural Net training. You can use Adagrad by specifying `AdagradOptimizer` instead of `GradientDescentOptimizer`. Note that you may need to use a larger learning rate with Adagrad.\n", + "\n", + "For non-convex optimization problems, Adam is sometimes more efficient than Adagrad. To use Adam, invoke the `tf.train.AdamOptimizer` method. This method takes several optional hyperparameters as arguments, but our solution only specifies one of these (`learning_rate`). In a production setting, you should specify and tune the optional hyperparameters carefully." + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "61GSlDvF7-7q", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#\n", + "# YOUR CODE HERE: Retrain the network using Adagrad and then Adam.\n", + "#\n", + "_, adagrad_training_losses, adagrad_validation_losses = train_nn_regression_model(\n", + " my_optimizer=tf.train.AdagradOptimizer(learning_rate=0.5),\n", + " steps=500,\n", + " batch_size=100,\n", + " hidden_units=[10, 10],\n", + " training_examples=normalized_training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=normalized_validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "3GTIE8n-TGpB", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "_, adagrad_training_losses, adagrad_validation_losses = train_nn_regression_model(\n", + " my_optimizer=tf.train.AdagradOptimizer(learning_rate=0.5),\n", + " steps=500,\n", + " batch_size=100,\n", + " hidden_units=[10, 10],\n", + " training_examples=normalized_training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=normalized_validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "_7iFzeTuTQg6", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "plt.ylabel(\"RMSE\")\n", + "plt.xlabel(\"Periods\")\n", + "plt.title(\"Root Mean Squared Error vs. Periods\")\n", + "plt.plot(adagrad_training_losses, label='Adagrad training')\n", + "plt.plot(adagrad_validation_losses, label='Adagrad validation')\n", + "plt.plot(adam_training_losses, label='Adam training')\n", + "plt.plot(adam_validation_losses, label='Adam validation')\n", + "_ = plt.legend()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "FSPZIiYgyh93" + }, + "cell_type": "markdown", + "source": [ + "### Solution\n", + "\n", + "Click below for the solution" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "X1QcIeiKyni4" + }, + "cell_type": "markdown", + "source": [ + "First, let's try Adagrad." + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "Ntn4jJxnypGZ", + "colab": {} + }, + "cell_type": "code", + "source": [ + "_, adagrad_training_losses, adagrad_validation_losses = train_nn_regression_model(\n", + " my_optimizer=tf.train.AdagradOptimizer(learning_rate=0.5),\n", + " steps=500,\n", + " batch_size=100,\n", + " hidden_units=[10, 10],\n", + " training_examples=normalized_training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=normalized_validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "5JUsCdRRyso3" + }, + "cell_type": "markdown", + "source": [ + "Now let's try Adam." + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "lZB8k0upyuY8", + "colab": {} + }, + "cell_type": "code", + "source": [ + "_, adam_training_losses, adam_validation_losses = train_nn_regression_model(\n", + " my_optimizer=tf.train.AdamOptimizer(learning_rate=0.009),\n", + " steps=500,\n", + " batch_size=100,\n", + " hidden_units=[10, 10],\n", + " training_examples=normalized_training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=normalized_validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "twYgC8FGyxm6" + }, + "cell_type": "markdown", + "source": [ + "Let's print a graph of loss metrics side by side." + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "8RHIUEfqyzW0", + "colab": {} + }, + "cell_type": "code", + "source": [ + "plt.ylabel(\"RMSE\")\n", + "plt.xlabel(\"Periods\")\n", + "plt.title(\"Root Mean Squared Error vs. Periods\")\n", + "plt.plot(adagrad_training_losses, label='Adagrad training')\n", + "plt.plot(adagrad_validation_losses, label='Adagrad validation')\n", + "plt.plot(adam_training_losses, label='Adam training')\n", + "plt.plot(adam_validation_losses, label='Adam validation')\n", + "_ = plt.legend()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "UySPl7CAQ28C" + }, + "cell_type": "markdown", + "source": [ + "## Task 3: Explore Alternate Normalization Methods\n", + "\n", + "**Try alternate normalizations for various features to further improve performance.**\n", + "\n", + "If you look closely at summary stats for your transformed data, you may notice that linear scaling some features leaves them clumped close to `-1`.\n", + "\n", + "For example, many features have a median of `-0.8` or so, rather than `0.0`." + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "QWmm_6CGKxlH", + "colab": {} + }, + "cell_type": "code", + "source": [ + "_ = normalized_training_examples.hist(bins=20, figsize=(18, 12), xlabelsize=10)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "Xx9jgEMHKxlJ" + }, + "cell_type": "markdown", + "source": [ + "We might be able to do better by choosing additional ways to transform these features.\n", + "\n", + "For example, a log scaling might help some features. Or clipping extreme values may make the remainder of the scale more informative." + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "baKZa6MEKxlK", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def log_normalize(series):\n", + " return series.apply(lambda x:math.log(x+1.0))\n", + "\n", + "def clip(series, clip_to_min, clip_to_max):\n", + " return series.apply(lambda x:(\n", + " min(max(x, clip_to_min), clip_to_max)))\n", + "\n", + "def z_score_normalize(series):\n", + " mean = series.mean()\n", + " std_dv = series.std()\n", + " return series.apply(lambda x:(x - mean) / std_dv)\n", + "\n", + "def binary_threshold(series, threshold):\n", + " return series.apply(lambda x:(1 if x > threshold else 0))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "-wCCq_ClKxlO" + }, + "cell_type": "markdown", + "source": [ + "The block above contains a few additional possible normalization functions. Try some of these, or add your own.\n", + "\n", + "Note that if you normalize the target, you'll need to un-normalize the predictions for loss metrics to be comparable." + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "8ToG-mLfMO9P", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def normalize(examples_dataframe):\n", + " \"\"\"Returns a version of the input `DataFrame` that has all its features normalized.\"\"\"\n", + " #\n", + " # YOUR CODE HERE: Normalize the inputs.\n", + " #\n", + " processed_features = pd.DataFrame()\n", + "\n", + " processed_features[\"households\"] = log_normalize(examples_dataframe[\"households\"])\n", + " processed_features[\"median_income\"] = log_normalize(examples_dataframe[\"median_income\"])\n", + " processed_features[\"total_bedrooms\"] = log_normalize(examples_dataframe[\"total_bedrooms\"])\n", + " \n", + " processed_features[\"latitude\"] = linear_scale(examples_dataframe[\"latitude\"])\n", + " processed_features[\"longitude\"] = linear_scale(examples_dataframe[\"longitude\"])\n", + " processed_features[\"housing_median_age\"] = linear_scale(examples_dataframe[\"housing_median_age\"])\n", + "\n", + " processed_features[\"population\"] = linear_scale(clip(examples_dataframe[\"population\"], 0, 5000))\n", + " processed_features[\"rooms_per_person\"] = linear_scale(clip(examples_dataframe[\"rooms_per_person\"], 0, 5))\n", + " processed_features[\"total_rooms\"] = linear_scale(clip(examples_dataframe[\"total_rooms\"], 0, 10000))\n", + "\n", + " return processed_features\n", + "\n", + " \n", + "\n", + "normalized_dataframe = normalize(preprocess_features(california_housing_dataframe))\n", + "normalized_training_examples = normalized_dataframe.head(12000)\n", + "normalized_validation_examples = normalized_dataframe.tail(5000)\n", + "\n", + "_ = train_nn_regression_model(\n", + " my_optimizer=tf.train.GradientDescentOptimizer(learning_rate=0.0007),\n", + " steps=5000,\n", + " batch_size=70,\n", + " hidden_units=[10, 10],\n", + " training_examples=normalized_training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=normalized_validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "GhFtWjQRzD2l" + }, + "cell_type": "markdown", + "source": [ + "### Solution\n", + "\n", + "Click below for one possible solution." + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "OMoIsUMmzK9b" + }, + "cell_type": "markdown", + "source": [ + "These are only a few ways in which we could think about the data. Other transformations may work even better!\n", + "\n", + "`households`, `median_income` and `total_bedrooms` all appear normally-distributed in a log space.\n", + "\n", + "`latitude`, `longitude` and `housing_median_age` would probably be better off just scaled linearly, as before.\n", + "\n", + "`population`, `totalRooms` and `rooms_per_person` have a few extreme outliers. They seem too extreme for log normalization to help. So let's clip them instead." + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "XDEYkPquzYCH", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def normalize(examples_dataframe):\n", + " \"\"\"Returns a version of the input `DataFrame` that has all its features normalized.\"\"\"\n", + " processed_features = pd.DataFrame()\n", + "\n", + " processed_features[\"households\"] = log_normalize(examples_dataframe[\"households\"])\n", + " processed_features[\"median_income\"] = log_normalize(examples_dataframe[\"median_income\"])\n", + " processed_features[\"total_bedrooms\"] = log_normalize(examples_dataframe[\"total_bedrooms\"])\n", + " \n", + " processed_features[\"latitude\"] = linear_scale(examples_dataframe[\"latitude\"])\n", + " processed_features[\"longitude\"] = linear_scale(examples_dataframe[\"longitude\"])\n", + " processed_features[\"housing_median_age\"] = linear_scale(examples_dataframe[\"housing_median_age\"])\n", + "\n", + " processed_features[\"population\"] = linear_scale(clip(examples_dataframe[\"population\"], 0, 5000))\n", + " processed_features[\"rooms_per_person\"] = linear_scale(clip(examples_dataframe[\"rooms_per_person\"], 0, 5))\n", + " processed_features[\"total_rooms\"] = linear_scale(clip(examples_dataframe[\"total_rooms\"], 0, 10000))\n", + "\n", + " return processed_features\n", + "\n", + "normalized_dataframe = normalize(preprocess_features(california_housing_dataframe))\n", + "normalized_training_examples = normalized_dataframe.head(12000)\n", + "normalized_validation_examples = normalized_dataframe.tail(5000)\n", + "\n", + "_ = train_nn_regression_model(\n", + " my_optimizer=tf.train.AdagradOptimizer(learning_rate=0.15),\n", + " steps=1000,\n", + " batch_size=50,\n", + " hidden_units=[10, 10],\n", + " training_examples=normalized_training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=normalized_validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "b7atJTbzU9Ca" + }, + "cell_type": "markdown", + "source": [ + "## Optional Challenge: Use only Latitude and Longitude Features\n", + "\n", + "**Train a NN model that uses only latitude and longitude as features.**\n", + "\n", + "Real estate people are fond of saying that location is the only important feature in housing price.\n", + "Let's see if we can confirm this by training a model that uses only latitude and longitude as features.\n", + "\n", + "This will only work well if our NN can learn complex nonlinearities from latitude and longitude.\n", + "\n", + "**NOTE:** We may need a network structure that has more layers than were useful earlier in the exercise." + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "T5McjahpamOc", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#\n", + "# YOUR CODE HERE: Train the network using only latitude and longitude\n", + "#" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "P8BLQ7T71JWd" + }, + "cell_type": "markdown", + "source": [ + "### Solution\n", + "\n", + "Click below for a possible solution." + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "1hwaFCE71OPZ" + }, + "cell_type": "markdown", + "source": [ + "It's a good idea to keep latitude and longitude normalized:" + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "djKtt4mz1ZEc", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def location_location_location(examples_dataframe):\n", + " \"\"\"Returns a version of the input `DataFrame` that keeps only the latitude and longitude.\"\"\"\n", + " processed_features = pd.DataFrame()\n", + " processed_features[\"latitude\"] = linear_scale(examples_dataframe[\"latitude\"])\n", + " processed_features[\"longitude\"] = linear_scale(examples_dataframe[\"longitude\"])\n", + " return processed_features\n", + "\n", + "lll_dataframe = location_location_location(preprocess_features(california_housing_dataframe))\n", + "lll_training_examples = lll_dataframe.head(12000)\n", + "lll_validation_examples = lll_dataframe.tail(5000)\n", + "\n", + "_ = train_nn_regression_model(\n", + " my_optimizer=tf.train.AdagradOptimizer(learning_rate=0.05),\n", + " steps=500,\n", + " batch_size=50,\n", + " hidden_units=[10, 10, 5, 5, 5],\n", + " training_examples=lll_training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=lll_validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "Dw2Mr9JZ1cRi" + }, + "cell_type": "markdown", + "source": [ + "This isn't too bad for just two features. Of course, property values can still vary significantly within short distances." + ] + } + ] +} \ No newline at end of file diff --git a/intro_to_neural_nets.ipynb b/intro_to_neural_nets.ipynb new file mode 100644 index 0000000..98e5988 --- /dev/null +++ b/intro_to_neural_nets.ipynb @@ -0,0 +1,619 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "intro_to_neural_nets.ipynb", + "version": "0.3.2", + "provenance": [], + "collapsed_sections": [ + "JndnmDMp66FL", + "O2q5RRCKqYaU", + "vvT2jDWjrKew" + ], + "include_colab_link": true + }, + "kernelspec": { + "name": "python2", + "display_name": "Python 2" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "metadata": { + "id": "JndnmDMp66FL", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "#### Copyright 2017 Google LLC." + ] + }, + { + "metadata": { + "id": "hMqWDc_m6rUC", + "colab_type": "code", + "cellView": "both", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "eV16J6oUY-HN", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Intro to Neural Networks" + ] + }, + { + "metadata": { + "id": "_wIcUFLSKNdx", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "**Learning Objectives:**\n", + " * Define a neural network (NN) and its hidden layers using the TensorFlow `DNNRegressor` class\n", + " * Train a neural network to learn nonlinearities in a dataset and achieve better performance than a linear regression model" + ] + }, + { + "metadata": { + "id": "_ZZ7f7prKNdy", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "In the previous exercises, we used synthetic features to help our model incorporate nonlinearities.\n", + "\n", + "One important set of nonlinearities was around latitude and longitude, but there may be others.\n", + "\n", + "We'll also switch back, for now, to a standard regression task, rather than the logistic regression task from the previous exercise. That is, we'll be predicting `median_house_value` directly." + ] + }, + { + "metadata": { + "id": "J2kqX6VZTHUy", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Setup\n", + "\n", + "First, let's load and prepare the data." + ] + }, + { + "metadata": { + "id": "AGOM1TUiKNdz", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "from __future__ import print_function\n", + "\n", + "import math\n", + "\n", + "from IPython import display\n", + "from matplotlib import cm\n", + "from matplotlib import gridspec\n", + "from matplotlib import pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "from sklearn import metrics\n", + "import tensorflow as tf\n", + "from tensorflow.python.data import Dataset\n", + "\n", + "tf.logging.set_verbosity(tf.logging.ERROR)\n", + "pd.options.display.max_rows = 10\n", + "pd.options.display.float_format = '{:.1f}'.format\n", + "\n", + "california_housing_dataframe = pd.read_csv(\"https://download.mlcc.google.com/mledu-datasets/california_housing_train.csv\", sep=\",\")\n", + "\n", + "california_housing_dataframe = california_housing_dataframe.reindex(\n", + " np.random.permutation(california_housing_dataframe.index))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "2I8E2qhyKNd4", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def preprocess_features(california_housing_dataframe):\n", + " \"\"\"Prepares input features from California housing data set.\n", + "\n", + " Args:\n", + " california_housing_dataframe: A Pandas DataFrame expected to contain data\n", + " from the California housing data set.\n", + " Returns:\n", + " A DataFrame that contains the features to be used for the model, including\n", + " synthetic features.\n", + " \"\"\"\n", + " selected_features = california_housing_dataframe[\n", + " [\"latitude\",\n", + " \"longitude\",\n", + " \"housing_median_age\",\n", + " \"total_rooms\",\n", + " \"total_bedrooms\",\n", + " \"population\",\n", + " \"households\",\n", + " \"median_income\"]]\n", + " processed_features = selected_features.copy()\n", + " # Create a synthetic feature.\n", + " processed_features[\"rooms_per_person\"] = (\n", + " california_housing_dataframe[\"total_rooms\"] /\n", + " california_housing_dataframe[\"population\"])\n", + " return processed_features\n", + "\n", + "def preprocess_targets(california_housing_dataframe):\n", + " \"\"\"Prepares target features (i.e., labels) from California housing data set.\n", + "\n", + " Args:\n", + " california_housing_dataframe: A Pandas DataFrame expected to contain data\n", + " from the California housing data set.\n", + " Returns:\n", + " A DataFrame that contains the target feature.\n", + " \"\"\"\n", + " output_targets = pd.DataFrame()\n", + " # Scale the target to be in units of thousands of dollars.\n", + " output_targets[\"median_house_value\"] = (\n", + " california_housing_dataframe[\"median_house_value\"] / 1000.0)\n", + " return output_targets" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "pQzcj2B1T5dA", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# Choose the first 12000 (out of 17000) examples for training.\n", + "training_examples = preprocess_features(california_housing_dataframe.head(12000))\n", + "training_targets = preprocess_targets(california_housing_dataframe.head(12000))\n", + "\n", + "# Choose the last 5000 (out of 17000) examples for validation.\n", + "validation_examples = preprocess_features(california_housing_dataframe.tail(5000))\n", + "validation_targets = preprocess_targets(california_housing_dataframe.tail(5000))\n", + "\n", + "# Double-check that we've done the right thing.\n", + "print(\"Training examples summary:\")\n", + "display.display(training_examples.describe())\n", + "print(\"Validation examples summary:\")\n", + "display.display(validation_examples.describe())\n", + "\n", + "print(\"Training targets summary:\")\n", + "display.display(training_targets.describe())\n", + "print(\"Validation targets summary:\")\n", + "display.display(validation_targets.describe())" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "RWq0xecNKNeG", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Building a Neural Network\n", + "\n", + "The NN is defined by the [DNNRegressor](https://www.tensorflow.org/api_docs/python/tf/estimator/DNNRegressor) class.\n", + "\n", + "Use **`hidden_units`** to define the structure of the NN. The `hidden_units` argument provides a list of ints, where each int corresponds to a hidden layer and indicates the number of nodes in it. For example, consider the following assignment:\n", + "\n", + "`hidden_units=[3,10]`\n", + "\n", + "The preceding assignment specifies a neural net with two hidden layers:\n", + "\n", + "* The first hidden layer contains 3 nodes.\n", + "* The second hidden layer contains 10 nodes.\n", + "\n", + "If we wanted to add more layers, we'd add more ints to the list. For example, `hidden_units=[10,20,30,40]` would create four layers with ten, twenty, thirty, and forty units, respectively.\n", + "\n", + "By default, all hidden layers will use ReLu activation and will be fully connected." + ] + }, + { + "metadata": { + "id": "ni0S6zHcTb04", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def construct_feature_columns(input_features):\n", + " \"\"\"Construct the TensorFlow Feature Columns.\n", + "\n", + " Args:\n", + " input_features: The names of the numerical input features to use.\n", + " Returns:\n", + " A set of feature columns\n", + " \"\"\" \n", + " return set([tf.feature_column.numeric_column(my_feature)\n", + " for my_feature in input_features])" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "zvCqgNdzpaFg", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def my_input_fn(features, targets, batch_size=1, shuffle=True, num_epochs=None):\n", + " \"\"\"Trains a neural net regression model.\n", + " \n", + " Args:\n", + " features: pandas DataFrame of features\n", + " targets: pandas DataFrame of targets\n", + " batch_size: Size of batches to be passed to the model\n", + " shuffle: True or False. Whether to shuffle the data.\n", + " num_epochs: Number of epochs for which data should be repeated. None = repeat indefinitely\n", + " Returns:\n", + " Tuple of (features, labels) for next data batch\n", + " \"\"\"\n", + " \n", + " # Convert pandas data into a dict of np arrays.\n", + " features = {key:np.array(value) for key,value in dict(features).items()} \n", + " \n", + " # Construct a dataset, and configure batching/repeating.\n", + " ds = Dataset.from_tensor_slices((features,targets)) # warning: 2GB limit\n", + " ds = ds.batch(batch_size).repeat(num_epochs)\n", + " \n", + " # Shuffle the data, if specified.\n", + " if shuffle:\n", + " ds = ds.shuffle(10000)\n", + " \n", + " # Return the next batch of data.\n", + " features, labels = ds.make_one_shot_iterator().get_next()\n", + " return features, labels" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "U52Ychv9KNeH", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def train_nn_regression_model(\n", + " learning_rate,\n", + " steps,\n", + " batch_size,\n", + " hidden_units,\n", + " training_examples,\n", + " training_targets,\n", + " validation_examples,\n", + " validation_targets):\n", + " \"\"\"Trains a neural network regression model.\n", + " \n", + " In addition to training, this function also prints training progress information,\n", + " as well as a plot of the training and validation loss over time.\n", + " \n", + " Args:\n", + " learning_rate: A `float`, the learning rate.\n", + " steps: A non-zero `int`, the total number of training steps. A training step\n", + " consists of a forward and backward pass using a single batch.\n", + " batch_size: A non-zero `int`, the batch size.\n", + " hidden_units: A `list` of int values, specifying the number of neurons in each layer.\n", + " training_examples: A `DataFrame` containing one or more columns from\n", + " `california_housing_dataframe` to use as input features for training.\n", + " training_targets: A `DataFrame` containing exactly one column from\n", + " `california_housing_dataframe` to use as target for training.\n", + " validation_examples: A `DataFrame` containing one or more columns from\n", + " `california_housing_dataframe` to use as input features for validation.\n", + " validation_targets: A `DataFrame` containing exactly one column from\n", + " `california_housing_dataframe` to use as target for validation.\n", + " \n", + " Returns:\n", + " A `DNNRegressor` object trained on the training data.\n", + " \"\"\"\n", + "\n", + " periods = 10\n", + " steps_per_period = steps / periods\n", + " \n", + " # Create a DNNRegressor object.\n", + " my_optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n", + " my_optimizer = tf.contrib.estimator.clip_gradients_by_norm(my_optimizer, 5.0)\n", + " dnn_regressor = tf.estimator.DNNRegressor(\n", + " feature_columns=construct_feature_columns(training_examples),\n", + " hidden_units=hidden_units,\n", + " optimizer=my_optimizer,\n", + " )\n", + " \n", + " # Create input functions.\n", + " training_input_fn = lambda: my_input_fn(training_examples, \n", + " training_targets[\"median_house_value\"], \n", + " batch_size=batch_size)\n", + " predict_training_input_fn = lambda: my_input_fn(training_examples, \n", + " training_targets[\"median_house_value\"], \n", + " num_epochs=1, \n", + " shuffle=False)\n", + " predict_validation_input_fn = lambda: my_input_fn(validation_examples, \n", + " validation_targets[\"median_house_value\"], \n", + " num_epochs=1, \n", + " shuffle=False)\n", + "\n", + " # Train the model, but do so inside a loop so that we can periodically assess\n", + " # loss metrics.\n", + " print(\"Training model...\")\n", + " print(\"RMSE (on training data):\")\n", + " training_rmse = []\n", + " validation_rmse = []\n", + " for period in range (0, periods):\n", + " # Train the model, starting from the prior state.\n", + " dnn_regressor.train(\n", + " input_fn=training_input_fn,\n", + " steps=steps_per_period\n", + " )\n", + " # Take a break and compute predictions.\n", + " training_predictions = dnn_regressor.predict(input_fn=predict_training_input_fn)\n", + " training_predictions = np.array([item['predictions'][0] for item in training_predictions])\n", + " \n", + " validation_predictions = dnn_regressor.predict(input_fn=predict_validation_input_fn)\n", + " validation_predictions = np.array([item['predictions'][0] for item in validation_predictions])\n", + " \n", + " # Compute training and validation loss.\n", + " training_root_mean_squared_error = math.sqrt(\n", + " metrics.mean_squared_error(training_predictions, training_targets))\n", + " validation_root_mean_squared_error = math.sqrt(\n", + " metrics.mean_squared_error(validation_predictions, validation_targets))\n", + " # Occasionally print the current loss.\n", + " print(\" period %02d : %0.2f\" % (period, training_root_mean_squared_error))\n", + " # Add the loss metrics from this period to our list.\n", + " training_rmse.append(training_root_mean_squared_error)\n", + " validation_rmse.append(validation_root_mean_squared_error)\n", + " print(\"Model training finished.\")\n", + "\n", + " # Output a graph of loss metrics over periods.\n", + " plt.ylabel(\"RMSE\")\n", + " plt.xlabel(\"Periods\")\n", + " plt.title(\"Root Mean Squared Error vs. Periods\")\n", + " plt.tight_layout()\n", + " plt.plot(training_rmse, label=\"training\")\n", + " plt.plot(validation_rmse, label=\"validation\")\n", + " plt.legend()\n", + "\n", + " print(\"Final RMSE (on training data): %0.2f\" % training_root_mean_squared_error)\n", + " print(\"Final RMSE (on validation data): %0.2f\" % validation_root_mean_squared_error)\n", + "\n", + " return dnn_regressor" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "2QhdcCy-Y8QR", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Task 1: Train a NN Model\n", + "\n", + "**Adjust hyperparameters, aiming to drop RMSE below 110.**\n", + "\n", + "Run the following block to train a NN model. \n", + "\n", + "Recall that in the linear regression exercise with many features, an RMSE of 110 or so was pretty good. We'll aim to beat that.\n", + "\n", + "Your task here is to modify various learning settings to improve accuracy on validation data.\n", + "\n", + "Overfitting is a real potential hazard for NNs. You can look at the gap between loss on training data and loss on validation data to help judge if your model is starting to overfit. If the gap starts to grow, that is usually a sure sign of overfitting.\n", + "\n", + "Because of the number of different possible settings, it's strongly recommended that you take notes on each trial to help guide your development process.\n", + "\n", + "Also, when you get a good setting, try running it multiple times and see how repeatable your result is. NN weights are typically initialized to small random values, so you should see differences from run to run.\n" + ] + }, + { + "metadata": { + "id": "rXmtSW1yKNeK", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "dnn_regressor = train_nn_regression_model(\n", + " learning_rate=0.001,\n", + " steps=2000,\n", + " batch_size=100,\n", + " hidden_units=[10, 10],\n", + " training_examples=training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "O2q5RRCKqYaU", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Solution\n", + "\n", + "Click below to see a possible solution" + ] + }, + { + "metadata": { + "id": "j2Yd5VfrqcC3", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "**NOTE:** This selection of parameters is somewhat arbitrary. Here we've tried combinations that are increasingly complex, combined with training for longer, until the error falls below our objective (training is nondeterministic, so results may fluctuate a bit each time you run the solution). This may not be the best combination; others may attain an even lower RMSE. If your aim is to find the model that can attain the best error, then you'll want to use a more rigorous process, like a parameter search." + ] + }, + { + "metadata": { + "id": "IjkpSqmxqnSM", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "dnn_regressor = train_nn_regression_model(\n", + " learning_rate=0.001,\n", + " steps=2000,\n", + " batch_size=100,\n", + " hidden_units=[10, 10],\n", + " training_examples=training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "c6diezCSeH4Y", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Task 2: Evaluate on Test Data\n", + "\n", + "**Confirm that your validation performance results hold up on test data.**\n", + "\n", + "Once you have a model you're happy with, evaluate it on test data to compare that to validation performance.\n", + "\n", + "Reminder, the test data set is located [here](https://download.mlcc.google.com/mledu-datasets/california_housing_test.csv)." + ] + }, + { + "metadata": { + "id": "icEJIl5Vp51r", + "colab_type": "code", + "cellView": "both", + "colab": {} + }, + "cell_type": "code", + "source": [ + "california_housing_test_data = pd.read_csv(\"https://download.mlcc.google.com/mledu-datasets/california_housing_test.csv\", sep=\",\")\n", + "\n", + "# YOUR CODE HERE\n", + "test_examples = preprocess_features(california_housing_test_data)\n", + "test_targets = preprocess_targets(california_housing_test_data)\n", + "\n", + "predict_testing_input_fn = lambda: my_input_fn(test_examples, \n", + " test_targets[\"median_house_value\"], \n", + " num_epochs=1, \n", + " shuffle=False)\n", + "\n", + "test_predictions = dnn_regressor.predict(input_fn=predict_testing_input_fn)\n", + "test_predictions = np.array([item['predictions'][0] for item in test_predictions])\n", + "\n", + "root_mean_squared_error = math.sqrt(\n", + " metrics.mean_squared_error(test_predictions, test_targets))\n", + "\n", + "print(\"Final RMSE (on test data): %0.2f\" % root_mean_squared_error)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "vvT2jDWjrKew", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Solution\n", + "\n", + "Click below to see a possible solution." + ] + }, + { + "metadata": { + "id": "FyDh7Qy6rQb0", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Similar to what the code at the top does, we just need to load the appropriate data file, preprocess it and call predict and mean_squared_error.\n", + "\n", + "Note that we don't have to randomize the test data, since we will use all records." + ] + }, + { + "metadata": { + "id": "vhb0CtdvrWZx", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "california_housing_test_data = pd.read_csv(\"https://download.mlcc.google.com/mledu-datasets/california_housing_test.csv\", sep=\",\")\n", + "\n", + "test_examples = preprocess_features(california_housing_test_data)\n", + "test_targets = preprocess_targets(california_housing_test_data)\n", + "\n", + "predict_testing_input_fn = lambda: my_input_fn(test_examples, \n", + " test_targets[\"median_house_value\"], \n", + " num_epochs=1, \n", + " shuffle=False)\n", + "\n", + "test_predictions = dnn_regressor.predict(input_fn=predict_testing_input_fn)\n", + "test_predictions = np.array([item['predictions'][0] for item in test_predictions])\n", + "\n", + "root_mean_squared_error = math.sqrt(\n", + " metrics.mean_squared_error(test_predictions, test_targets))\n", + "\n", + "print(\"Final RMSE (on test data): %0.2f\" % root_mean_squared_error)" + ], + "execution_count": 0, + "outputs": [] + } + ] +} \ No newline at end of file diff --git a/intro_to_pandas.ipynb b/intro_to_pandas.ipynb new file mode 100644 index 0000000..23906c8 --- /dev/null +++ b/intro_to_pandas.ipynb @@ -0,0 +1,672 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "intro_to_pandas.ipynb", + "version": "0.3.2", + "provenance": [], + "collapsed_sections": [ + "JndnmDMp66FL", + "YHIWvc9Ms-Ll", + "TJffr5_Jwqvd" + ], + "include_colab_link": true + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "JndnmDMp66FL" + }, + "cell_type": "markdown", + "source": [ + "#### Copyright 2017 Google LLC." + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "hMqWDc_m6rUC", + "cellView": "both", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "I0L5IypUyffB", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "rHLcriKWLRe4" + }, + "cell_type": "markdown", + "source": [ + "# Intro to pandas" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "QvJBqX8_Bctk" + }, + "cell_type": "markdown", + "source": [ + "**Learning Objectives:**\n", + " * Gain an introduction to the `DataFrame` and `Series` data structures of the *pandas* library\n", + " * Access and manipulate data within a `DataFrame` and `Series`\n", + " * Import CSV data into a *pandas* `DataFrame`\n", + " * Reindex a `DataFrame` to shuffle data" + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "TIFJ83ZTBctl" + }, + "cell_type": "markdown", + "source": [ + "[*pandas*](http://pandas.pydata.org/) is a column-oriented data analysis API. It's a great tool for handling and analyzing input data, and many ML frameworks support *pandas* data structures as inputs.\n", + "Although a comprehensive introduction to the *pandas* API would span many pages, the core concepts are fairly straightforward, and we'll present them below. For a more complete reference, the [*pandas* docs site](http://pandas.pydata.org/pandas-docs/stable/index.html) contains extensive documentation and many tutorials." + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "s_JOISVgmn9v" + }, + "cell_type": "markdown", + "source": [ + "## Basic Concepts\n", + "\n", + "The following line imports the *pandas* API and prints the API version:" + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "aSRYu62xUi3g", + "colab": {} + }, + "cell_type": "code", + "source": [ + "from __future__ import print_function\n", + "\n", + "import pandas as pd\n", + "pd.__version__" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "daQreKXIUslr" + }, + "cell_type": "markdown", + "source": [ + "The primary data structures in *pandas* are implemented as two classes:\n", + "\n", + " * **`DataFrame`**, which you can imagine as a relational data table, with rows and named columns.\n", + " * **`Series`**, which is a single column. A `DataFrame` contains one or more `Series` and a name for each `Series`.\n", + "\n", + "The data frame is a commonly used abstraction for data manipulation. Similar implementations exist in [Spark](https://spark.apache.org/) and [R](https://www.r-project.org/about.html)." + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "fjnAk1xcU0yc" + }, + "cell_type": "markdown", + "source": [ + "One way to create a `Series` is to construct a `Series` object. For example:" + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "DFZ42Uq7UFDj", + "colab": {} + }, + "cell_type": "code", + "source": [ + "pd.Series(['San Francisco', 'San Jose', 'Sacramento'])" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "U5ouUp1cU6pC" + }, + "cell_type": "markdown", + "source": [ + "`DataFrame` objects can be created by passing a `dict` mapping `string` column names to their respective `Series`. If the `Series` don't match in length, missing values are filled with special [NA/NaN](http://pandas.pydata.org/pandas-docs/stable/missing_data.html) values. Example:" + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "avgr6GfiUh8t", + "colab": {} + }, + "cell_type": "code", + "source": [ + "city_names = pd.Series(['San Francisco', 'San Jose', 'Sacramento'])\n", + "population = pd.Series([852469, 1015785, 485199])\n", + "\n", + "pd.DataFrame({ 'City name': city_names, 'Population': population })" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "oa5wfZT7VHJl" + }, + "cell_type": "markdown", + "source": [ + "But most of the time, you load an entire file into a `DataFrame`. The following example loads a file with California housing data. Run the following cell to load the data and create feature definitions:" + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "av6RYOraVG1V", + "colab": {} + }, + "cell_type": "code", + "source": [ + "california_housing_dataframe = pd.read_csv(\"https://download.mlcc.google.com/mledu-datasets/california_housing_train.csv\", sep=\",\")\n", + "california_housing_dataframe.describe()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "WrkBjfz5kEQu" + }, + "cell_type": "markdown", + "source": [ + "The example above used `DataFrame.describe` to show interesting statistics about a `DataFrame`. Another useful function is `DataFrame.head`, which displays the first few records of a `DataFrame`:" + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "s3ND3bgOkB5k", + "colab": {} + }, + "cell_type": "code", + "source": [ + "california_housing_dataframe.head()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "w9-Es5Y6laGd" + }, + "cell_type": "markdown", + "source": [ + "Another powerful feature of *pandas* is graphing. For example, `DataFrame.hist` lets you quickly study the distribution of values in a column:" + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "nqndFVXVlbPN", + "colab": {} + }, + "cell_type": "code", + "source": [ + "california_housing_dataframe.hist('housing_median_age')" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "XtYZ7114n3b-" + }, + "cell_type": "markdown", + "source": [ + "## Accessing Data\n", + "\n", + "You can access `DataFrame` data using familiar Python dict/list operations:" + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "_TFm7-looBFF", + "colab": {} + }, + "cell_type": "code", + "source": [ + "cities = pd.DataFrame({ 'City name': city_names, 'Population': population })\n", + "print(type(cities['City name']))\n", + "cities['City name']" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "code", + "id": "V5L6xacLoxyv", + "colab": {} + }, + "cell_type": "code", + "source": [ + "print(type(cities['City name'][1]))\n", + "cities['City name'][1]" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "code", + "id": "gcYX1tBPugZl", + "colab": {} + }, + "cell_type": "code", + "source": [ + "print(type(cities[0:2]))\n", + "cities[0:2]" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "65g1ZdGVjXsQ" + }, + "cell_type": "markdown", + "source": [ + "In addition, *pandas* provides an extremely rich API for advanced [indexing and selection](http://pandas.pydata.org/pandas-docs/stable/indexing.html) that is too extensive to be covered here." + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "RM1iaD-ka3Y1" + }, + "cell_type": "markdown", + "source": [ + "## Manipulating Data\n", + "\n", + "You may apply Python's basic arithmetic operations to `Series`. For example:" + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "XWmyCFJ5bOv-", + "colab": {} + }, + "cell_type": "code", + "source": [ + "population / 1000." + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "TQzIVnbnmWGM" + }, + "cell_type": "markdown", + "source": [ + "[NumPy](http://www.numpy.org/) is a popular toolkit for scientific computing. *pandas* `Series` can be used as arguments to most NumPy functions:" + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "ko6pLK6JmkYP", + "colab": {} + }, + "cell_type": "code", + "source": [ + "import numpy as np\n", + "\n", + "np.log(population)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "xmxFuQmurr6d" + }, + "cell_type": "markdown", + "source": [ + "For more complex single-column transformations, you can use `Series.apply`. Like the Python [map function](https://docs.python.org/2/library/functions.html#map), \n", + "`Series.apply` accepts as an argument a [lambda function](https://docs.python.org/2/tutorial/controlflow.html#lambda-expressions), which is applied to each value.\n", + "\n", + "The example below creates a new `Series` that indicates whether `population` is over one million:" + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "Fc1DvPAbstjI", + "colab": {} + }, + "cell_type": "code", + "source": [ + "population.apply(lambda val: val > 1000000)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "ZeYYLoV9b9fB" + }, + "cell_type": "markdown", + "source": [ + "\n", + "Modifying `DataFrames` is also straightforward. For example, the following code adds two `Series` to an existing `DataFrame`:" + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "0gCEX99Hb8LR", + "colab": {} + }, + "cell_type": "code", + "source": [ + "cities['Area square miles'] = pd.Series([46.87, 176.53, 97.92])\n", + "cities['Population density'] = cities['Population'] / cities['Area square miles']\n", + "cities" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "6qh63m-ayb-c" + }, + "cell_type": "markdown", + "source": [ + "## Exercise #1\n", + "\n", + "Modify the `cities` table by adding a new boolean column that is True if and only if *both* of the following are True:\n", + "\n", + " * The city is named after a saint.\n", + " * The city has an area greater than 50 square miles.\n", + "\n", + "**Note:** Boolean `Series` are combined using the bitwise, rather than the traditional boolean, operators. For example, when performing *logical and*, use `&` instead of `and`.\n", + "\n", + "**Hint:** \"San\" in Spanish means \"saint.\"" + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "zCOn8ftSyddH", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# Your code here\n", + "cities['required'] = (cities['Area square miles'] > 50) & cities['City name'].apply(lambda name: name.startswith('San'))\n", + "cities" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "YHIWvc9Ms-Ll" + }, + "cell_type": "markdown", + "source": [ + "### Solution\n", + "\n", + "Click below for a solution." + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "T5OlrqtdtCIb", + "colab": {} + }, + "cell_type": "code", + "source": [ + "cities['Is wide and has saint name'] = (cities['Area square miles'] > 50) & cities['City name'].apply(lambda name: name.startswith('San'))\n", + "cities" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "f-xAOJeMiXFB" + }, + "cell_type": "markdown", + "source": [ + "## Indexes\n", + "Both `Series` and `DataFrame` objects also define an `index` property that assigns an identifier value to each `Series` item or `DataFrame` row. \n", + "\n", + "By default, at construction, *pandas* assigns index values that reflect the ordering of the source data. Once created, the index values are stable; that is, they do not change when data is reordered." + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "2684gsWNinq9", + "colab": {} + }, + "cell_type": "code", + "source": [ + "city_names.index" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "code", + "id": "F_qPe2TBjfWd", + "colab": {} + }, + "cell_type": "code", + "source": [ + "cities.index" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "hp2oWY9Slo_h" + }, + "cell_type": "markdown", + "source": [ + "Call `DataFrame.reindex` to manually reorder the rows. For example, the following has the same effect as sorting by city name:" + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "sN0zUzSAj-U1", + "colab": {} + }, + "cell_type": "code", + "source": [ + "cities.reindex([2, 0, 1])" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "-GQFz8NZuS06" + }, + "cell_type": "markdown", + "source": [ + "Reindexing is a great way to shuffle (randomize) a `DataFrame`. In the example below, we take the index, which is array-like, and pass it to NumPy's `random.permutation` function, which shuffles its values in place. Calling `reindex` with this shuffled array causes the `DataFrame` rows to be shuffled in the same way.\n", + "Try running the following cell multiple times!" + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "mF8GC0k8uYhz", + "colab": {} + }, + "cell_type": "code", + "source": [ + "cities.reindex(np.random.permutation(cities.index))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "fSso35fQmGKb" + }, + "cell_type": "markdown", + "source": [ + "For more information, see the [Index documentation](http://pandas.pydata.org/pandas-docs/stable/indexing.html#index-objects)." + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "8UngIdVhz8C0" + }, + "cell_type": "markdown", + "source": [ + "## Exercise #2\n", + "\n", + "The `reindex` method allows index values that are not in the original `DataFrame`'s index values. Try it and see what happens if you use such values! Why do you think this is allowed?" + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "PN55GrDX0jzO", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# Your code here\n", + "cities.reindex([7, 0, 3, 6, 5, 4, 2, 1])" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "TJffr5_Jwqvd" + }, + "cell_type": "markdown", + "source": [ + "### Solution\n", + "\n", + "Click below for the solution." + ] + }, + { + "metadata": { + "colab_type": "text", + "id": "8oSvi2QWwuDH" + }, + "cell_type": "markdown", + "source": [ + "If your `reindex` input array includes values not in the original `DataFrame` index values, `reindex` will add new rows for these \"missing\" indices and populate all corresponding columns with `NaN` values:" + ] + }, + { + "metadata": { + "colab_type": "code", + "id": "yBdkucKCwy4x", + "colab": {} + }, + "cell_type": "code", + "source": [ + "cities.reindex([0, 4, 5, 2])" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "colab_type": "text", + "id": "2l82PhPbwz7g" + }, + "cell_type": "markdown", + "source": [ + "This behavior is desirable because indexes are often strings pulled from the actual data (see the [*pandas* reindex\n", + "documentation](http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.reindex.html) for an example\n", + "in which the index values are browser names).\n", + "\n", + "In this case, allowing \"missing\" indices makes it easy to reindex using an external list, as you don't have to worry about\n", + "sanitizing the input." + ] + } + ] +} \ No newline at end of file diff --git a/logistic_regression.ipynb b/logistic_regression.ipynb new file mode 100644 index 0000000..992c849 --- /dev/null +++ b/logistic_regression.ipynb @@ -0,0 +1,1554 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "logistic_regression.ipynb", + "version": "0.3.2", + "provenance": [], + "collapsed_sections": [ + "JndnmDMp66FL", + "dPpJUV862FYI", + "i2e3TlyL57Qs", + "wCugvl0JdWYL" + ], + "include_colab_link": true + }, + "kernelspec": { + "name": "python2", + "display_name": "Python 2" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "metadata": { + "id": "JndnmDMp66FL", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "#### Copyright 2017 Google LLC." + ] + }, + { + "metadata": { + "id": "hMqWDc_m6rUC", + "colab_type": "code", + "cellView": "both", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "g4T-_IsVbweU", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Logistic Regression" + ] + }, + { + "metadata": { + "id": "LEAHZv4rIYHX", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "**Learning Objectives:**\n", + " * Reframe the median house value predictor (from the preceding exercises) as a binary classification model\n", + " * Compare the effectiveness of logisitic regression vs linear regression for a binary classification problem" + ] + }, + { + "metadata": { + "id": "CnkCZqdIIYHY", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "As in the prior exercises, we're working with the [California housing data set](https://developers.google.com/machine-learning/crash-course/california-housing-data-description), but this time we will turn it into a binary classification problem by predicting whether a city block is a high-cost city block. We'll also revert to the default features, for now." + ] + }, + { + "metadata": { + "id": "9pltCyy2K3dd", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Frame the Problem as Binary Classification\n", + "\n", + "The target of our dataset is `median_house_value` which is a numeric (continuous-valued) feature. We can create a boolean label by applying a threshold to this continuous value.\n", + "\n", + "Given features describing a city block, we wish to predict if it is a high-cost city block. To prepare the targets for train and eval data, we define a classification threshold of the 75%-ile for median house value (a value of approximately 265000). All house values above the threshold are labeled `1`, and all others are labeled `0`." + ] + }, + { + "metadata": { + "id": "67IJwZX1Vvjt", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Setup\n", + "\n", + "Run the cells below to load the data and prepare the input features and targets." + ] + }, + { + "metadata": { + "id": "fOlbcJ4EIYHd", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "from __future__ import print_function\n", + "\n", + "import math\n", + "\n", + "from IPython import display\n", + "from matplotlib import cm\n", + "from matplotlib import gridspec\n", + "from matplotlib import pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "from sklearn import metrics\n", + "import tensorflow as tf\n", + "from tensorflow.python.data import Dataset\n", + "\n", + "tf.logging.set_verbosity(tf.logging.ERROR)\n", + "pd.options.display.max_rows = 10\n", + "pd.options.display.float_format = '{:.1f}'.format\n", + "\n", + "california_housing_dataframe = pd.read_csv(\"https://download.mlcc.google.com/mledu-datasets/california_housing_train.csv\", sep=\",\")\n", + "\n", + "california_housing_dataframe = california_housing_dataframe.reindex(\n", + " np.random.permutation(california_housing_dataframe.index))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "lTB73MNeIYHf", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Note how the code below is slightly different from the previous exercises. Instead of using `median_house_value` as target, we create a new binary target, `median_house_value_is_high`." + ] + }, + { + "metadata": { + "id": "kPSqspaqIYHg", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def preprocess_features(california_housing_dataframe):\n", + " \"\"\"Prepares input features from California housing data set.\n", + "\n", + " Args:\n", + " california_housing_dataframe: A Pandas DataFrame expected to contain data\n", + " from the California housing data set.\n", + " Returns:\n", + " A DataFrame that contains the features to be used for the model, including\n", + " synthetic features.\n", + " \"\"\"\n", + " selected_features = california_housing_dataframe[\n", + " [\"latitude\",\n", + " \"longitude\",\n", + " \"housing_median_age\",\n", + " \"total_rooms\",\n", + " \"total_bedrooms\",\n", + " \"population\",\n", + " \"households\",\n", + " \"median_income\"]]\n", + " processed_features = selected_features.copy()\n", + " # Create a synthetic feature.\n", + " processed_features[\"rooms_per_person\"] = (\n", + " california_housing_dataframe[\"total_rooms\"] /\n", + " california_housing_dataframe[\"population\"])\n", + " return processed_features\n", + "\n", + "def preprocess_targets(california_housing_dataframe):\n", + " \"\"\"Prepares target features (i.e., labels) from California housing data set.\n", + "\n", + " Args:\n", + " california_housing_dataframe: A Pandas DataFrame expected to contain data\n", + " from the California housing data set.\n", + " Returns:\n", + " A DataFrame that contains the target feature.\n", + " \"\"\"\n", + " output_targets = pd.DataFrame()\n", + " # Create a boolean categorical feature representing whether the\n", + " # median_house_value is above a set threshold.\n", + " output_targets[\"median_house_value_is_high\"] = (\n", + " california_housing_dataframe[\"median_house_value\"] > 265000).astype(float)\n", + " return output_targets" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "FwOYWmXqWA6D", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1153 + }, + "outputId": "ff9f1c8d-fd9d-40f5-ba09-53cdaa9251fd" + }, + "cell_type": "code", + "source": [ + "# Choose the first 12000 (out of 17000) examples for training.\n", + "training_examples = preprocess_features(california_housing_dataframe.head(12000))\n", + "training_targets = preprocess_targets(california_housing_dataframe.head(12000))\n", + "\n", + "# Choose the last 5000 (out of 17000) examples for validation.\n", + "validation_examples = preprocess_features(california_housing_dataframe.tail(5000))\n", + "validation_targets = preprocess_targets(california_housing_dataframe.tail(5000))\n", + "\n", + "# Double-check that we've done the right thing.\n", + "print(\"Training examples summary:\")\n", + "display.display(training_examples.describe())\n", + "print(\"Validation examples summary:\")\n", + "display.display(validation_examples.describe())\n", + "\n", + "print(\"Training targets summary:\")\n", + "display.display(training_targets.describe())\n", + "print(\"Validation targets summary:\")\n", + "display.display(validation_targets.describe())" + ], + "execution_count": 4, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Training examples summary:\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + " latitude longitude housing_median_age total_rooms total_bedrooms \\\n", + "count 12000.0 12000.0 12000.0 12000.0 12000.0 \n", + "mean 35.6 -119.6 28.5 2643.5 538.9 \n", + "std 2.1 2.0 12.6 2199.4 422.7 \n", + "min 32.5 -124.3 2.0 2.0 1.0 \n", + "25% 33.9 -121.8 18.0 1461.0 297.0 \n", + "50% 34.2 -118.5 28.0 2139.5 434.0 \n", + "75% 37.7 -118.0 37.0 3148.0 648.0 \n", + "max 42.0 -114.6 52.0 37937.0 6445.0 \n", + "\n", + " population households median_income rooms_per_person \n", + "count 12000.0 12000.0 12000.0 12000.0 \n", + "mean 1432.6 500.7 3.9 2.0 \n", + "std 1139.0 385.0 1.9 1.2 \n", + "min 3.0 1.0 0.5 0.0 \n", + "25% 790.0 282.0 2.6 1.5 \n", + "50% 1168.0 409.0 3.5 1.9 \n", + "75% 1720.0 605.0 4.8 2.3 \n", + "max 28566.0 6082.0 15.0 52.0 " + ], + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
latitudelongitudehousing_median_agetotal_roomstotal_bedroomspopulationhouseholdsmedian_incomerooms_per_person
count12000.012000.012000.012000.012000.012000.012000.012000.012000.0
mean35.6-119.628.52643.5538.91432.6500.73.92.0
std2.12.012.62199.4422.71139.0385.01.91.2
min32.5-124.32.02.01.03.01.00.50.0
25%33.9-121.818.01461.0297.0790.0282.02.61.5
50%34.2-118.528.02139.5434.01168.0409.03.51.9
75%37.7-118.037.03148.0648.01720.0605.04.82.3
max42.0-114.652.037937.06445.028566.06082.015.052.0
\n", + "
" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "Validation examples summary:\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + " latitude longitude housing_median_age total_rooms total_bedrooms \\\n", + "count 5000.0 5000.0 5000.0 5000.0 5000.0 \n", + "mean 35.6 -119.6 28.8 2643.9 540.6 \n", + "std 2.1 2.0 12.5 2132.9 418.7 \n", + "min 32.5 -124.3 1.0 11.0 3.0 \n", + "25% 33.9 -121.8 18.0 1464.5 295.0 \n", + "50% 34.3 -118.5 29.0 2111.0 434.0 \n", + "75% 37.7 -118.0 37.0 3165.2 649.0 \n", + "max 42.0 -114.3 52.0 25187.0 4819.0 \n", + "\n", + " population households median_income rooms_per_person \n", + "count 5000.0 5000.0 5000.0 5000.0 \n", + "mean 1422.2 502.4 3.9 2.0 \n", + "std 1168.9 383.4 1.9 1.2 \n", + "min 8.0 4.0 0.5 0.1 \n", + "25% 789.0 281.0 2.6 1.5 \n", + "50% 1164.0 408.0 3.5 2.0 \n", + "75% 1723.2 607.0 4.7 2.3 \n", + "max 35682.0 4769.0 15.0 55.2 " + ], + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
latitudelongitudehousing_median_agetotal_roomstotal_bedroomspopulationhouseholdsmedian_incomerooms_per_person
count5000.05000.05000.05000.05000.05000.05000.05000.05000.0
mean35.6-119.628.82643.9540.61422.2502.43.92.0
std2.12.012.52132.9418.71168.9383.41.91.2
min32.5-124.31.011.03.08.04.00.50.1
25%33.9-121.818.01464.5295.0789.0281.02.61.5
50%34.3-118.529.02111.0434.01164.0408.03.52.0
75%37.7-118.037.03165.2649.01723.2607.04.72.3
max42.0-114.352.025187.04819.035682.04769.015.055.2
\n", + "
" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "Training targets summary:\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + " median_house_value_is_high\n", + "count 12000.0\n", + "mean 0.2\n", + "std 0.4\n", + "min 0.0\n", + "25% 0.0\n", + "50% 0.0\n", + "75% 0.0\n", + "max 1.0" + ], + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
median_house_value_is_high
count12000.0
mean0.2
std0.4
min0.0
25%0.0
50%0.0
75%0.0
max1.0
\n", + "
" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "Validation targets summary:\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + " median_house_value_is_high\n", + "count 5000.0\n", + "mean 0.3\n", + "std 0.4\n", + "min 0.0\n", + "25% 0.0\n", + "50% 0.0\n", + "75% 1.0\n", + "max 1.0" + ], + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
median_house_value_is_high
count5000.0
mean0.3
std0.4
min0.0
25%0.0
50%0.0
75%1.0
max1.0
\n", + "
" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "metadata": { + "id": "uon1LB3A31VN", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## How Would Linear Regression Fare?\n", + "To see why logistic regression is effective, let us first train a naive model that uses linear regression. This model will use labels with values in the set `{0, 1}` and will try to predict a continuous value that is as close as possible to `0` or `1`. Furthermore, we wish to interpret the output as a probability, so it would be ideal if the output will be within the range `(0, 1)`. We would then apply a threshold of `0.5` to determine the label.\n", + "\n", + "Run the cells below to train the linear regression model using [LinearRegressor](https://www.tensorflow.org/api_docs/python/tf/estimator/LinearRegressor)." + ] + }, + { + "metadata": { + "id": "smmUYRDtWOV_", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def construct_feature_columns(input_features):\n", + " \"\"\"Construct the TensorFlow Feature Columns.\n", + "\n", + " Args:\n", + " input_features: The names of the numerical input features to use.\n", + " Returns:\n", + " A set of feature columns\n", + " \"\"\"\n", + " return set([tf.feature_column.numeric_column(my_feature)\n", + " for my_feature in input_features])" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "B5OwSrr1yIKD", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def my_input_fn(features, targets, batch_size=1, shuffle=True, num_epochs=None):\n", + " \"\"\"Trains a linear regression model.\n", + " \n", + " Args:\n", + " features: pandas DataFrame of features\n", + " targets: pandas DataFrame of targets\n", + " batch_size: Size of batches to be passed to the model\n", + " shuffle: True or False. Whether to shuffle the data.\n", + " num_epochs: Number of epochs for which data should be repeated. None = repeat indefinitely\n", + " Returns:\n", + " Tuple of (features, labels) for next data batch\n", + " \"\"\"\n", + " \n", + " # Convert pandas data into a dict of np arrays.\n", + " features = {key:np.array(value) for key,value in dict(features).items()} \n", + " \n", + " # Construct a dataset, and configure batching/repeating.\n", + " ds = Dataset.from_tensor_slices((features,targets)) # warning: 2GB limit\n", + " ds = ds.batch(batch_size).repeat(num_epochs)\n", + " \n", + " # Shuffle the data, if specified.\n", + " if shuffle:\n", + " ds = ds.shuffle(10000)\n", + " \n", + " # Return the next batch of data.\n", + " features, labels = ds.make_one_shot_iterator().get_next()\n", + " return features, labels" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "SE2-hq8PIYHz", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def train_linear_regressor_model(\n", + " learning_rate,\n", + " steps,\n", + " batch_size,\n", + " training_examples,\n", + " training_targets,\n", + " validation_examples,\n", + " validation_targets):\n", + " \"\"\"Trains a linear regression model.\n", + " \n", + " In addition to training, this function also prints training progress information,\n", + " as well as a plot of the training and validation loss over time.\n", + " \n", + " Args:\n", + " learning_rate: A `float`, the learning rate.\n", + " steps: A non-zero `int`, the total number of training steps. A training step\n", + " consists of a forward and backward pass using a single batch.\n", + " batch_size: A non-zero `int`, the batch size.\n", + " training_examples: A `DataFrame` containing one or more columns from\n", + " `california_housing_dataframe` to use as input features for training.\n", + " training_targets: A `DataFrame` containing exactly one column from\n", + " `california_housing_dataframe` to use as target for training.\n", + " validation_examples: A `DataFrame` containing one or more columns from\n", + " `california_housing_dataframe` to use as input features for validation.\n", + " validation_targets: A `DataFrame` containing exactly one column from\n", + " `california_housing_dataframe` to use as target for validation.\n", + " \n", + " Returns:\n", + " A `LinearRegressor` object trained on the training data.\n", + " \"\"\"\n", + "\n", + " periods = 10\n", + " steps_per_period = steps / periods\n", + "\n", + " # Create a linear regressor object.\n", + " my_optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n", + " my_optimizer = tf.contrib.estimator.clip_gradients_by_norm(my_optimizer, 5.0)\n", + " linear_regressor = tf.estimator.LinearRegressor(\n", + " feature_columns=construct_feature_columns(training_examples),\n", + " optimizer=my_optimizer\n", + " )\n", + " \n", + " # Create input functions.\n", + " training_input_fn = lambda: my_input_fn(training_examples, \n", + " training_targets[\"median_house_value_is_high\"], \n", + " batch_size=batch_size)\n", + " predict_training_input_fn = lambda: my_input_fn(training_examples, \n", + " training_targets[\"median_house_value_is_high\"], \n", + " num_epochs=1, \n", + " shuffle=False)\n", + " predict_validation_input_fn = lambda: my_input_fn(validation_examples, \n", + " validation_targets[\"median_house_value_is_high\"], \n", + " num_epochs=1, \n", + " shuffle=False)\n", + "\n", + " # Train the model, but do so inside a loop so that we can periodically assess\n", + " # loss metrics.\n", + " print(\"Training model...\")\n", + " print(\"RMSE (on training data):\")\n", + " training_rmse = []\n", + " validation_rmse = []\n", + " for period in range (0, periods):\n", + " # Train the model, starting from the prior state.\n", + " linear_regressor.train(\n", + " input_fn=training_input_fn,\n", + " steps=steps_per_period\n", + " )\n", + " \n", + " # Take a break and compute predictions.\n", + " training_predictions = linear_regressor.predict(input_fn=predict_training_input_fn)\n", + " training_predictions = np.array([item['predictions'][0] for item in training_predictions])\n", + " \n", + " validation_predictions = linear_regressor.predict(input_fn=predict_validation_input_fn)\n", + " validation_predictions = np.array([item['predictions'][0] for item in validation_predictions])\n", + " \n", + " # Compute training and validation loss.\n", + " training_root_mean_squared_error = math.sqrt(\n", + " metrics.mean_squared_error(training_predictions, training_targets))\n", + " validation_root_mean_squared_error = math.sqrt(\n", + " metrics.mean_squared_error(validation_predictions, validation_targets))\n", + " # Occasionally print the current loss.\n", + " print(\" period %02d : %0.2f\" % (period, training_root_mean_squared_error))\n", + " # Add the loss metrics from this period to our list.\n", + " training_rmse.append(training_root_mean_squared_error)\n", + " validation_rmse.append(validation_root_mean_squared_error)\n", + " print(\"Model training finished.\")\n", + " \n", + " # Output a graph of loss metrics over periods.\n", + " plt.ylabel(\"RMSE\")\n", + " plt.xlabel(\"Periods\")\n", + " plt.title(\"Root Mean Squared Error vs. Periods\")\n", + " plt.tight_layout()\n", + " plt.plot(training_rmse, label=\"training\")\n", + " plt.plot(validation_rmse, label=\"validation\")\n", + " plt.legend()\n", + "\n", + " return linear_regressor" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "TDBD8xeeIYH2", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 619 + }, + "outputId": "1e5dfd28-c426-4ba6-e221-5f302d87c54e" + }, + "cell_type": "code", + "source": [ + "linear_regressor = train_linear_regressor_model(\n", + " learning_rate=0.000001,\n", + " steps=200,\n", + " batch_size=20,\n", + " training_examples=training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 9, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Training model...\n", + "RMSE (on training data):\n", + " period 00 : 0.45\n", + " period 01 : 0.45\n", + " period 02 : 0.45\n", + " period 03 : 0.44\n", + " period 04 : 0.45\n", + " period 05 : 0.45\n", + " period 06 : 0.45\n", + " period 07 : 0.44\n", + " period 08 : 0.44\n", + " period 09 : 0.44\n", + "Model training finished.\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjgAAAGACAYAAACgBBhzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3Wd4VGXawPH/lPTeEwhJINQESCCE\njhBqKHZEigHL6iqiCO7aVtYK6vqCigiWVVRsgEaUJkVEOgRCQi8JkISQ3nuZOe8HNqNISJ1J4/5d\nFx9m5pzn3JMTJvc85X5UiqIoCCGEEEK0IermDkAIIYQQwtgkwRFCCCFEmyMJjhBCCCHaHElwhBBC\nCNHmSIIjhBBCiDZHEhwhhBBCtDna5g5AiNasW7du+Pj4oNFoANDpdISGhvLiiy9ibW3d4HbXrFnD\nlClTrns+MjKS559/ng8//JCwsDDD86WlpQwePJixY8fy5ptvNvi6dZWYmMiiRYu4ePEiAFZWVsyZ\nM4fRo0eb/Nr1sXz5chITE6/7mRw8eJCHHnoIb2/v68755Zdfmiq8Rrl8+TKjRo2iY8eOACiKgqur\nK//6178ICAioV1uLFy+mXbt2TJs2rc7n/PTTT3z//fesWrWqXtcSoqlIgiNEI61atQpPT08AysvL\nmTdvHh999BHz5s1rUHsZGRn897//rTbBAfDy8mLDhg3XJDi//fYb9vb2DbpeQ/zjH//g9ttv58MP\nPwQgNjaWWbNmsXnzZry8vJosjsbw8vJqNcnMjWg0mmvew6ZNm3j88cfZsmUL5ubmdW7n6aefNkV4\nQjQrGaISwojMzc0ZNmwYp0+fBqCsrIx///vfjBs3jvHjx/Pmm2+i0+kAOHPmDFOnTiU8PJzbb7+d\n3bt3AzB16lSuXLlCeHg45eXl112jb9++HDx4kJKSEsNzmzZtYsiQIYbH5eXlvP7664wbN46RI0ca\nEhGAo0ePctdddxEeHs6ECRPYt28fcLVHYOjQoXz55ZfceuutDBs2jE2bNlX7Ps+dO0dQUJDhcVBQ\nEFu2bDEkesuWLWP48OHccccdfPzxx4wcORKA5557juXLlxvO+/Pj2uJatGgR9913HwBHjhzh7rvv\nZsyYMUyZMoWkpCTgak/WU089RVhYGPfddx+pqam13LHqRUZGMmfOHGbNmsV//vMfDh48yNSpU5k7\nd64hGdi8eTOTJk0iPDycmTNnkpiYCMD777/Piy++yOTJk/n888+vaXfu3Ll89tlnhsenT59m6NCh\n6PV63nnnHcaNG8e4ceOYOXMmaWlp9Y57woQJlJaWcuHCBQBWr15NeHg4I0eOZP78+ZSWlgJXf+5v\nvPEGt956K5s3b77mPtzo91Kv1/Pqq68yYsQIJk+ezJkzZwzXPXToEHfeeScTJkxg/PjxbN68ud6x\nC2F0ihCiwbp27aqkpKQYHufm5iozZsxQli9friiKonz00UfKww8/rFRUVCglJSXK3Xffraxbt07R\n6XTK+PHjlfXr1yuKoijHjh1TQkNDlYKCAuXAgQPK6NGjq73eDz/8oDz77LPKP/7xD8O5BQUFyqhR\no5S1a9cqzz77rKIoirJs2TJl1qxZSllZmVJUVKTccccdyo4dOxRFUZRJkyYpGzZsUBRFUX788UfD\ntZKSkpSAgABl1apViqIoyqZNm5QxY8ZUG8cTTzyhhIWFKV988YUSFxd3zWtnz55V+vXrp6SnpysV\nFRXKY489poSFhSmKoijPPvus8sEHHxiO/fPjmuIKDAxUIiMjDe83NDRU2bNnj6IoirJ+/Xrlzjvv\nVBRFUb766itlxowZSkVFhZKdna2EhYUZfiZ/VtPPuOrnHBwcrFy8eNFwfK9evZR9+/YpiqIoycnJ\nSkhIiHLp0iVFURTl008/VWbNmqUoiqIsXbpUGTp0qJKVlXVduxs3blRmzJhhePzee+8pr732mnLu\n3Dll7NixSnl5uaIoivLll18qP/744w3jq/q59OjR47rnQ0NDlfj4eCUqKkoZNGiQkpqaqiiKoixY\nsEB58803FUW5+nO/9dZbldLSUsPjDz74oMbfy507dypjx45VCgsLlZKSEmXy5MnKfffdpyiKotx1\n113KwYMHFUVRlIsXLyrz58+vMXYhmoL04AjRSBEREYSHhzNq1ChGjRrFwIEDefjhhwHYuXMnU6ZM\nQavVYmlpya233srevXu5fPkymZmZTJw4EYBevXrRrl07jh8/XqdrTpw4kQ0bNgCwfft2wsLCUKv/\n+O/822+/MX36dMzNzbG2tub2229n69atAKxbt47x48cDEBISYuj9AKisrOSuu+4CIDAwkCtXrlR7\n/bfffpsZM2awfv16Jk2axMiRI/n222+Bq70roaGhuLm5odVqmTRpUp3eU01xVVRUMGbMGEP7Hh4e\nhh6rSZMmkZiYyJUrVzh8+DBjxoxBq9Xi5OR0zTDeX6WkpBAeHn7Nvz/P1fHz88PPz8/w2NLSkkGD\nBgGwd+9eBgwYgK+vLwD33HMPBw8epLKyErjao+Xs7HzdNUeMGMGpU6fIzc0FYNu2bYSHh2Nvb092\ndjbr168nLy+PiIgI7rjjjjr93KooisLq1avx8PDAz8+PHTt2MGHCBDw8PACYNm2a4XcAYNCgQVhY\nWFzTRk2/l1FRUQwfPhwbGxssLS0N9wrAxcWFdevWER8fj5+fH4sXL65X7EKYgszBEaKRqubgZGdn\nG4ZXtNqr/7Wys7NxcHAwHOvg4EBWVhbZ2dnY2dmhUqkMr1X9kXN1da31mkOGDOHFF18kNzeXjRs3\nMnv2bMOEX4CCggLeeOMNlixZAlwdsurduzcA69ev58svv6SoqAi9Xo/yp+3oNBqNYXK0Wq1Gr9dX\ne30LCwseeughHnroIfLz8/nll19YtGgR3t7e5OXlXTMfyMXFpdb3U5e4bG1tAcjPzycpKYnw8HDD\n6+bm5mRnZ5OXl4ednZ3heXt7e4qKiqq9Xm1zcP583/76OCcn55r3aGdnh6Io5OTkVHtuFWtrawYP\nHszOnTsJCQkhPz+fkJAQVCoV77//Pp999hmvvfYaoaGhvPLKK7XOZ9LpdIafg6IodO7cmeXLl6NW\nqykoKGDbtm3s2bPH8HpFRcUN3x9Q4+9lXl4e7u7u1zxfZdGiRaxYsYIHHngAS0tL5s+ff839EaI5\nSIIjhJE4OzsTERHB22+/zYoVKwBwdXU1fFsHyM3NxdXVFRcXF/Ly8lAUxfDHJDc3t87JgJmZGWFh\nYaxbt46EhAT69OlzTYLj7u7Ogw8+eF0PRlpaGi+++CJr166lR48eXLp0iXHjxtXrfWZnZ3P69GlD\nD4q9vT1Tpkxh9+7dnDt3Djs7OwoKCq45vspfk6a8vLx6x+Xu7k6nTp2IjIy87jV7e/sbXtuYXFxc\nOHr0qOFxXl4earUaJyenWs8dN24c27ZtIycnh3Hjxhnu/8CBAxk4cCDFxcW89dZb/N///V+tPSF/\nnWT8Z+7u7tx55508++yz9XpfN/q9rOln6+rqyoIFC1iwYAF79uzhiSeeYNiwYdjY2NT52kIYmwxR\nCWFEDzzwAEePHuXQoUPA1SGJ77//Hp1OR3FxMT/99BPDhw/H29sbT09PwyTe6OhoMjMz6d27N1qt\nluLiYsNwx41MnDiRTz75pNql2aNGjWLt2rXodDoURWH58uXs2rWL7OxsrK2t6dSpE5WVlaxevRrg\nhr0c1SktLeXJJ580TD4FSEhIIDY2ln79+tGnTx8OHz5MdnY2lZWVrFu3znCcm5ubYXJqUlIS0dHR\nAPWKKygoiIyMDGJjYw3t/POf/0RRFIKDg9mxYwc6nY7s7Gx27dpV5/dVH0OGDOHw4cOGYbTvvvuO\nIUOGGHruahIWFsbRo0fZvn27YZhnz549vPLKK+j1eqytrenevfs1vSgNMXLkSLZu3WpIRLZv387H\nH39c4zk1/V726dOHPXv2UFJSQklJiSGxqqioICIigvT0dODq0KZWq71myFSI5iA9OEIYka2tLY88\n8ghvvfUW33//PRERESQlJTFx4kRUKhXh4eGMHz8elUrFkiVLeOmll1i2bBlWVla89957WFtb061b\nNxwcHBgyZAg//vgj7dq1q/Za/fv3R6VSMWHChOtemz59OpcvX2bixIkoikLPnj2ZNWsW1tbW3HLL\nLYwbNw4XFxeee+45oqOjiYiIYOnSpXV6j+3atWPFihUsXbqU119/HUVRsLW15fnnnzesrLr33nu5\n8847cXJyYuzYsZw/fx6AKVOmMGfOHMaOHUtAQIChl6Z79+51jsvS0pKlS5fy2muvUVRUhJmZGXPn\nzkWlUjFlyhQOHz7M6NGjadeuHaNHj76m1+HPqubg/NV//vOfWn8Gnp6evP7668yePZuKigq8vb15\n7bXX6vTzs7W1JTAwkLNnzxIcHAxAaGgoGzduZNy4cZibm+Ps7MyiRYsAeOaZZwwroeojMDCQRx99\nlIiICPR6PS4uLrzyyis1nlPT72VYWBg7d+4kPDwcV1dXhg8fzuHDhzEzM2Py5Mncf//9wNVeuhdf\nfBErK6t6xSuEsamUPw90CyGEkR0+fJhnnnmGHTt2NHcoQoibiPQhCiGEEKLNkQRHCCGEEG2ODFEJ\nIYQQos2RHhwhhBBCtDmS4AghhBCizWmTy8QzMqpfFmosTk7W5OQUm/Qaov7kvrRccm9aJrkvLZfc\nm7pzc7Or9nnpwWkArVbT3CGIash9abnk3rRMcl9aLrk3jWfSHpxFixYRGxuLSqXihRdeMOyF82eL\nFy8mJiaGVatWcfDgQebOnUuXLl0A6Nq1KwsWLCAlJYXnn3+eyspKtFotb7/9Nm5ubqYMXQghhBCt\nmMkSnEOHDpGQkMDq1auJj4/nhRdeMJRfrxIXF0dUVBRmZmaG5/r3739d5dJ3332XKVOmMGHCBL7+\n+mtWrlzJM888Y6rQhRBCCNHKmWyIav/+/YY9cvz9/cnLy6OwsPCaY958803mzZtXa1svvfSSoaS7\nk5PTNZsXCiGEEEL8lckSnMzMzGt21nV2diYjI8PwODIykv79+9O+fftrzouLi+PRRx9l2rRp7N27\nFwBra2s0Gg06nY5vvvmGW2+91VRhCyGEEKINaLJVVH+uJ5ibm0tkZCQrV64kLS3N8Lyfnx9z5sxh\n/PjxJCUlMXPmTLZu3Yq5uTk6nY5nnnmGgQMHMmjQoBqv5eRkbfIJWjeatS2al9yXlkvuTcsk96Xl\nknvTOCZLcNzd3cnMzDQ8Tk9PN0wMPnDgANnZ2cyYMYPy8nISExNZtGgRL7zwgmFnZB8fH1xdXUlL\nS6NDhw48//zz+Pr6MmfOnFqvbeqldW5udiZfii7qT+5LyyX3pmWS+9Jyyb2puyZfJj5kyBC2bNkC\nwMmTJ3F3d8fW1haA8PBwNm3axJo1a1i2bBmBgYG88MIL/Pzzz3z66acAZGRkkJWVhYeHBz///DNm\nZmY8+eSTpgpXCCGEEG2IyRKcvn37EhgYyNSpU3n99dd56aWXiIyMZNu2bTc8Z+TIkURFRTF9+nRm\nz57Nyy+/jLm5Od988w2nTp0iIiKCiIgIXn75ZVOFLYQQQtwUdu78tU7HvffeYq5cSb7h6889N99Y\nIRlVm9xs09TdetJ12DLJfWm55N60THJfWi5T35uUlCt88MG7vP76f0x2jaZyoyGqNrlVgxBCCCFu\nbMmStzh9+iTDhoUydux4UlKu8O67y3njjVfJyEinpKSEBx98hCFDhjFnziPMn/8Mv/32K0VFhSQm\nJpCcfJknn3yaQYOGMHHiKDZu/JU5cx4hNHQA0dGHyc3N5a233sHV1ZVXX11AamoKvXr1ZseO7fz4\n46YmeY+S4AghhBDNZM2OOKLOpF/3vEajQqdr2ABLaHd3pozsXOMx06ZFEBm5ho4d/UlMvMTy5f8l\nJyeb/v0HMn78JJKTL7NgwXMMGTLsmvPS09P4v/9byoED+/jppx8YNGjINa/b2Njw3nsrWLHifXbt\n2kG7dt6Ul5fx8cefs3fvbtas+bZB76khJMERQpjU2cQcFI0GVXMHIoSoVo8egQDY2dlz+vRJfv45\nEpVKTX5+3nXH9u4dDFxdKf3X4r0AQUF9DK/n5eWRkHCRXr2CABg0aAgaTdPtsSUJjhDCZHIKyvjP\nt0fx9bRnwcwQVCpJc4T4sykjO1fb29KU86Oqtkvatu0X8vPz+eCD/5Kfn8/f/hZx3bF/TlCqm8L7\n19cVRUGtvvqcSqVq0s8A2U1cCGEyMXGZKApcSsknNj6rucMRQvyPWq1Gp9Nd81xubi5eXu1Qq9X8\n/vsOKioqGn2d9u29OXv2FACHDh247pqmJAmOEMJkYuP+KPa5cd+lar/xCSGanq9vR86ePUNR0R/D\nTCNGjGTfvt3MnfsYVlZWuLu7s3LlJ426zuDBwygqKuKxxx4iNvYo9vYOjQ29zmSZeAPI0sqWSe5L\ny1JaXsmT7+3B09ma9u62HDyZyj+n9aGHr1PtJ4smIf9nWq62cm/y8/OIjj7MiBGjyMhIZ+7cx/jm\nmx+Meg1ZJi6EaFInL+ZQqdMT3MWFsFBfDp5MZcO+S5LgCHETsba2YceO7XzzzSoURc8TTzRdUUBJ\ncIQQJlE1PBXc2Y2uPk4E+Dlx6lIO8Vfy8G/XdN3UQojmo9VqefXVN5rl2jIHRwhhdHq9Qmx8Jg42\n5vh5Xe0+njTID4CN+xKaMTIhxM1CEhwhhNFdSMmnoLiCoM4uqP+3LLSbjyP+7e2Jicvkcvr19TOE\nEMKYJMERQhhdzPk/hqeqqFSqP3pxDkgvjhDCtCTBEUIYXUxcJmZaNT38rp1Q3NvfhQ7uthw6nUZa\nTnEzRSeEuBlIgiOEMKr0nGKuZBYR6OeMhdm1ZdlVKhUTB/miKLBZenGEaNEmT76V4uJiVq36nBMn\njl3zWnFxMZMn31rj+Tt3/grApk3r+f3330wW541IgiOEMKqYuKsVi4M6u1T7er9u7ng4W7P3eCrZ\n+aVNGZoQogEiIu6nZ8/e9TonJeUK27dvAWDChFsZPjzMFKHVSJaJCyGMqmp5eFBn12pfV6tVTBjo\nw8pNZ/jlUCLTR3dtyvCEuOk9+OAMFi1ajKenJ6mpKTz//NO4ublTUlJCaWkp8+b9k4CAnobjFy58\nmREjRhEc3Id//esZysvLDZtuAmzdupnvv1+NRqPGz8+fZ5/9F0uWvMXp0ydZufIT9Ho9jo6O3H33\nvSxf/h7Hj8dSWanj7runEB4+kTlzHiE0dADR0YfJzc3lrbfewdPTs9HvUxIcIYTRFJdWcC4pl45e\ndjjaWtzwuEGBnvy05yK7Yq4wabAf9tbmTRilEC1HZNwGjqYfv+55jVqFTt+wjQb6uPfirs6Tbvj6\nLbeEsXfvLu6+ewq7d//OLbeE4e/fhVtuGcGRI1F8/fUXLFz49nXnbdmymU6d/Hnyyaf59dethh6a\nkpISFi9+Hzs7Ox5//GHi4+OYNi2CyMg1PPDAw3z66UcAxMREc+FCPCtWfEZJSQmzZk3llltGAGBj\nY8N7761gxYr32bVrB1OmTG/Qe/8zGaISQhjN8QvZ6PQKwTfovami1agZP8CX8ko926KSmig6IQRU\nJTi7Adiz53eGDh3O77//ymOPPcSKFe+Tl5dX7XmXLl2gZ88gAPr0CTE8b29vz/PPP82cOY+QkHCR\nvLzcas8/c+YUwcF9AbCyssLPrxNJSVf//wcF9QHA3d2dwkLjlJGQHhwhhNHEVFUv7uJWy5EwrLcX\n6/deZEf0ZcYP8MXaUj6OxM3nrs6Tqu1tMeVeVJ06+ZOVlUFaWioFBQXs3r0TV1d3Fix4jTNnTrFs\n2bvVnqcoV4eY4WoxT4CKigqWLPkPn3/+DS4urjzzzFM3vK5KpeLPu19WVlYY2tNo/liQYKwtMqUH\nRwhhFJU6Pcfis3Cxt8DbzabW483NNIzt70NJmY4d0ZebIEIhRJVBg4by8cfLGTZsOHl5ubRv7w3A\n77//RmVlZbXn+Pj4cubMaQCiow8DUFxchEajwcXFlbS0VM6cOU1lZSVqtRqdTnfN+d27B3L06JH/\nnVdMcvJlvL19TPUWJcERQhjH+ct5lJRVEtzZDdX/qhfXJqxPe6wttGyNSqKsQlf7CUIIoxg+PIzt\n27cwYsQowsMnsnr118yb9ziBgT3Jyspi48afrzsnPHwiJ08eZ+7cx0hKSkClUuHg4Eho6AD+9reZ\nrFz5CdOnR7B06RJ8fTty9uwZli5dbDg/KCiYbt268/jjDzNv3uM8+ugcrKysTPYeVYqx+oJaEFNv\nMd9WtrFva+S+NK9vt59n2+Ek5t8bRM+O1y4Rr+ne/LjrAuv3XWLaqC6MCe3QFKGK/5H/My2X3Ju6\nc3Ozq/Z56cERQjSaoijExmViaa6hWwen2k/4k9H9vDE3U/PLoUQqdXoTRSiEuNlIgiOEaLQrWcWk\n55bQs6MzZtr6fazYWZszIrg9OQVl7DuRaqIIhRA3G0lwhBCNFmtYPVXz8vAbGdffB61Gxab9Cej0\n0osjhGg8SXCEEI0Wcz4TlQp6+zcswXGys2BoLy/Sc0uIOpNu5OiEEDcjSXCEEI2SX1ROfHIeXdo7\nYGtl1uB2wgf6olLBxv0J6Nve2gchRBOTBEcI0SjH4rNQqFtxv5q4O1oxIMCD5Iwiw5CXEEI0lCQ4\nQohGiTFsrln97uH1MXGgLwAb9iUYrZqpEOLmJAlOPcmHrhB/qKjUcfJiNh7O1ni51F69uDbt3Wzp\n08WViyn5nE7IMUKEQoiblSQ49XClMJVn97zCz2e2NXcoQrQIpxNyKavQEWyE3psqkwb7AVfn4ggh\nRENJglMPdua2mKnN+Co2knVxm6Q3R9z0DMvDa9k9vD46etkT6OfE6YQc4pOr39VYCCFqIwlOPdiZ\n2/J0yGy87NzZlriTb878gF6Rmh3i5qQoCjFxmdhYauns7WDUtqUXRwjRWJLg1JOzpROvjfwHHeza\nsy/lEJ+e+IoKffU7rwrRliWmFZJTUEZvfxc0auN+lHTt4Ejn9g7ExGWSlF5o1LaFEDcHSXAawN7S\njrl9/k4Xx07EZJxgRexnlFaWNndYQjSpGEP14sYtD6+OSqVi0uCrK6o27r9k9PaFEG2fJDgNZKW1\n5PGghwhyDeRsThzvHf2YwvKi5g5LiCYTcz4TjVpFz47OJmm/VycXfNxtiTqTTlp2sUmuIYRouyTB\naQQzjRkP9byPgV79SCy4zJLoFeSU5jZ3WEKYXHZ+KQlpBXT3ccTKQmuSa6hUKiYO9kNRYNMBmYsj\nhKgfkyY4ixYt4t5772Xq1KkcO3as2mMWL15MREQEAAcPHmTgwIFEREQQERHBa6+9BkBKSgoRERFM\nnz6duXPnUl5ebsqw60Wj1nBf93sY1eEW0orTWXxkOWlFspeOaNti47MACDLi6qnqhHR1w9PZmn0n\nUsnOl2FgIUTdmSzBOXToEAkJCaxevZqFCxeycOHC646Ji4sjKirqmuf69+/PqlWrWLVqFQsWLABg\n6dKlTJ8+nW+++QZfX1++//57U4XdICqVijs7T+T2TuPJKctlSfQKEvMvN3dYQpiMKZaHV0etVjFh\noC86vcIvBxNNei0hRNtisgRn//79jB49GgB/f3/y8vIoLLx2NcSbb77JvHnzam3r4MGDjBo1CoCw\nsDD2799v/IAbSaVSMdYvjGnd7qKoopj3jn7EuZz45g5LCKMrLa/k1KUcvN1scXW0Mvn1BgZ64GJv\nwa7YK+QXtZzeWyFEy2aawXMgMzOTwMBAw2NnZ2cyMjKwtbUFIDIykv79+9O+fftrzouLi+PRRx8l\nLy+POXPmMGTIEEpKSjA3NwfAxcWFjIyMGq/t5GSNVqsx8ju6lpubXbXP3+k2Bi8XF9478BnLYz9l\n7qCH6O8dbNJYxB9udF+E8ew/foVKnZ7BQe3q9fNuzL25Z1RXPvzxOHtPpTFzQkCD2xHXk/8zLZfc\nm8YxWYLzV3+u+pubm0tkZCQrV64kLS3N8Lyfnx9z5sxh/PjxJCUlMXPmTLZu3XrDdm4kJ8e0Ky7c\n3OzIyCi44ev+ll14rPcDfHz8Sxbv/ZgZPe5hkFc/k8Ykar8vwjh2Hbk6/Nq1nX2df96NvTfBnZyx\ntzFnw54LDO/libWlWYPbEn+Q/zMtl9yburtRImiyISp3d3cyMzMNj9PT03Fzu1ov48CBA2RnZzNj\nxgzmzJnDyZMnWbRoER4eHkyYMAGVSoWPjw+urq6kpaVhbW1NaenVCYZpaWm4u7ubKmyj6eHclSeD\nH8Faa8VXp9fwa+Ku5g5JiEbT6xVi4zNxsDHHz6vpvl2am2kYF9qBkjIdv0YnN9l1hRCtl8kSnCFD\nhrBlyxYATp48ibu7u2F4Kjw8nE2bNrFmzRqWLVtGYGAgL7zwAj///DOffvopABkZGWRlZeHh4cHg\nwYMNbW3dupVhw4aZKmyj6ujgw1N9H8XB3J7IuA38FL9Z9q8SrdqFlHwKiisI6uyCWqVq0muP6NMe\nawst26KSKCvXNem1hRCtj8kSnL59+xIYGMjUqVN5/fXXeemll4iMjGTbthvvxD1y5EiioqKYPn06\ns2fP5uWXX8bc3JwnnniCdevWMX36dHJzc7njjjtMFbbRtbP15OmQ2bhbubI14Te+PRsp+1eJVivm\nfNXqKeNXL66NlYWW0f28KSyp4PfYK01+fSFE66JS2mCXginHLfOLy+nk40xmZv32x8kvL+CDmE+5\nXHiFPm69mBU4DTN1k02BuinImLXpLfjvQdJzS1g6dxgWZnWfyG+se1NYUsE/l+/D2lLLm38fhJlW\napU2hvyfabnk3tRdk8/BaYsuXMnnqaV7eOOLKIpL67fBpr25HU/1/TudHTtyNOM4H8aupLSyzESR\nCmF86TnFJGcWEejnXK/kxphsrcwY0acdOQVl7DuR0iwxCCFaB0lw6qG9qw1dOziy/3gKr34RRWJa\n/bJrK60Vjwf9jV6uAZzJOc/SmI8prJD9q0TrEBNXVb3YpVnjGBvqg1ajYvOBRHR6Ge4VQlRPEpx6\nsDDX8M9pwdwd1pn0nBIWrjrC7mP1mwtgrjHj4Z4RDPAMISE/iXeiPyS3LM9EEQthPFXVi029PUNt\nnOwsGNq7Hem5JUSdlm1RhBDhihrBAAAgAElEQVTVkwSnnjRqNfdPCuSJu3thplGzctMZPtt0mvKK\nuq/q0Kg13NfjHsI6DCW1KO3q/lXFNRcvFKI5FZdWcC4pl45edjjaWjR3OIwf4INapWLjgQT0bW8a\noRDCCCTBaaA+Xdz49wOh+HrYsedYCgtXHSGtHgUG1So1d3e+lVs7jSO7NIclR5aTVCD1PUTLdPxC\nNjq9Uu+9pyr1lSw5spy1JzYYNR43RysGBHiQnFFE7PnM2k8QQtx0JMFpBHdHK16I6MuI4HYkpRfy\n6udRHDlb954YlUpFuN8opna7k6KKYt6N/ojzORdMGLEQDRNTtblml/otDz+eeZr4vEv8eHqL0Ydi\nJwzyBWDD/gSpLyWEuI4kOI1kptUwM7w7f5vUA51O4YMfj7N6x3kqdXWf/Dis/SAeCJxGhb6CD2L/\ny/HMUyaMWIj6qdTpOR6fhYu9Bd5uNvU690DK4att6CvZnvi7UeNq72pDSFc3Lqbkcyohx6htCyFa\nP0lwjGRwTy9enNUPD2drthxK4j/fHiWnoO7LwEM8gnm09/2oUPHx8S85mHLEhNEKUXfnL+dRXFZJ\ncGc3VPWoXpxXVsCp7LN427bDxdqJPckHKSivX/2o2kwcfLUXZ+O+S0ZtVwjR+kmCY0Tebrb8e1Y/\nQru7E3c5j5dXHuLUpew6nx/g0o0n+jyCpcaCL0+vZkfSbhNGK0TdVFUvDupSv+Xhh1KPoFf0DG7X\nn9u7j6VCX2H032k/T3t6dnTmTGIuccmyGrE+cgrK+GbLGUrK6lfTS4jWQhIcI7Oy0PLo7YHMGNOV\n4tJKFn8Xw/q9F+u80qOTgy/z+j6Gg7kdP5xfz/oLW2R+gWg2iqIQG5eJpbmGbh2c6nXegdQjaFUa\n+nkEM7LjYOzN7dh1eR/FFXWfjF8XEwdJL0595ReX8/a3R/l261l+j5FtL0TbJAmOCahUKkaFePPc\njL442Vvw4+6LvLf2GIUlFXU6v52tJ/NDHsfVyoVfLv3Kd+d+lP2rRLO4klVMem4JPTs612tbhISC\nJFKL0ujlFoiNmTXmWnNG+dxCqa6MnZf3GjXGbj5OdPF2IDY+q97FN29GJWWVvLMmltTsq4nmsXhZ\nhSbaJklwTMi/vQMv3R9Kz47OHL+QxSsrD3HhSn6dznW1cmZ+39m0t/ViT/IBPj/5LZV66UoWTSvW\nsHqqfsvD9/9vcvEgr36G54a2G4iN1prfkvZQWllqvCCBiYP8ANh0IMGo7bY1FZU63v/hGAmpBQzt\n7UVXH0fOJeVRVFq3L19CtCaS4JiYnbU5T00J4o5hHcnOL+ONr47w65HLdRp2crCw46k+j+Lv4MeR\n9Fg+PPY5ZbryJohaiKtizmeiUkFv/7onOBW6Co6kxeBgbk8P566G5y21FoR1GEZxZQm7kw8YNc5e\nnZzx8bAl6nS6oWdCXEun1/PhTyc5k5hL365uzArvRv9AT/SKwokLdZ8rKERrIQlOE1CrVNw2pCPz\n7w3GykLL19vO8dHPJyktr71HxtrMijnBf6OnS3dOZ5/j/aMfU2TkOQxCVCe/uJz45Dy6tHfA1sqs\nzufFZp6kpLKU/p59Uauu/YgZ7j0YS40lvybuotyIybpKpWLSID8UpBenOnpF4fPNZzh6PpMevk78\n/bYANGo1/QM8gT966oRoSyTBaUKBHZ15+YFQOrd34NDpdF774jDJmbVvtmmuMeeRXrMI9ejLxfxE\n3pX9q0QTOBaXhUL9i/sdqGZ4qoq1mRUjvAdTUFHI3iuHjBGmQd9ubni5WLP/RCpZecYdAmvNFEVh\nzY449h5PpaOXHXPu6oWZ9upu8H5e9jjbW3D8QpZsXCraHElwmpizvSXPTO/D2NAOpGQV89oXUew/\nmVrreRq1hpkBUxjhPYQrRaksObKc9GL51iVMJ8awuWbdl4fnlOZyJvs8He198bBxr/aYsA7DMFeb\nsT3xdyqMOK9MrVIxYaAvOr3CL4cSjdZua7dxfwJbo5LwcrHmqXuCsLLQGl5TqVQEdXalqLSSuMvy\npUm0LZLgNAOtRs3UUV2YfUdP1CoVn6w/xZdbzlJRWfOGnWqVmsldbmNSx7FkleawJHo5lwtkiacw\nvopKHScvZuPhbI2XS92rFx9MjUZBYaBXyA2PsTW3YWj7geSW5XHIyAUtBwR44GJvya7YK+QVyXy1\n344mE7nrAi72Fjx9bzB21ubXHRP0v/lVsXFZTR2eECYlCU4z6tfdnZfuD8XbzZadR5NZ9FU0mbkl\nNZ6jUqkY33E0U7reQWF5Ee8e/ZC43ItNFLG4WZxOyKWsQkdwPXpvFEXhYMphzNRaQjyCajx2lM8t\naNVatiT8hk5fc2JfH1qNmvEDfaio1LMtKslo7bZGh06n8dWWs9hZm/H01D4421tWe1wPX0cszDSG\nHjsh2gpJcJqZh7M1/5oZwpBeniSkFvDK51F1+qAZ7j2Y+wOmUqYrZ1nMJ5zIPN0E0YqbhWF5eD12\nD7+Ql0B6SSZBbj2x0lrVeKyjhQODvULJKs3mcFpMo2L9q2G9vbC3MWdH9OWbdvnziQtZfLL+FBbm\nGuZPCcbT2fqGx5ppNQT4OZGaXUyarEATbYgkOC2AhZmGhyYG8MD47pRX6ln6/TF++D2+1kl//Tz7\n8Gjv+wEVHx3/gkOp0U0Sr2jbFEUhJi4TG0stnb0d6nzegZQoAAZ5hdbp+NE+I1Cr1GxJ+M2ohSzN\ntBrG9e9AabmOHUcuG63d1iLuch7LfjyOWq1i7uTe+Hra1XpOUOeqYSrpxRFthyQ4LciwoHb8KyIE\nd0crNu5PYPF3MeQV1rxhZ6BLd54IfhgLjQVfnPqOnUnGrRIrbj6JaYXkFJTR298FjbpuHxFlunKi\n04/hZOFIVyf/Op3jYuVEf8++pBWnE5NxojEhX2dEcHtsLLVsO3y5TuUY2orL6YW8uzaWykqFx27v\nSTefum2vEeR/dShShqlEWyIJTgvj42HHv+8PpW9XN84k5vLy51GcTcyp8Rx/Rz/m9X0Ue3M71p7/\niQ0Xtsr+VaLBYgzVi+u+PDwm/TilujIGeIVcV/umJmN9w1Ch4pdLvxr1d9bKQsuoEG8KSyrYdZPs\ntZSeW8Li1TEUl1Xy4MTu9ao+7WBrQUcv+6s7x9+kw3qi7ZEEpwWyttTy+J09mRLWmYKiCt7+NobN\nBxNq/APQ3taL+X1n42rpzOZL21lz7ifZv0o0SMz5TDRqFT07Otf5nKraNwM9r699UxMPazdCPIJI\nLkzhRJZx55GN7tcBCzMNvxxKpKKybf9fyC0sY/F3R8krKmfaqC4M7ulV7zaCOrug0yucuChVjUXb\nIAlOC6VSqQgf4MMz0/tgZ2PG2t/iWRZ5vMZvV27WLswPmU07G092Je/ji1Pfyf5Vol6y80tJSCug\nu4/jNfVSapJVks253Hg6O3bEzbruq66qjPMdCcAvl3YYtRfH1sqMsD7tyS0sZ++JFKO129IUlVaw\nZHUMGbml3DrYjzGhHRrUTtWEchmmEm2FJDgtXNcOjrz8QH96+Dpx9HwmL6+MIiH1xjsmO1jYM6/v\no3Ry8ONwWgwfHf/CqCXxRdsWG3+1FkpQPVZPHUi9Wsumvr03VdrZehLk1pNL+YmczYlrUBs3MrZ/\nB7QaNZsPJLTJSr1l5TreW3uMyxlFjOzbnjuGdWxwWx3cbXGys+B4vFQ1Fm2DJDitgIONOU/fG8yk\nwX5k5pWycNURfo9JvuG3XWsza54I/hsBLt04lXWW92M+oVj2rxJ1UN/l4XpFz8GUI5hrzOnj3rvB\n1w039OL82uA2quNoa8Gw3l5k5JZy6HS6UdtubpU6PR+sO05cch4DAjyYPqYrKpWqwe39uapxfHK+\nESMVonlIgtNKqNUq7rqlE0/dE4SFmZovfjnLpxtPU1ZRfZE0c405j/a6n34ewVzIS+Cd6A/JK5MP\nLXFjZeU6Tl3KwdvNFlfHmuvYVInLvUhWaTZ93XpjqbVo8LV97L0JcO7G+dwLRi9cGT7AB7VKxab9\nCejbyOR7vV7hvxtOceJCNr06ufDQxB6oG5HcVKkq7CjDVKItkASnlent78JLD4TS0cuOfSdSef3L\nw6TeoDiXRq1hVsBUbmk/2LB/VWaJlGMX1Tt5KZtKnZ7gLnWfR2OYXFzD1gx1Fe43CoAtl3Y0uq0/\nc3O0YmCgB8mZRcScb/1/uBVF4ett5zh0Op3O3g7MvrMnWo1xPsq7+zhhrlVLPRzRJkiC0wq5Oljx\n3IwQRvZtT3JGEa9+HkXUmeq739UqNVO63s4Ev9Fklmaz+Mhykgvb7oRL0XBVf/yDO9dteXhpZSlH\n04/haumMv2PD535U8Xf0o4tjJ05lnyUx37gF+iYM9EUFbNx/qdWXUPhx90V+O5pMB3dbnprcGwsz\njdHaNjfTEODnTEpWMek5MqwtWjdJcFopM62a+8Z24++3BaIosGLdCb7Zfo5K3fWTA1UqFRM7jWVy\nl9vILy/gnegPic+91PRBixZLr1eIjc/EwcYcP6/aK98CRKcfp1xfUe/aNzWp6sX5JcG4vTjtXG3o\n282NiykFnLpUc12plmzroUQ27LuEu6MV86cEYW1pZvRrVNXPiZHNN0UrJwlOKzcgwIMFs/rRztWG\n7Ycv89bX0WTnl1Z7bFiHocwKmEqZroz3Yz7hZNaZJo5WtFQXUvIpKK4gqLNLnedyHEiJQoWKAQ1c\nPVWdbk6d8bP3ITbjBFcKU43WLsDEQb7A1V6c1mjv8RS+2xGHo605T08NxsG24XOeatL7f1WNZZhK\ntHaS4LQB7VxtWDCzHwMDPYi/ks/LK6M4caH6b1/9Pfvy916zAIUPj33O4dSjTRusaJGqhqfqujw8\nvTiT+LxLdHXyx8WqbtsB1IVKpSLc7+qKqi1G7sXx87SnZydnziTmEnc5z6htm9rRcxms3HQGG0st\n8+8Nxq2Ok8AbwtHWAj9PO84l5VJcKnW0ROslCU4bYWGu4eFJAUSM60ZpeSXvrIll3e4L6PXXzzfo\n6dqDOcEPY6Ex5/NT37HzsuxfdbOLjcvETKsmwK9u1YsPGiYXG6/3pkpPlx60t/XiSFos6cUZRm17\n0iA/ADbsv2TUdk3pTEIOK346iZlWzVP3BOHtZmvyawZ3dv1fVWMZphKtlyQ4bYhKpSKsT3uevy8E\nZ3tLft57iXfWxpJffH2hv86OHXmqz6PYmtuw9pzsX3UzS88tITmziEA/5zpNWNUreg6kHsFSY0mw\nW0+jx3O1F2cUCgpbE3Yate2uHRzp6u3AsfgsEtNuXDCzpbiUms/SH46hKApz7uqFf/u67+7eGH/s\nLi4Jjmi9JMFpgzp62fPSA6EE+btw8mI2r6yMIi75+i55b7t2PN33ccP+VavPrZP9q25CsYbhqbot\nDz+bE0duWR4hHr0x15ibJKZgt554WLtzMPUIWSXGnRQ8cbAfABv3Jxi1XWNLySpiyepYysp1PHJb\nIIH12BussXw8/lfV+EJWtb3AQrQGkuC0UbZWZjwxuTd3D+9EbmEZb30dzbaopOt6aar2r2pv68Xu\n5P18fvJb2b/qJlNV1K2u828OmHB4qopapWacbxh6Rc/2xJ1GbbtnR2d8Pew4fCadlKwio7ZtLFl5\npSxeHUNhSQUzw7sR2t29Sa+vUqkI8nehsKSi2i9HQrQGJk1wFi1axL333svUqVM5duxYtccsXryY\niIiIa54rLS1l9OjRREZGAhAVFcW0adOIiIjg73//O3l58h+uLtQqFRMH+fHPqX2wsTLj21/Ps2Ld\nCUrKrk1gHCzsearPo/g7dORIeiwrYldSWlnWTFGLplRcWsG5pFw6etnhWIdVOcUVJcRmnMDd2pWO\n9r4mja2fRzAuls7sS4kyahVulUrFxEG+KMDmA4lGa9dY8ovLWbw6huz8Mu4e3onhwe2bJY7eVcNU\n8bKaSrROJktwDh06REJCAqtXr2bhwoUsXLjwumPi4uKIioq67vkVK1bg4PDHWPMbb7zBwoULWbVq\nFX369GH16tWmCrtN6u7rxMsPhNK1gyOHz2bw6heHuZxeeM0x1mZWzAn+G71ce3Am5zxLYz6msLxl\nfrsVxnP8QjY6vVLnvaeOpMdSoa9koGe/Ru17VBcatYZxvmFU6iv5NXGXUdvu280NLxdr9p9MJTOv\nxKhtN0ZJ2dUFAqnZxYT392HCQNMmkTUJ8K2qaizzcETrZLIEZ//+/YwePRoAf39/8vLyKCy89o/q\nm2++ybx58655Lj4+nri4OEaMGGF4zsnJidzcXADy8vJwcjLestSbhaOtBf+cFsz4AT6kZRfz+peH\n2Xv82orG5hozHu45kwGeISTkJ7EkegU5pbnNFLFoCg0ZnlKhYoARtmaoi/5eIThaOLA7eb9RE261\nSsWEgb7o9ApbDiYZrd3GqKjU8f4Px0hILWBoby/uCfM3eRJZk6qqxlcyi0jPbTlJoBB1pTVVw5mZ\nmQQGBhoeOzs7k5GRga3t1SWOkZGR9O/fn/btr+1+feutt1iwYAHr1q0zPPfCCy9w3333YW9vj4OD\nA08//XSN13ZyskarNV758uq4udWt2mtLM3tKH0ICPHnn22g+3XiahPQiHrmzF1YWf/wqzHN/kK9i\nndhwdjvvxKzgxeFP0t7esxmjrrvWel+aQ6VOz4mL2bg5WdE30KvWP6aX81O4lJ9IsGcAXby96329\nht6bOwLG8vnRtRzMPsjUXrc3qI3qTBpuw/r9Cew+doVZtwXiZGdptLbrS6fT88YXUZxJzGVQLy/+\ncV8/NEbaX6o2Nd2XIcHtiYnLJD61gMAuTTsPSMjnWWOZLMH5qz9Pbs3NzSUyMpKVK1eSlpZmeH7d\nunUEBwfToUOHa8597bXXWLZsGSEhIbz11lt88803zJw584bXyjHxHipubnZkZLT8JaY30snDlgX3\nh7Ji3Qm2RyVy7HwGD98WgH+7P4YFw9uNQVNpxk/xm3lx+9s8HvQQvvYdami1+bX2+9LUTifkUFRS\nwcAeHmRmFtZ6/Oa4q8NEfV2C6/1zbsy9CbIPws5sM5vO7mSQyyCszYxX5G5cP29WbT3Ht7+c5p4R\nnY3Wbn3oFYWVm05z8GQqPXyduH9cV7Kzm2Z4uLb70snj6hfSvTHJDGriic43O/k8q7sbJYIm+4rg\n7u5OZuYfk9PS09Nxc7u6id+BAwfIzs5mxowZzJkzh5MnT7Jo0SJ27tzJr7/+ypQpU1i7di3Lly9n\n3759nD17lpCQq13igwcP5sSJE6YK+6bh7mjFvyJCGD/Qh4zcEt5YFc3Pey+i019dJq5SqRjrG8b0\n7ndTXFHCu0c/4kz2+WaOWhhTVSn+oDrsHq7T6ziYegQrrRW9XQNrPd6YzDXmjPK5hVJdKbuS9xm1\n7aG9vXCwMee36GSKSiuM2nZdKIrCmh1x7D2eSkcvO+bc1QszE/c+14eTnQW+nnacTcy9bnGCEC2d\nyRKcIUOGsGXLFgBOnjyJu7u7YXgqPDycTZs2sWbNGpYtW0ZgYCAvvPAC7777Lj/88ANr1qzhnnvu\nYfbs2QwePBhXV1fi4uIAOH78OL6+zTfxri3RatTcM6Iz/5zWB0c7c9btvshbXx8l40/j7UPaDeBv\nPe9Dr9exPPYzotOrXw0nWhdFUYg5n4mluYZuHWqf03Y6+xz55QWEegRjpjH+Bo+1GdZ+INZaK3Yk\n7TbqCj8zrYZx/X0oLdfx6xHj7mBeFxv3J7A1KgkvF2ueuifomqHiliLI3wWdXuHkxezmDkWIejFZ\ngtO3b18CAwOZOnUqr7/+Oi+99BKRkZFs27at3m298sorvPjii0RERHDq1KnrlpWLxunu68QrD/an\nfw934pLzeOmzQ+w9nmIYVgx278XjwQ9hptby2Ymv2Z18oJkjFo11JauY9NwSenZ0xkxb+8dAU9S+\nqYml1pIRHYZSVFHMnivG/f0b0acdNpZatkUlUVredL0Uvx1NJnLXBVzsLXj63mDsrE1TNLGx/thd\nXJaLi9ZFpbTB+vymHrdsq2OjiqJw4GQaX207S0mZjtDu7swM74aN5dVv7IkFl/kg5lMKK4qY1HEc\n4X4jm3WVx1+11ftiCpsOJPD9znj+NqkHg3t61XhsYUUR/9rzOm7Wrvyr//wG3XNj3JuiimL+ve8N\nzDXmvDLoOcyN2JP0056L/LTnIveO7My4/j5Ga/dGDp1O46OfTmJrbcbz94Xg6Wxt8mtWpy73RVEU\nnv5gL5U6hXefGIpa3XL+z7dl8nlWd00+B0e0PiqVikE9PXnlgf508XYg6kw6//70EKcTrpbK97Hz\nZn7IbJwtndhwcQvfn/9ZtnZopWLOZ6JSQW//2peHH06LoVLRMdDL9LVvamJjZs0t3oPJLy9gf8r1\n9bMaY1SINxbmGn45lEhFpc6obf/ViQtZfLL+FBbmGuZPCW625KauVCoVvf1dKSyp4MIV4xVcFMLU\nJMER13F1tOLZ6X2585ZO5BeV83/fHmXNb3FUVOrxsHbj6ZDZeNl4sPPyXr48tRqd3rR/EIRx5ReX\nE5+cR5f2Dtha1d4LciDlMGqVmlCPvk0QXc1GdhiGmdqMbQk7jbqliK2VGWF92pNXWM7e46lGa/ev\n4i7nsezH46jVKuZO7o2vZ+tYBlxVCFKGqURrIgmOqJZareLWwX68EBGCu5MVvxxMZOGXh7mSWYSj\nhQPz+j5GR3tfotKO8tHxLyjXXb9juWiZjsVloQDBXdxqPTa5MIWkgmQCXbrhYNH8f4ztzG0Z2m4A\nOWW5HEo9atS2x4V2QKtRs+lAgmE1oTFdTi/k3bWxVFYqPHZ7T7r5tJ6CpT38nDDTqg0r74RoDSTB\nETWq2pn8lqB2JKYX8srnUfx65DLWWiue6PMwAc7dOJl1hvdjPqGowrT1h4RxGJaH12H3cMPkYs/m\nmVxcnVE+t6BVadiasMOovYcOthYMC/IiM6+UQ6fSjdYuQHpuCYtXx1BcVsmDE7sbJu62FhZmGnr4\nOpGcWXTNKkshWjJJcEStLM213D++O3Pu6oWFmYavt53jve+PUVoKf+89i34ewVzIS+Dd6A/JLZON\nUFuyikodJy5m4+FsjZeLTY3H6vQ6DqVGY2tmQ0/XHk0UYe2cLB0Z6NWPjJIso5ctGD/AB41axcYD\nCeiNtP4it7CMxd8dJa+onGmju9Q6qbulqhqmkl4c0VpIgiPqrG9XN159qD89OzpzLD6Lf396kBPx\nucwKmMpw7yFcKUplyZHlpBdnNHeo4gZOJ+RSVqEjuA69NyeyzlBYUUSoRx+06pZVn2WMbxhqlZot\nCTuMOtHd1cGKgQEeXMks4ui5xv8hLyqtYMnqGDJyS7ltiB9j+rXsauA1CZIER7QykuCIenG0teCp\nKUFMG92FkjIdS384xldbznGb70QmdRxLVmkOS46sIKkgublDFdWo+uNUl93Dq4anBjRT7ZuauFo5\nE+rRh5SiNI5lnjJq2xMG+aICNuy/RGOqaJSV63hv7TEuZxQxqq83tw/taLQYm4OTnQU+HrackarG\nopWQBEfUm1qlYky/Dvz7/n54u9myM+YKr35xmB5W/bm3650UVhTxbvSHnMuJb+5QxZ8oikJMXCY2\nllo6ezvUeGxBeSEnsk7jbduODnbtmijC+hnrG4YKFb9c+rVRichfebnYENLNjYTUAk5ealj13kqd\nng/WHScuOY+BAR5MG9OlRdWMaqjgzq5S1Vi0GpLgiAbzdrNlwawQxoZ2IDW7mIVfHqEgyYv7A6ZR\noa/kg9hPic2QfcNaisS0QnIKyujt74JGXfN//ajUaPSKvtkqF9eFp407fdx7kVSQzKnss0Zte+Ig\nPwA27Euo97l6vcJ/N5zixIVsevu78ODEHqjbQHIDMkwlWhdJcESjmGk1TB3VhaenBmNnbcYPv19g\n+696ZnSegVql5pPjq9h3xbhF2UTDVNUwqW15uKIo7E85jEalIdSjT1OE1mDhfqMAjN6L4+tpR69O\nLpxLyuVcUm6dz1MUha+3nePQ6XS6eDvw2B090Wrazsesr6cdDjbmHLuQhV7f5orgizam7fzPE80q\n0M+ZVx8aQEhXN84l5fLl99mMcrgbazMrvj6zlm0JO5s7xJteTFwmGrWKnh2dazwuqSCZK0Wp9HLt\nga15zSutmlt7Wy96ufbgQl4C53MvGLXtSYOvbuq7cX/de3F+3H2B344m08HdlrmTe2Nh1nJ2BjcG\ntUpFUGcXCooruJAiVY1FyyYJjjAaWyszZt/ZkwfGd0evV4j8JQfvvDE4mDuwLn4TkXEbjPotW9Rd\ndn4pCakFdPdxrHXH6gOpzbuxZn2N8/2jF8eYung70rWDI8cvZJGQWvueQFsPJbJhXwLujlbMnxKE\ntWXT77reFGSYSrQWkuAIo1KpVAwLasfLD4bSqZ09MSfKKDs1ACczZ35N3MVXp9fK1g7NIDY+C/jj\nj9ONVOgrOZwag525LQHO3ZoitEbr6OBDd6cunM2J40Je/efM1OSPXpxLNR6393gK3+2Iw9HWnKen\nBuNga2HUOFqSAF9ntBqpaixaPklwhEl4OFnz3Iy+3DbEj9wcNSmHgrHDjQOph/nkxCrKdRXNHeJN\npa7Lw49nnqKospj+nn3RqFvP8ErVXJwtRu7FCfRzxtfTjiNnM0jJKqr2mKPnMli56Qw2llrm3xuM\nm6OVUWNoaSzMNQT4OXE5o4jMPKlqLFouSXCEyWg1au4Y1onnZ4TgYm1P+uEgzEvcOZ55ig9i/0tJ\npXw4NoWych2nLuXg7WaLay1/fFvi1gx10cWpE/4OHTmRdcaoNZhUKhWTBvmhAJuqmYtzJiGHFT+d\nxEyr5ql7gvB2szXatVuyP4apspo5EiFuTBIcYXKdvR145cH+DAnwJu9EMPocT+JyL/JO9IfkldU+\nt0E0zslL2VTq9AR3qbl6cW5ZHqeyzuJr14F2tp5NFJ3xhPuNBGDLpR1GbbdPV1faudqw/2QamX/a\nh+lSaj5LfziGoijMuTlrgoMAACAASURBVKsX/u1rri3UlgT5X/1dkmEq0ZJJgiOahJWFlocmBfDY\n7b3RJIVQmdaB5MIU/u/wB2SWyLdAU4o5XzU8VfPy8KjUoygorWZy8V/1cO6Kj503MRknSClKM1q7\napWKiQN90SsKvxxKBCAlq4glq2MpK9fxyG2BBNayMq0lyisrYMv536nQ178qsbO9JT7utpxJzJGq\nxqLFkgRHNKnQ7u68+uAA/BlCRbI/2WXZvHVoGcmFKc0dWpuk1yvExmfiYGOOn5fdDY+rqn2jVWvp\n5xHUhBEaj0qlItxvFAoKWy79ZtS2+we44+pgya7YFC6m5LN4dQyFJRXMDO9GaHd3o16rKWSV5LAk\nejmfRn/H75f3NqiNoM6uVOoUTjWw2rMQpiYJjmhyzvaW/GNqH+7qOp7KxB4U64p46+AHnMmUrR2M\n7UJKPgXFFQR1dqmxmu6l/ETSitMJcg3E2sy6CSM0rl6uPWhn48mR9Bgyio3XM6hRq5kw0JdKnZ5F\nq46QnV/G3cM7MTy4vdGu0VTSijNYEr3c0HMak96wauMyD0e0dJLgiGahVqkIH+DDv8ZPxjqtH5VK\nBe/HfMKOuCPNHVqbUjU8VdvycMPk4lY6PFVFrVIzzm8kekXPtkTj9uIM6eWJg605Or1CeH8fJgz0\nNWr7TSG5MIV3jqwgtyyPO/wnEOjelYv5CeSU1r1acxU/r/9VNY7PRC/1rUQLJAmOaFY+HnYsnHw3\nAcpYFAW+T1jDJ3u2ygemkcTGZWKmVRPgd+M5IuW6Co6kx+Jo4UB35y5NGJ1p9HXvjbuVKwdSjjTo\nD/eNmGk1zLmrF7PCu3FPmH+r2zzzUn4i70Z/SEFFIfd2vZMxviMY6N0XgJgG7BmnVqno7e9CfnEF\nF69IVWPR8kiCI5qduZmGOWNGcUe7aaj0WmLKt/Pvn78jO7+0uUNr1dJzS0jOLPp/9u48Pqr6evz/\na5ZM9ky2mSSE7AEC2QMiAipgULQq7kQQxdZ9LVXr8mm1v2+FQluXqhWstqhQFNQUd3HFlUiA7ARI\nAiSBrJN9X2bm90dIAAkhCZnMTDjPx4PHgxnu+94zXBJO7nm/z5voUO8BtwzIqs6lrbud6f5JKBX2\n/y1BqVByceg8jGYjX5R8O6Lnjhin5cKEQLtLbgrqingh41+0dbdz8+RFXDD+PACmj09AgYKMqpxh\nnbevTFUkq6mE7bH/72ZizLg4Op4HE+5CbXKmzj2DP37yJun5I7ca5myT1VeeGnh5+FgpTx1vul8i\n3k5e/FT2M42dZ3crgryaffwz6990m4z8JuYmzg2Y2vdnXs5awrWhHGg4REPH0J/CRIf2dDXOLJB5\nOML2SIIjbMpEXRB/nPkAbkot6At5LWsTr32cK0tRh6F39/CB5t/Uttexr66QcG0Ifi4DLyO3Jyql\nivnBc+gydfN1yffWDsdqMqpyeCX7dQDujFtGoj72pGMS9bGYMZM1jDKVo0bF5BAvDlc3S1djYXMk\nwRE2x9fFhz/MfBB/Z3/U+sPsbNvKU69vp/BIg7VDsxut7V3sL60nLMAdzwH2Rfq5fLdd974ZyHkB\n09Bq3PnuyE80d/W/zcJY9nP5Lv6duwG1UsW98b8h2qf/vcUSdDEAZAwjwYFjTwizi+QpjrAtkuAI\nm+SucePhc+4hUhuOyruSRv2PrNq4gy3fH8BoMlk7PJuXc6AWo8k84N5TZrOZtIqdOCgdSNLbZ++b\ngTioHEgOvpAOYyfbSofX68VefXd4O2/mb8JZ7cQDiXcwwSvilMd6OXkS5hFMQV0RTZ3NQ75WfETP\nv7FM6WosbIwkOMJmOauduC/hN8T7RqPS1uI8ZQcf/LyPVRt2U1XXau3wbNpgylNFDYcwtNWQoIvF\nWe00WqGNqlmBM3BzcGXb4R9p6z47Jq1/UbyNTfv/h7uDG79NuotQj+DTjkk4WqbKrs4b8vV8tE4E\n6d3YW1xHe6eUkoXtkARH2DQHlQO/ibmJmQHnYHJuwDNhFwcMlTy1Lp3vs8swy3Lyk3QbTeQU1eDj\n4UiQ/tSbP/ZOLj5vDJanejmqNMwLOp+27ja+P7zd2uFYlNls5qMDW9lS9AmejlqWT72bQLeAQY1N\n1PXMzcmoHu5qKp+jXY3rhjVeCEuQBEfYPJVSxeKo65gfPIcOZSPeSbtQOjWx7pO9vLwll+a2LmuH\naFMKDjfQ2tFNQqTulMuZO4yd7K7KwtvJiwle4aMc4ei6YPx5OKud+Kr0OzqMndYOxyLMZjPvFX7I\np4e+wtfZh98l3T2kSeM+zt4Eu49nX10hLV1Dfzra+6RQylTClkiCI+yCQqHgqsjLuDryV7SamnGO\nTickrItd+6p58t8/kyf74fTp3eE5foDdwzOqsukwdnKu/9Qx0ftmIM5qZ+aMn0VzVws/lv1s7XBG\nnMlsYuPe9/im9Af8Xf34XdLd+DgPffPPRF0sJrOJbMOeIY8NC/DAw8WB7KIaadIpbMbY/s4mxpzk\n4AtZOvkGOkwd1Pt9x/mz1DS1dvHM25n858O8s/6bq9lsJrPAgJNGxaQgr1Med6z3zdRTHjOWzAma\njUal4cvib+kyjp0nfkaTkTf2vM1P5TsIcg9keeJdaB09hnWuBH3PaqrMquwhj+3pauxLY0snh8rP\n7r5DwnZIgiPszoyAadwesxQzZnZ3f8LVVzri5+3C/7YVsu7jfEymszfJKa9ppaq+jZgwbxzU/X95\nG9pqKKg/wATPcHydB24COFa4ObhyQeB5NHQ2klax09rhjIguYxev5q5nZ2Um4dpQHky8AzeN67DP\np3fREegWQH5tAW3dQ+9pI2UqYWskwRF2KU4Xzb3xt6FRavik7H3mze9kYrAnP+ZW8NrHe87apeS9\n/7kkTDj16qm08p4NTcdi75uBzAu6ALVSzRfF2zCajNYO54x0GDtZm/06OYY9RHlN4L6E23BWO5/x\neRN1cRjNRnIM+UMeGx3mhVql6CuRCmFtkuAIuzXBK5zlSXfhpnFly8GPSLqgnohAD9LyKnn1w7Mz\nycksMKBQQFxE/wmOyWzi54pdaFQaEnQnd7Udy7SO7swaN52a9jrSKzOsHc6wtXW38VLma+ytKyDW\ndwp3xS3DUaUZkXP3djoezt5UTho1UcFelFY1yz5ywiZIgiPs2nj3cTyUdC8+Tt68v/8zZpzfzoTx\nWnbkV/HKB3voNp49SU5jaydFRxqYEKjFzdmh32MK6g5Q215Hkj4OJ/WpOxyPVcnBF6JSqNha/DUm\ns/3922jubOEfGf/iQMMhpvklcHvMUhxU/d/r4fB31RPg6see2n20D6NvUN/mm/IUR9gASXCE3dO5\n+PBg4h14OLqx5eBH/OpiVyYGebJzbxVr3887a5Kc7MIazEDChFMvD97e1/vmnFGKyrZ4O3lxrv9U\nqloNZAxjMq011Xc08FzGWkqbjjAzYDq3TElBpTz1LvHDlaCLpdvUTV7N3iGP7d22IbNQtm0Q1mfR\nBGflypUsWrSIlJQUsrP7/2byzDPPsHTp0hPea29vJzk5mdTUVAC6urp46KGHuO6667jllltoaJA9\nicSJfJy9eXjWnShR8Obet1hy+Tiigj3Zvb+al/+XS1f32E9y+paHn2L38LbudjKrc/B19iFCGzqK\nkdmWi0PmokDBZ4fs5ylOTVstz+1aQ0VLJfOCzmdx1LUWW95/JmUqX60z43Wu5BfX0dFp3/OchP2z\nWIKzY8cOiouL2bRpEytWrGDFihUnHVNYWEh6evpJ769ZswatVtv3evPmzXh5efHuu+9y2WWXsXPn\n2FgFIUZWlC6SG6Oupa27jf/kv8ntV01kSqgXmYUG/vm/HLq6x+433K5uI7kHa/HzdiHAp/+VNLur\nsugydTHDf9opGwCeDXQuPkzzS6CspYLcYUymHW2VLVU8u3sNhvZaLg1N5prIyy16/8a5+qN38SWv\nZu+wGiPGR/rSbTSxR3pTCSuzWIKzfft2kpOTAYiIiKChoYHm5hM3clu1ahXLly8/4b2ioiIKCwuZ\nM2dO33vffPMNV155JQCLFi3ioosuslTYws7NCJjG/OA5VLUaWL9vI/deHU1MmDfZRTW8mDp2k5z8\n4no6uowknOLpDfSsnlKg4NyApFGMzDZdEjoPgM8OfW3T230caS7nud1rqe9o4KqIy7g8/GKLJ6cK\nhYJEXRydpi721Owb8vgEWS4ubITFEhyDwYCX17FGY97e3lRXV/e9Tk1NZfr06QQGBp4wbvXq1Tz2\n2GMnvHfkyBG+++47li5dyvLly6mvr7dU2GIMuDJiAbG+U9hXV8gHhz7m/mtjiYvwIfdALS+8m01H\n19hLcnrLU6faPbyytZoDDYeY5BWJt9OpGwCeLQJc/UjQxVLcVMre2gJrh9Ovgw0lPLd7LU1dzSya\neDXzQ+aM2rWPlamGPk8pLMADd+lqLGyAerQudPxPSfX19aSmprJu3ToqKyv73t+yZQsJCQkEBQWd\nNDYsLIz77ruPl19+mVdeeYVHH330lNfy8nJBrR75yXfH0+ncLXp+MTy99+XhC27nya/+zndHthPp\nF8yf7jifVW/sZMeeCta8n8cff30uTo6j9s/fosxmMzkHanBzduC8hPGoVCf/3PJF9lcAXDxpttX+\n7dra18yNiVeQ+XkOX5Vt44Io2+ronFe1n5eyXqXD2Ml95y7jgtBzLXat/u6Lr+8k/Pb4kle7F623\nE5ohrtSaHu3PV+mlNLQbmRgsCfVw2drXjL2x2Hd4vV6PwXDsEWVVVRU6Xc/qjrS0NGpra1myZAmd\nnZ2UlJSwcuVKqqqqKC0tZdu2bVRUVKDRaPD398fX15dzzulZ9TF79mxefPHFAa9dVzf0zeKGQqdz\np7pa2pHbml/el99MuZm/7XyR1zPewcXkzm2/iqKrq5uMAgN/WPMjD14fh5PG/pOc4oomDA3tnBft\nR21ty0l/bjKb2HYgDSeVE2GOEVb5t2uLXzNueBLtE0Ve9V5+2p9lM5uO5tXs5dWcNzGZzdwWfROT\nXadY7O9uoPsS6xPNlyXf8v2+XcTpood03qjxWr5KL2Vbeglezvb/NWYNtvg1Y6tOlQharEQ1a9Ys\ntm7dCkBeXh56vR43NzcAFixYwCeffMLmzZt56aWXiI6O5oknnuD555/nvffeY/PmzVx//fXcc889\nzJw5kwsuuIDvv/++71xhYWGWCluMIT7OXtwRdzNKFPw7dwOGdgN3XxXD1Ek69pXW89zmLNo6uq0d\n5hk71r24/+Xhe2sLqO9oYKpfPJoRagg3ViwI7ZnPt7X4aytH0iOjKodXst8A4M64ZSTordeMsa9M\nVT301VRTQr17uhoXyTwcYT0WS3CSkpKIjo4mJSWFp59+mqeeeorU1FS++OKLIZ9r6dKlfPvtt9x4\n4418+eWX3HHHHRaIWIxF4dpQFkddR1t3O2uz19FhaufOK6OZPllPweEGnt2cafdJTmahAZVSQUxY\n/ztIp/X1vjm7tmYYjHBtCBO9Ismv3c+hxhKrxpJWvpN/527AQanm3vjbiPaZZNV4QtyD8HL0JMew\nhy7T0L5GnB3VTAr2oqRSuhoL67Hos8OHH374hNdRUVEnHTN+/HjWr19/0vv3339/3++dnZ154YUX\nRj5AcVY4N2AqFa1VfF78Da/lbuC++N9w+xVTUCoVpOVV8symTH53QzwuTiPXEXa01Da2U1zRRHSo\nF879zClq7Woly5CHn4uOUI9gK0Ro+y4Nncf+ukI+O/Q1d8Uts0oM3x3+iU37t+Cidua+hNsI8Qg6\n/SALUygUJOpj+br0e/bVFhDjO3lI4xMifck7WEtWUQ1zEwNPP0CIESadjMVZ4YrwS4j3jWZ/XSGb\n929BqVBw26+mMDPGnwNljfz97Uxa2rusHeaQZRX1dIyNP8XqqZ2VWXSbupkRcHb3vhnIBM8IwjxC\nyDHs4Uhz+ahf/4vibWzavwV3jRu/TbrLJpKbXmdSpoqP6GlZINs2CGsZdoJz6NChEQxDCMtSKpTc\nPCWFQLcAfij7mW8P/4RSqeDXl01mdlwAhyqa+NtbGTS32VeSc7rl4WnlO1GgYLq/9L45FYVCwYKj\nfXG2Hhq9uThms5kPD2xlS9EneDl6sjzpbgLdAkbt+oMR6hGMVuNBTvWeIe/A7uvpTGBvV+Mx2JpB\n2L4BE5xbb731hNcvv/xy3++ffPJJy0QkhIU4qR25K24Z7ho33i34gPya/SiVCpZdGsUF8eMoqWzm\nb29l0NQ69O6t1tDRaWTPoTrG69zw9XQ+6c/Lmisobiplss9EPB21/ZxB9Ir2iSLIbRy7q7KpbKmy\n+PXMZjPvFXzIZ4e+wtfZh+VJd+Pncuo9xKxFqVCSoI+hpbuV/fVFQx4fH+FLV7d0NRbWMWCC0919\n4sSytLS0vt/bcvdPIU7F28mLO2JvQaVU8e+8DVS0VKJUKLh5wSTmJgZSWtWT5DS22H6Sk3eolm6j\niYQJ/XcvTqs4uzfWHIqepzgXYcbM1uJvLHotk9nExr3v8s3hHwhw9eN3SXfj42y7vWISdcPfmyqh\nb3dx2XxTjL4BE5xf1uyPT2qkni/sVbg2hCVHV1atyX6d5q4WlAoFN108kYumjudwdQt/fSuDBhtP\ncjILestTJ//kbzQZ2VGxGxe1M7G+U0Y7NLsUp4vG39WP9MoMatos88TBaDLyet5b/FSeTrB7IL9N\nvAuto4dFrjVSIjzDcHdwI6s6d8hlqvBxHrg5O5BVZJCuxmLUDWkOjiQ1YqyY7p/EJSHzMLTV8FrO\nerpN3SgUChYnT2D+tCDKDC38deNu6ps7rB1qv0wmM1lFBrSuGkIDTm5ytad2H02dzUzzS8RBKY3W\nBkOpUHJJyFxMZhOfl2wb8fN3Gbt4NfdNdlVlEaEN5YHEO3DT9L8xqi1RKpTE62No7mqhqOHg0MYq\nFcRF+NDQ3ElxhTStE6NrwASnoaGB7du39/1qbGwkLS2t7/dC2LPLwy8mXhdDQf0BNu/fgtlsRqFQ\nkHJRJAumB1Ne08rqjRnUNdleknOgvJGm1i7iI31Q9vODh/S+GZ6p+nh8nX1IK0unvqNhxM7b3t3B\nmux15BjyifKawL0Jt+GsPnnelK0amTKVrKYSo2vABMfDw4OXX36575e7uzv//Oc/+34vhD1TKpTc\nMiWF8W7j+LFsB9sO/wj0PKm8fm4El80IobK2ldX/3W1zzcp6y1P9LQ9v7mwhx5DPOFd/gtyl/8hQ\nqJQqLgmZS7fZyJcl347IOVu72vhn1mvsqyskzjeau+JvxdHOOkpP8AzH1cGFzOpcTGbTkMZGh3mj\nUipkHo4YdQM+u+6vAZ8QY4mjSsNdcctYvfMF3iv4EL2LjmifSSgUCq69MBylUsFHPx1i1X938/vF\nifhqbeOn7qxCAw5qJVNCT+5enF6ZgdFslN43wzTdP4lPDn7JD0d+5pKQebhr3IZ9rqbOZv6Z+Rql\nzWVM80vg5smLUCktuxGwJaiUKuJ9o/mpPJ0DDcVEeg5+u5yersae7DlUR11TB17ujhaMVIhjBnyC\n09zczOuvv973+u2332bhwoU88MADJ2ykKYQ983Ly5M7YZaiUKv6T+1/KW3p2uFcoFFxzQTgLZ4dh\naGhn9X8zqK5vs3K0UFXfxhFDC9Gh3jg6nPyfZVr5TpQKpfS+GSa1Uk1yyIV0mbr4uvT7YZ+nvqOB\n53evpbS5jFnjpnPLlBS7TG56JejjAMgcRpmq90mj7E0lRtOACc6TTz5JTU3PY8WDBw/y7LPP8uij\njzJz5kxWrFgxKgEKMRrCtMEsjbqedmM7a7PW0dx5bFfuhbPDuPqCcGoa21m9cTdVFt6t/nSy+spT\nJy8PL20q43BzGdE+UWf05OFsNzNgOu4aN747/BOtXUO/3zVttTy3aw0VrVXMCzqfGyddi1Jh343j\nJ3lF4Kx2JqM6Z8hlqr4Ep0ASHDF6BvyKKy0t5aGHHgJg69atLFiwgJkzZ5KSkiJPcMSYM80/kQWh\nF2For+W13J6VVb2umBnKdXMiqG3sYPXGDCprrZfk9O4e3t/8m59lcvGI0KgcSA6+kHZjR9/crMGq\naKni2d1rMLTXclloMtdEXj4mSoVqpZo43ynUdzRQ3Fg6pLF6T2fG+bqyR7oai1E0YILj4uLS9/sd\nO3YwY8aMvtdj4QtWiF/6Vdh8EnSxFNQfYNO+/53Q++myGSHcMDeSuqYOVm3cTXlNywBnsozW9i72\nl9YTFuCOp9uJcxm6Td2kV2bg5uBKjM/QNkYUJ5s9bgauahe+Kf2B9u7BTTI/3FTGc7vXUN/RwNWR\nv+JX4RePqe+VZ7Q3VaQPXd0m8ovrRjosIfo1YIJjNBqpqamhpKSEjIwMZs2aBUBLSwttbdafiyDE\nSOvZs2oRQe6B/FSezje/mIOx4NxgbrxoAg3NnazemMERw+gmOTkHajGazP3uPZVbs5fmrhbO8U+0\n67ketsJJ7cjcoPNp7W7j+yNppz3+YEMxz2e8QktXKymTriY5+MJRiHJ0RXlPxEnlSGZVzpC72cty\ncTHaBkxwbr/9di677DKuuOIK7rnnHrRaLe3t7SxevJirrrpqtGIUYlQ5qjTcGXsLWo07qYUfk2vI\nP+HP558TxJL5E2ls6eRvG3dzuLp51GIbqDyVVp4OyNYMI+nC8TNxUjnxVcl3dBpPvRHr/roiXsh8\nlQ5jBzdPWcT5geeNYpSjx0GpJsZ3MjXtdZQ2HRnS2Ihx2p6uxoUG2epHjIoBE5wLL7yQH374gR9/\n/JHbb78dACcnJx555BGWLFkyKgEKYQ1eTp7cEXcLaqWKdXkbKWuuOOHPL5o6nqWXTKKxtYu/bsyg\ntMrySU630UROUQ0+Ho4E6U+cQNzY2URezT6C3MbZ3I7U9szFwZkLx8+kqauZn8p29HtMriGfl7P+\njdFk5DcxN4351WuJR1dTDbVMpVQqiA33ob65k5LK0fuhQJy9BkxwysrKqK6uprGxkbKysr5f4eHh\nlJWVjVaMQlhFqEcwSyffQLuxg7XZr5+wsgpgbmIgyy6NoqWti79u3G3xVvQFhxto7egmIVJ30ryO\nHRW7MZlNzJCnNyNubtBsNEoHvijZRpfpxA2Id1dl86+cNwEFd8UtI0EXY50gR9EU70loVBoyqrKH\nXqaa0PPkMVPKVGIUDNjob968eYSFhaHT9Wzm98vNNt98803LRieElU31S6C8pYpPD33Jv3Le5IHE\n21Eft7fTBfHjUCjg9U/28ve3M3goJYFQf8tsntg7dyH+F7uHm81m0sp3olKomOafYJFrn83cNW7M\nDpzB16Xfs6N8F7MCzwV6+g1tyH/naLPIW5ngFW7lSEeHRuVAjE8Uu6uyOdJcznj3cYMeGx3a09U4\ns9DAwtmDbxYoxHAM+ARn9erVBAQE0NHRQXJyMv/4xz9Yv34969evl+RGnDUuC0smUR9HUcNB3tqX\netJPrefHjeM3l0+mtaObv72VyYGykd+nzWw2k1lgwEmjYlKQ1wl/VtJ0mPKWSmJ9p+DmYPubN9qj\ni4IvQK1Q8XnxNxhNRr49/BPr8zfjrHbigcQ7zprkpldvmSpziGUqFyc1E4M8Ka5ossk93sTYMmCC\ns3DhQv7zn//w/PPP09zczJIlS7jtttv48MMPaW+3rb15hLAUpULJzZNvINg9kLTynXxV+t1Jx8yM\nCeD2y6fQ3tnNM5syKDwychs1ApTXtFJV30ZMmDcO6hO/bGVjTcvzdNRy3rjpGNprWZvzOpv3b8Fd\n48Zvk+4ixCPI2uGNuinek3BQqs9o881s6WosLGxQrTUDAgK45557+PTTT7nkkkt4+umnmT17tqVj\nE8JmaFQa7oxbhlbjzpbCT8gx7DnpmBnR/tx5ZTQdnSae2ZTJ/tL6Ebt+75yF3jkMvbqMXaRXZuKh\ncWey98QRu5442fzgC1EqlOyp2YeXoyfLk+4+ayd0O6kdmeITRUVrVd/WJoPV24FbNt8UljaoBKex\nsZENGzZwzTXXsGHDBu68804++eQTS8cmhE3xdNRyZ9yyU66sApg+2Y+7FkbT3W3iuc1Z7CsZmaZm\nmQUGFAqIizgxwck27KGtu43p/knS+8bCfJy9WRAyj3BtKMuT7sbPRWftkKwqUXe06V9V9pDG6b1c\nCPBxYc+hWjqlq7GwoAETnB9++IHly5dz7bXXUl5ezqpVq3j//ff59a9/jV6vH60YhbAZIR5BLJ28\niA5jJ2uz19HUefJy12lReu6+KoZuo4nn3sk6486tja2dFB1pYEJgTx+R40l5anT9KvxiHpp6Dz7O\nXqc/eIyL8Z2MWqEadpmqU7oaCwsbMMG57bbbyM/PJykpidraWtatW8fjjz/e90uIs9FUv3guC5tP\nTXsd/8p586SlwwBJE3Xce00sJpOZf7yTRd6h2mFfL7uwBjOQMOHEJwb1HQ3k1+4n1CMYf1e/YZ9f\niOFwVjsx2WciZS0VVLZWD2lsvHQ1FqNgwGXivSul6urq8PI68SeWw4cPWy4qIWzcZaHJVLZUsasq\ni7f3pnLT5OtP6k2TEOnLfdfE8lJqLi+8m83918QSE37yDuCn07c8/Be7h+8o340ZMzMCpg7/gwhx\nBhJ0seQY8smsyuGS0HmDHhcR6IGrk5qsohrMZvOY2q9L2I4Bn+AolUoeeugh/vjHP/Lkk0/i5+fH\n9OnT2b9/P88///xoxSiEzVEoFNw0+QZC3INIq9jJlyXf9ntcXIQvD1zbM1fhhfdyhrxypKvbSO7B\nWvy8XQjwObYE3Gw2s70iHbVSzVS99L4R1hHnOwWlQjnkrsYqpZK4CB/qmjqkq7GwmAETnOeee47X\nX3+dHTt28Mgjj/Dkk0+ydOlS0tLSeOedd0YrRiFskkblwB1xN+PpqOX9ok/Jrs7r97iYcB8euC4O\npQJeSs0hs2DwSc7ekno6uowk/OLpzcHGEqpaDcT7RuPi4HxGn0OI4XJxcCHKawKlTUcwtA1tVVRf\nmUqWiwsLOe0TJE1MOwAAIABJREFUnIiICAAuuugijhw5ws0338xLL72En5/U/IXwdNRyZ+wtqJVq\nXt/zFkeay/s9LjrUmwevj0epVPDP/+Wwe//g5iz0JkO/3D1cNtYUtiJR37uaamhPcWLCfFApFTIP\nR1jMgAnOL+uiAQEBzJ8/36IBCWFvgj3Gc/OUnpVVa7LW0djZ/55Uk0O8WH59PGqVkjVbctm5t2rA\n85rNZjILDbg6qYkcr+17v9PYya7KLDwdtUzyjhzRzyLEUMX5Rg+rTNXb1fhgeRP1zdLVWIy8QfXB\n6SUTwYToX5I+jsvDLqauo55XT7GyCmBSsBe/WxSPWq1k7ft57Mg/dZO0kspm6po6iIvwQaU89qWa\nWZ1Lu7GDc/2nolQM6UtYiBHnpnFlomcExY2l1LYPbdl3fERP6TW7SJr+iZE34HfHjIwM5syZ0/er\n9/WFF17InDlzRilEIezDgtCLmKqP50BDMW/tfe+UOy1PGO/JQ4sScNQoeeWDPNLyTm4YCMd3Lz5x\neXhv7xtZPSVsRYK+Zxf1zOrcIY2LnyDLxYXlDLhM/LPPPhutOISwe70rqwzttfxcsQt/Vz0Xh8zt\n99jIQC0PLUrkmU2ZvPrRHkxmMzNjTmz7n1loQKVUEBPm3fdeTVsd++uKiNCGoj/LO+kK2xGvi2HT\nvi1kVOUwL+j8QY/zO9rVOO9QLV3dRhzU0o1bjJwBn+AEBgYO+EsIcSKNyoE7Y2/B01HLB0WfkXWK\nlVUA4eM8eOTGBFwc1fz7o3y+zy7r+7O6pg6KK5qICvbE2fHYzyE7KnYd7X0jnYuF7fDQuBPpGcaB\nhkPUdwxto9n4CF86u0zkF4/c3m1CwBDn4AghTk/r6MFdcctwOLqy6nBT2SmPDfX34OGURFyc1Kz7\nZC/fZh4Bjm/ud2z1lMlsIq18JxqlA0n6OMt+CCGGKOHoaqohl6n6Nt+UMpUYWZLgCGEBQe6B3DIl\nhU5jJ2uzX6eho/+VVQAh/u48cmMibs4OvPHZPr7JOHJs/s1xCU5R/UEM7bUk6uNwUjtZ/DMIMRQJ\nuqPzcIa4XDxyvPZoV2PDKeetCTEckuAIYSEJ+liuCL/k6MqqN+gydp3y2GA/d36/OBEPFwfWb91H\n7oFaxutc8fU81sQvrXwXIJOLhW3ydNQSrg2lsP7gKVsl9EelVBIb7kNtYwelVdLVWIwcSXCEsKBL\nQuYxzS+Bg40l/HeAlVUA43Vu/H5xElpXDSazmYQJx57etHd3sLs6Gx8nLyI9w0cjdCGGLFEXgxnz\ngHPP+iObbwpLsGiCs3LlShYtWkRKSgrZ2dn9HvPMM8+wdOnSE95rb28nOTmZ1NTUE97//vvvmTRp\nksXiFWKkKRQKboq6nlCPYNIrd/N58TcDHj/O15VHlyRxQXwAcxPH972fUZVNp7FTet8Im9Y3D2eI\nZarYcG+UCgWZhdIPR4wci32n3LFjB8XFxWzatIkVK1awYsWKk44pLCwkPT39pPfXrFmDVqs94b2O\njg7+9a9/odPJ0lhhXxxUDtwRewtejp58cOCz007C9Pd2Ydmlk/Fyd+x7L62ip/fNubJ6Stgwbycv\nQjyC2F9fRHNny6DHuTg5MDFIy8HyRhpaOi0YoTibWCzB2b59O8nJyQBERETQ0NBAc/OJ9dVVq1ax\nfPnyE94rKiqisLDwpEaCa9euZfHixWg0GkuFLITFaB3duTNuGRqlA2/kvUVp05FBj61uraGw/iAT\nPSPwdfY+/QAhrChRF4vJbCLbMLwyVbaUqcQIGbDR35kwGAxER0f3vfb29qa6uho3NzcAUlNTmT59\n+kn9dFavXs0f//hHtmzZ0vfewYMH2bt3Lw8++CB/+9vfTnttLy8X1BZuGKXTuVv0/GJ4bPm+6HST\neEDza/7+4yu8mvsmf5n/KJ7O2tOO+zpnGwDzJ8626c93OvYc+1g20vflIucZbCn6hLz6PSyMv2jQ\n4+ZND2HT14Xkl9ZzTbJMRQD5mjlTFktwfun4yZX19fWkpqaybt06KiuP7cWzZcsWEhISCAoKOmHs\nX/7yF/7whz8M+lp1da1nHvAAdDp3qqsHv0pAjA57uC9hjhFcGb6ADw58xsptL/PbxDtxUDmc8niT\n2cTXRT/hpHIkwnmCzX++U7GHe3M2ssR9UeJEkHsgOZX7KC6rxMXBZVDjHAA/bxcy9lVTVl5/1nc1\nlq+ZwTtVImixBEev12MwHHvUWFVV1Td/Ji0tjdraWpYsWUJnZyclJSWsXLmSqqoqSktL2bZtGxUV\nFWg0GhQKBQcOHODhhx/uO89NN93Ehg0bLBW6EBZ1cchcyluqSK/czYa977Bsyo2n3Mh2f10RdR31\nzAw4B0eVlGeFfUjQxVLadIQcQz7nDqGtQUKkD1t3lLK3pJ7YcB8LRijOBhZLcGbNmsWLL75ISkoK\neXl56PX6vvLUggULWLBgAQCHDx/m8ccf54knnjhh/IsvvkhgYCBXX301V199dd/78+bNk+RG2DWF\nQsGSqGsxtNWwszKTAFc/FoT2/yi/d2NNmVws7EmiPpYPD3xGRnX2EBMcX7buKCWz0CAJjjhjFptk\nnJSURHR0NCkpKTz99NM89dRTpKam8sUXX1jqkkLYDQeVA3fE3YyXoycfHthKRj/Latu628iszkHn\n7EOENnT0gxRimPxcdIxz9Se/Zj9t3e2DHhcRqMXFUU12oXQ1FmfOonNwestKvaKiok46Zvz48axf\nv/6k9++///5+z/n111+PTHBCWJmHxp274pbxzO6XeXPP2/g4exHsfqz3za7KLLpM3cwImHbKEpYQ\ntipRH8vHB78g15DPOf6JgxqjVimJjfDh5z2VHK5uIUjvZuEoxVgmHcOEsKLx7uO4dcqNdJm6eSX7\nDRo6Gvv+LK18FwoUnOsvWzMI+5N4dEPYzOqhNf3r3XwzU5aLizMkCY4QVhani2ZhxKXUdzTwSvYb\ndBq7qGip4mBjMVHeE/By8rR2iEIMWYCrH34uevJq9tLe3THocbHhPigVCumHI86YJDhC2IDk4As5\n138qxU2lbMjf3De5eIY8vRF2LFEfS5epmz21+wY9xtXJgQnjtRwoa6RRuhqLMyAJjhA2QKFQcGPU\ntYRrQ9lVlcVXpd/hrHYiThdj7dCEGLZEXc/eVBlV/e9FeCrxkb6YgawieYojhk8SHCFshINSzR2x\nN+Pt5IXJbGKqXwKaAZoACmHrAt0C0Dn7kFuzl07j4J/G9M7DyZbNN8UZkARHCBvirnHj7rhbidfF\nkBx0obXDEeKMKBQKEvVxdBo72VO7f9DjAnxc8fNyJvdQLV3dJgtGKMYySXCEsDHj3Py5I/ZmdC7S\n6EzYv94yVWY/vZ4GEh/pS0enkX0ldZYIS5wFJMERQghhMUHugfg4eZFj2EOXqXvQ43p3F8+SMpUY\nJklwhBBCWIxCoSBBF0u7sYO9QyhTTRivxdlRTaZ0NRbDJAmOEEIIi0rU966mGnyZSq1SEhvuTU1j\nO0eqWywVmhjDJMERQghhUSEeQXg6ask27KF7OGUqWS4uhkESHCGEEBalVChJ1MXS1t3GvrqiQY/r\n7Wos2zaI4ZAERwghhMUl6Ie+msrN2YHI8VoOHJGuxmLoJMERQghhceHaEDw07mQZcjGajIMeFx/p\ngxnIOSCrqcTQSIIjhBDC4pQKJQm6GFq6WimoPzDocQlH5+FImUoMlSQ4QgghRkXfaqrqwZep/L1d\n0Hs5k3tQuhqLoZEERwghxKiI0Ibh5uBKVlUuJvPgkhWFQkF8RE9X4/2l9RaOUIwlkuAIIYQYFSql\ninhdDE1dzRTVHxr0uISjm29KmUoMhSQ4QgghRs1wylQTgjxxdlSRJV2NxRBIgiOEEGLUTPSMwEXt\nTGZVzqDLVGqVkpgwHwwN7ZQZpKuxGBxJcIQQQowalVJFnC6ahs5GDjWWDHqcrKYSQyUJjhBCiFGV\nqBv63lSxET4oFLK7uBg8SXCEEEKMqkneE3BWO5FRlTPoOTVuzg5EBmopOtJAU6t0NRanJwmOEEKI\nUeWgVBPrO4W6jnpKmg4PelxCpC9mILtInuKI05MERwghxKhLGEaZKq53d3GZhyMGQRIcIYQQo26y\n90QcVRoyqrIHXaYa5+OCztOJ3IO1dBulq7EYmCQ4QgghRp1G5UCMz2QM7bUcbi4b1BiFQkF8pC/t\nnUb2SVdjcRqS4AghhLCKRH0cMLQyVXxvmapAylRiYJLgCCGEsIpon0lolA5kVA++TDXpaFfjTOlq\nLE5DEhwhhBBWoVFpiPaJoqrVQHlL5aDGqFVKonu7Gte0WjhCYc8kwRFCCGE1Cb17U1VlD3pMfETP\n5puymkoMRBIcIYQQVhPjE4VaqR7S5ptxfV2NJcERpyYJjhBCCKtxUjsxxXsS5S2VVAyyTOXuoiEi\nUEvhkQaa27osHKGwV5LgCCGEsKrEvjJV7qDHxEf4YDZDdpE8xRH9kwRHCCGEVcX6TkalUJE5hDJV\nQl9XY9m2QfRPEhwhhBBW5ax2ZrL3BA43l1HVOrgnMuN8XfHVOpF7sEa6Got+WTTBWblyJYsWLSIl\nJYXs7P5nyD/zzDMsXbr0hPfa29tJTk4mNTUVgPLycpYtW8ZNN93EsmXLqK6utmTYQgghRlnv3lSD\nfYrT29W4rcPIfulqbLPKa1pI/a6Ito7uUb+2xRKcHTt2UFxczKZNm1ixYgUrVqw46ZjCwkLS09NP\nen/NmjVotdq+188//zw33HADGzZsYP78+axbt85SYQshhLCCOF00SoVySF2NpUxl2woPN7By/S4+\n+qkYQ0P7qF/fYgnO9u3bSU5OBiAiIoKGhgaam5tPOGbVqlUsX778hPeKioooLCxkzpw5fe899dRT\nXHLJJQB4eXlRXy/ZuhBCjCWuDi5M8oqkpOkwNW21gxozKdgTR42KLOlqbHMyCqr529sZtHUYufWy\nKIL0bqMeg9pSJzYYDERHR/e99vb2prq6Gje3ng+ZmprK9OnTCQwMPGHc6tWr+eMf/8iWLVv63nNx\ncQHAaDSyceNG7r333gGv7eXlglqtGqmP0i+dzt2i5xfDI/fFdsm9sU22dF/ODz+H/Nr9FLQWEBWc\nPKgxU6P0/JRdTodZQZDedj7LSLClezMUn20/xJrUHBwcVDyxbDrTJvtZJQ6LJTi/dHx2XV9fT2pq\nKuvWraOy8ljfgy1btpCQkEBQUNBJ441GI7///e+ZMWMG55133oDXqquzbPtunc6d6uomi15DDJ3c\nF9sl98Y22dp9CXeKQIGCHw7uZIbPuYMaMznIk5+yy/kmvZhLzw2xcISjx9buzWCYzWbe/+EgH/x4\nCDdnB357fTwhvi4W/xynSgQtluDo9XoMhmOz4auqqtDpdACkpaVRW1vLkiVL6OzspKSkhJUrV1JV\nVUVpaSnbtm2joqICjUaDv78/M2fO5PHHHyckJIT77rvPUiELIYSwIneNGxM8w9lfX0Rdez1eTp6n\nHRMb4YOCnt3Fx1KCY2+MJhPrt+7ju6xyfLVOPLQoAT9vF6vGZLEEZ9asWbz44oukpKSQl5eHXq/v\nK08tWLCABQsWAHD48GEef/xxnnjiiRPGv/jiiwQGBjJz5kw++OADHBwceOCBBywVrhBCCBuQqI9l\nf30RmdW5zA2afdrjPVw0hAd6UHC0q7Gbs8MoRCmO19FlZO2WXLKKagjxc+e3N8SjddVYOyzLJThJ\nSUlER0eTkpKCQqHgqaeeIjU1FXd3d+bPnz+kc23cuJGOjo6+5eQRERH86U9/skDUQgghrCleF8Pm\n/e+TUZUzqAQHelZTFR1pJOdADedF+1s4QnG8ptZOXng3m6KyRqJDvbjn6licHUdt9suAFOYxOPV8\nNOp99lYbPRvIfbFdcm9sk63el2d3reFAwyFWzPoDWsfTT7Q9XN3Mk//egc7TiXuvjiXYzz4n5x7P\nVu/N8Qz1bTyzOYvK2lbOi/bj1ssmo1aNfv/gU83BkU7GQgghbEqiPhYzZrKqB7c31XidG5fPDKW6\nvp2n39zFtowjsmzcwkoqm1ixfheVta1cem4wv7l8ilWSm4HYVjRCCCHOegm6GAAyhrA31TUXhPPb\n6+NwdFDy5tZ9/OvDPVbpnns22HOollX/3U1jSyc3Jk/g+rmRKBUKa4d1EklwhBBC2BQvJ0/CPIIp\nqCuiqbP59AOOiovw5f/79XQiA7X8vKeS//d6OiWVtl3msTdpeyp4bnMW3UYTdy6MZv60k9u62ApJ\ncIQQQtichKNlquzqvCGN8/Zw4veLE7n03GAq69p6SlaZUrIaCZ/9XMK/PtiDxkHJ725IYLqVGvgN\nliQ4QgghbE7i0c03h1Km6qVWKbl+biQPXne0ZPWZlKzOhMls5u2vCtj8TSGebhoeWzKVqBAva4d1\nWpLgCCGEsDk+zt4Eu49nX10hLV3D604fH9lTsooI9OgpWb2xk9KqwZe8BHR1m3j1wz18nl5KgI8L\n/7d0mlX2lRoOSXCEEELYpER9LCaziWzDnmGfw9vDiUcXJ7Hg3GAqa1t5+s2dfCslq0Fp6+jm+Xey\n+HlPJZGBWh6/aSo+WidrhzVokuAIIYSwSQlHy1SZVdlndB61SskNcyN54Lo4NGolb3y2j1elZDWg\n+uYOVv13N/nFdSRO8OXhlAS76xItCY4QQgibpHfxJdAtgPzaAtq62874fAmRvvzp1ulEjPMgTUpW\np1Re08KKN3dRWtXMnMRA7r06Fo2DytphDZkkOEIIIWxWoi4Oo9lIjiF/RM7no3Xi0SVJLJh+rGT1\nXVaZlKyOKjzSwMr1u6hpbOfq88NYevFElErb63EzGJLgCCGEsFmJ+qOrqaqGvprqVNQqJTfMi+SB\na3tKVq9/updXP9pDe+fZXbLKLDDw97cyaOswsuzSKK6YFYbCBhv4DZYkOEIIIWyWv6ueAFc/9tTu\no727fUTPnTDBl6duPYfwcR6k5VXy/14/e0tW32WV8WJqz1yn+6+N5YL4cVaO6MxJgiOEEMKmJepi\n6TZ1k1ezd8TP7at15rElSVwyPYiKs7BkZTabef+Hg7z+6V5cnRx4ZHEi8ZG+1g5rREiCI4QQwqYl\n6uOAkS1THU+tUrJo3gTuvza2r2T12llQsjKaTLzx2T7e/+Egvlonnlg6lYhxWmuHNWIkwRFCCGHT\nAlz90Lv4klezlw5jp8WukzhB11ey2n60ZHV4jJasOrqM/DM1l++yygj2c+P/lk7F39vF2mGNKElw\nhBBC2DSFQkGiLo5OUxd7avZZ9Fq9JauLz+kpWf15DJasmtu6+PvbGWQWGpgS6sWji5PQujlaO6wR\nJwmOEEIIm3dsNdWZNf0bDLVKScpFE7j/mlgcVGOrZGWob2Pl+l0UHWlkxhQ/fnt9PM6OamuHZRFj\n81MJIYQYU8a7jcPXyZvcmny6jF04qCzfVTdxoo4/6d1Y834e2/MqOVTRxN1XxTBeZx97Mf1SSWUT\nz72TRUNzJwumB3Pd3AiUdrwM/HTkCY4QQgibp1AoSNTH0WHsJL92/6hd19fTmcdv6ilZlde08vQb\nO/neDktW+YdqWfXf3TQ0d5Jy0QRumBc5ppMbkARHCCGEnUjQxwCQUW2Z1VSncnzJSq1Ssu7Tvbz2\nUb7dlKx+3lPJs5uz6DaauGthNBefE2TtkEaFJDhCCCHsQoh7EF6OnuQY9tBlGv3kInGijj/deg5h\nAR5sz6vgz2/s5HC1ba+y2rqjhFc+yEPjoGT5DQlMn+xn7ZBGjSQ4Qggh7EJPmSqWtu529tUWWCWG\nfktW2bZXsjKZzbz9VQGbvi5E66bh0cVJTA7xsnZYo0oSHCGEEHajbzXVKJepjtdbsrrvmlhUKiXr\nPtnLvz/Op6PTaLWYjtdtNPHqh3v4PL0Uf28X/m/pVIL93K0d1qiTVVRCCCHsRqhHMFqNBznVezBO\nMqJSqqwWS9JEHUF6N9a+n8tPuRUcLG+0+iqrto5uXkrNIb+4johADx68Lh43Z8uvOLNF8gRHCCGE\n3VAqlCToY2npbmV/fZG1w0Hn6czjN00ledr4E0pW1lDf3MHq/+4mv7iOhEhfHk5JPGuTG5AERwgh\nhJ1J1B1dTWWhvamGSq1Ssjh5IvdefVzJ6qM9o1qyKq9pYeX6XZRUNXNhwjjuvSYGRwfrPd2yBZLg\nCCGEsCsRnmG4O7iRVZ2L0WQb814Apk7qWWUV6u/Oj7kV/PnNnRwxtFj8ukVHGvjLht0YGtq5anYY\nN18yCZVS/nuXvwEhhBB2RalQEq+PobmrhaKGg9YO5wTHl6zKDC38+Y10fswpt9j1MgsN/O2tDFra\nu1h2aRRXzg5DMcYb+A2WJDhCCCHsTqKud28q2yhTHc9B3VuyikGlVPLvj/P598cjX7L6LquMF9/r\n2Zvr/mviuCB+3Iie395JgiOEEMLuTPAMx9XBhczqXExmk7XD6dfUSXqe6i1Z5YxcycpsNvPBDwd5\n/dO9uDo58MiNiSRM8B2BiMcWSXCEEELYHZVSRbxvDI2dTRxoKLZ2OKek7y1ZTR2ZkpXRZOLNrfvY\n8sNBfDycePymJCICtSMY8dghCY4QQgi7lHC06V+mDZapjuegVrJ4/oklq/98nE9H19BKVh1dRv6Z\nmsu3mWUE6d34v5unEuDjaqGo7Z8kOEIIIezSJK8InNXOZFTn2GyZ6ni9JasQf3d+yCnn6TcGX7Jq\nbuvi729nkFloYHKIF48tScLTzdHCEds3SXCEEELYJbVSTZzvFOo7GihuLLV2OIOi93TmiZumctHU\n8RwZZMnK0NDGXzbsouhII+dO8WP5DfE4O8pGBKcjCY4QQgi7ZQt7Uw2Vg1rJkvkTueeqGFRKRU/J\n6pP+S1YllU2sWL+L8ppWLpkexO1XTEGtkv+6B0P+loQQQtitKO+JOKkcyazKsbkdvU9nWpSep5Yd\nLVll95Ssyo4rWeUX17F6424amjtZNC+SRfMmoJQeN4Nm0QRn5cqVLFq0iJSUFLKzs/s95plnnmHp\n0qUnvNfe3k5ycjKpqakAlJeXs3TpUhYvXsyDDz5IZ2enJcMWQghhJxyUamJ9p1DTXkdp0xFrhzNk\nei+XnpJVUk/J6v+9kc5PueV8n3GE5zZn0tll4o4rp3DJ9GBrh2p3LJbg7Nixg+LiYjZt2sSKFStY\nsWLFSccUFhaSnp5+0vtr1qxBqz227O2FF15g8eLFbNy4kZCQEN59911LhS2EEMLO9K6m+rLkW7qM\nXVaOZugc1EqWXHysZPXaR/n8dcNO1Coly2+IZ8YUf2uHaJcsluBs376d5ORkACIiImhoaKC5ufmE\nY1atWsXy5ctPeK+oqIjCwkLmzJnT997PP//MRRddBMDcuXPZvn27pcIWQghhZ6Z4T8LPRc+uqixW\npj9HQd0Ba4c0LL0lq7AAd3y1Tjy2JIkpod7WDuuM1LTV8v2RNKsknhZLcAwGA15eXn2vvb29qa6u\n7nudmprK9OnTCQwMPGHc6tWreeyxx054r62tDY1GA4CPj88J5xFCCHF206gc+P20+5kbNJvq1hqe\nz1jLW3vfo627zdqhDZney4U/3DyN1/5vPsF+7tYOZ9iMJiOfF3/Dn39+hrf3pVLeWjnqMYzaOrPj\nJ3/V19eTmprKunXrqKw89qG3bNlCQkICQUFBgzrPqXh5uaBWW3abeJ3Ofv/hjWVyX2yX3BvbNHbu\nizt3BywhuWYma9M38EPZz+TV7eW2qTdyTmC8tYMbFnu9N/nVBby2+y1KG8vROrpzc8JNTA2dPOpx\nWCzB0ev1GAyGvtdVVVXodDoA0tLSqK2tZcmSJXR2dlJSUsLKlSupqqqitLSUbdu2UVFRgUajwd/f\nHxcXF9rb23FycqKyshK9Xj/gtevqWi31sYCef3TV1U0WvYYYOrkvtkvujW0ai/fFE18eTryPL4q3\n8dmhr/jbD2tJ1Mdxw8SFeGjsJ2Gwx3vT3NnClqJP2F6ejgIFswNnsDB8AS4OLhb9LKdKBC2W4Mya\nNYsXX3yRlJQU8vLy0Ov1uLm5AbBgwQIWLFgAwOHDh3n88cd54oknThj/4osvEhgYyMyZM5k5cyZb\nt25l4cKFfP7555x//vmWClsIIYSdUyvVXBqWTKI+lv/ufY+Mqmz21RZwTeTlzAiYhkKWWo8os9lM\nWvlO/lf0MS1drQS6BXDjpGsI04ZYNS6LJThJSUlER0eTkpKCQqHgqaeeIjU1FXd3d+bPnz+kc91/\n//08+uijbNq0iXHjxnHVVVdZKGohhBBjhb+rH8uT7uL7I2m8X/QJG/a+Q3plBoujrsXX2cfa4Y0J\nZc0VvL3vfxQ1HESj0nBt5OVcOH4WKqVlp4kMhsJsb52RBsHSj/Xs8dHh2UDui+2Se2Obzqb7Utde\nz9v7Usmt2YuD0oHLwy9m7vjZNvEfcX9s/d50Gjv59NBXfFnyLSaziQRdDNdNuBIvJ89Rj2XUS1RC\nCCGErfBy8uSuuFvZVZXFO/vf53+FH7OrMoslUdcx3n2ctcOzK7mGfDbv30JNex3eTl7cMHEhsb5T\nrB3WSSTBEUIIcVZQKBRM80sgynsCqQUf8XPFLlbvfIH5wXO4NPQiHFQO1g7RptW11/NuwYdkVueg\nVCh7/t7CknFUaawdWr8kwRFCCHFWcXNw5eYpi5jml8Bb+1LZWvw1GdXZLJ50HRO8wq0dns0xmox8\ne+QnPjqwlQ5jJ+HaUG6cdA3j3Gy7w7IkOEIIIc5KU3wm8X/Tf8dHB7eyrfRHns9Yy+zAGVwVcSnO\namdrh2cTDjWW8PbeVEqby3BVu3Bd1EJmBExFqbD9vbolwRFCCHHWclI7ct2EK5mqT2Dj3nf54Uga\nuYZ8Fk28ijhdtLXDs5rWrjY+PPAZ3x9Jw4yZGf7TuDryV7hpXK0d2qBJgiOEEOKsF6YN5tFzHuhr\nEPhKzhsk6eO43s4aBJ4ps9nMrspM3i38kKbOZvxd9KRMupoJXhHWDm3IJMERQgghONYgMEEfy8a9\n77K7KpvD4pyAAAANyklEQVS9tQVcM+EKZvhPHfMNAqtaq9m0bwt76wpwUKq5MnwBFwVfgFppn6mC\nfUYthBBCWEiAqx/Lk+4+1iAwfzM7KzK4MeqaMdkgsMvUzefF3/B58Td0m7qZ4jOJRROvsvvPKgmO\nEEII8QtKhZILx88k1ncyb+/7H3k1e1nx87NcHn4Jc4Nm28Uk28HYW1vApv3/o6rVgFbjwfUTF5Kg\nixkTT6skwRFCCCFOwdvJi7vjbmVXZSbvFHxAauFHPQ0CJ19HoFuAtcMbtsbOJt4r+JCdlZkoUDB3\n/Gx+FX4xzmona4c2YiTBEUIIIQagUCiY5p9IlPdE3i34kPTK3axK/wcXB89hgZ01CDSZTfxY9jPv\nF31KW3c7Ie5BpERdTbD7eGuHNuIkwRFCCCEGwU3jyrLoFM7xT+Stve/xWfHXZFTnsDjqOiI9w6wd\n3mmVNpXx9r5UDjWW4KRyYtHEq5gdOGPMlNt+SRIcIYQQYgiifSbxh3Mf4sMDn/Ht4Z94bveaow0C\nL7PJEk97dzsfH/yCb0p/wIyZaX4JXBN5OVpHD2uHZlGS4AghhBBD5KR25PqJC5nml8B/j2sQmDLp\napvZeNJsNpNVncs7BR9Q39GAztmHRZOuZrL3RGuHNiokwRFCCCGGKUwbwmPnPMjW4m/Yeuhr1ma/\nbhMNAmvaatm8/31ya/JRK1RcFprMxSFz7Wq+0JmSBEcIIYQ4A2qlml+FzSdJH8d/8481CLx2whWc\nO8oNAo0mI1+VfscnB7+ky9TFRK9IUiZehZ+rftRisBWS4AghhBAjIMDVj99NvZvvDm/n/QOfsj5/\nM+kVGdwYdS2+zt4Wv35h/UHe3pdKeUsl7g5uLI66lnP8EsdET5vhkARHCCGEGCFKhZI5QbOI003h\nrX2p7KnZx4qfn+GK8EuYY6EGgc2dLWwp+oTt5ekoUDA7cAYLwxfg4uAy4teyJ5LgCCGEECPM28mL\ne+J+TXplBu8WfMB7hR+xc4QbBJrNZtIqdvG/wo9o6Wol0C2AGyddQ5g2ZETOb+8kwRFCCCEsQKFQ\nMN0/icneE3mv4EPSKzN6GgSGzGVByLwzmvBb3lLJ2/tSKaw/iEal4drIy7lw/CxUStUIfgL7JgmO\nEEIIYUHuGjeWRd94tEFgKp8d+oqMqhwWR1075AaBncZOPj30FV+WfIvJbCJeF8P1E67Ey8nTQtHb\nL0lwhBBCiFEQ7RPFH879HR8c2Mp3RxsEXhB4HldGXDqoBoG5hnw2799CTXsd3k5e3DBxoc303LFF\nkuAIIYQQo8RJ7cQNvQ0C89/huyPbyTbsGbBBYF17Pe8WfEhmdQ5KhZL5wXO4NCwZR5VmlKO3L5Lg\nCCGEEKMsXBvCY9N/y+eHvmZr8TeszX6dqfp4rp+4EHeNG9DT0+a7I9v58MBndBg7CdeGcuOkaxjn\n5m/l6O2DJDhCCCGEFTgo1fwq/GIS9XFs3Psuu6qy+hoERilDWbNzPaXNZbiqXbgu6kpmBEwbsxtj\nWoLCbDabrR3ESKuubrLo+XU6d4tfQwyd3BfbJffGNsl9sR0ms4lvD//EBwc+o9PY2ff+DP9pXBV5\nWd9THXEyna7/LTHkCY4QQghhZUqFkrlBs4nzjeadgvdpNjaxMPQyJnhFWDs0uyUJjhBCCGEjfJy9\nuCtumTxdGwFSzBNCCCHEmCMJjhBCCCHGHElwhBBCCDHmSIIjhBBCiDFHEhwhhBBCjDmS4AghhBBi\nzJEERwghhBBjjiQ4QgghhBhzJMERQgghxJhj0U7GK1euJCsrC4VCwRNPPEFcXNxJxzzzzDNkZmay\nfv162traeOyxx6ipqaGjo4N77rmHuXPnkp6ezrPPPotarcbFxYW//vWvaLVaS4YuhBBCCDtmsSc4\nO3bsoLi4mE2bNrFixQpWrFhx0jGFhYWkp6f3vf7mm2+IiYlhw4YNPP/886xatQqAv/zlL6xYsYL1\n69eTmJjIpk2bLBW2EEIIIcYAiyU427dvJzk5GYCIiAgaGhpobm4+4ZhVq1axfPnyvteXXXYZt99+\nOwDl5eX4+fkB4OXlRX19PQANDQ14eXlZKmwhhBBCjAEWK1EZDAaio6P7Xnt7e1NdXY2bW8+W76mp\nqUyfPp3AwMCTxqakpFBRUcHatWsBeOKJJ7jpppvw8PBAq9Xy0EMPDXhtLy8X1GrVCH6ak51qe3Zh\nXXJfbJfcG9sk98V2yb05M6M2ydhsNvf9vr6+ntTUVG699dZ+j3377bdZs2YNjzzyCGazmT//+c+8\n9NJLbN26lalTp7Jx48YBr2Xp5EYIIYQQts1iT3D0ej0Gg6HvdVVVFTqdDoC0tDRqa2tZ8v+3d28h\nUe19GMe/UxnhqW1GB7Ek7cLS6GBeZFlBWVGQpNWYOXUVhHRRVCQek0IwCKKMDlQgRqhpJ8nsQBlC\nGUVhImklQgczCSe0dLQZ3ReV2K43fNvb1jg9nzsXa8mzQPRx/df8f+vX09XVxYsXL8jKymLlypX4\n+voyfvx4pkyZgsPhoKWlhbq6OsLCwgCIiIigpKRkoGKLiIiICxiwJzhz587l6tWrANTU1DBmzJje\n5ally5ZRWlpKYWEhOTk5hISEkJyczIMHDzh16hTweYmrvb0dHx8fRo8ezfPnzwGorq4mICBgoGKL\niIiICxiwJzizZs0iJCSEuLg4TCYTGRkZnDt3Di8vL6Kion54TVxcHCkpKcTHx2Oz2UhPT2fIkCFk\nZmaSmpqKm5sbI0eOJCsra6Bii4iIiAsw9fR9OUZERETEBWgnYxEREXE5KjgiIiLiclRw/g9ZWVmY\nzWbi4uJ4/Pix0XGkj3379mE2m4mNjeXatWtGx5E+bDYbixcv5ty5c0ZHkT4uXbrEypUriYmJoby8\n3Og48sXHjx/ZsmULFouFuLg4KioqjI40aA3oLCpX0nf0RH19PcnJyRoZ4SQqKyt59uwZBQUFWK1W\nVq1axZIlS4yOJV8cOXJEs+OcjNVq5fDhwxQXF9Pe3s6hQ4dYuHCh0bEEOH/+PJMmTWL79u28ffuW\njRs3UlZWZnSsQUkFp5/+1+iJrx99F+OEh4f3DnL19vamo6MDh8PB0KHa8NFo9fX1PH/+XH88nczd\nu3eZM2cOnp6eeHp6smfPHqMjyRc+Pj7U1dUB0NraqtFE/4KWqPrp3bt33/ygfR09IcYbOnQo7u7u\nABQVFTF//nyVGyeRnZ1NUlKS0THkH169eoXNZmPz5s3Ex8dz9+5doyPJFytWrKCxsZGoqCgSEhLY\ntWuX0ZEGLT3B+UX6dL3zuXHjBkVFRb2bRYqxLly4wIwZM5gwYYLRUeQH3r9/T05ODo2NjWzYsIFb\nt25hMpmMjvXHu3jxIn5+fpw8eZLa2lqSk5P1/tovUsHpp5+NnhDjVVRUcPToUU6cOIGXlwbUOYPy\n8nJevnxJeXk5TU1NDB8+nHHjxhEREWF0tD+er68vM2fOZNiwYUycOBEPDw9aWlrw9fU1Otof7+HD\nh8ybNw+A4OBgmpubteT+i7RE1U8/Gz0hxmpra2Pfvn0cO3aMv/76y+g48sWBAwcoLi6msLCQNWvW\nkJiYqHLjJObNm0dlZSXd3d1YrdbesThivICAAKqqqgB4/fo1Hh4eKje/SE9w+ulHoyfEOZSWlmK1\nWtm6dWvvsezsbPz8/AxMJeK8xo4dy9KlS1m7di0AqampDBmi/3edgdlsJjk5mYSEBOx2O7t37zY6\n0qClUQ0iIiLiclTZRURExOWo4IiIiIjLUcERERERl6OCIyIiIi5HBUdERERcjgqOiBju1atXhIaG\nYrFYeqcob9++ndbW1n5/D4vFgsPh6Pf569at4969e78SV0QGARUcEXEKo0aNIi8vj7y8PPLz8xkz\nZgxHjhzp9/V5eXnaEE1EemmjPxFxSuHh4RQUFFBbW0t2djZ2u51Pnz6Rnp7O1KlTsVgsBAcH8+TJ\nE3Jzc5k6dSo1NTV0dXWRlpZGU1MTdrud6Oho4uPj6ejoYNu2bVitVgICAujs7ATg7du37NixAwCb\nzYbZbGb16tVG3rqI/AdUcETE6TgcDq5fv05YWBg7d+7k8OHDTJw48bvhg+7u7pw+ffqba/Py8vD2\n9mb//v3YbDaWL19OZGQkd+7cYcSIERQUFNDc3MyiRYsAuHLlCoGBgWRmZtLZ2cnZs2d/+/2KyH9P\nBUdEnEJLSwsWiwWA7u5uZs+eTWxsLAcPHiQlJaX3vA8fPtDd3Q18HqHyT1VVVcTExAAwYsQIQkND\nqamp4enTp4SFhQGfh+cGBgYCEBkZyZkzZ0hKSmLBggWYzeYBvU8R+T1UcETEKXx9B6evtrY23Nzc\nvjv+lZub23fHTCbTN1/39PRgMpno6en5Zt7S15IUFBTE5cuXuX//PmVlZeTm5pKfn/9vb0dEDKaX\njEXEaXl5eeHv78/t27cBaGhoICcn56fXTJ8+nYqKCgDa29upqakhJCSEoKAgHj16BMCbN29oaGgA\noKSkhOrqaiIiIsjIyODNmzfY7fYBvCsR+R30BEdEnFp2djZ79+7l+PHj2O12kpKSfnq+xWIhLS2N\n9evX09XVRWJiIv7+/kRHR3Pz5k3i4+Px9/dn2rRpAEyePJmMjAyGDx9OT08PmzZtYtgw/WoUGew0\nTVxERERcjpaoRERExOWo4IiIiIjLUcERERERl6OCIyIiIi5HBUdERERcjgqOiIiIuBwVHBEREXE5\nKjgiIiLicv4GC5NPK7Hm6ccAAAAASUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "metadata": { + "id": "JjBZ_q7aD9gh", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Task 1: Can We Calculate LogLoss for These Predictions?\n", + "\n", + "**Examine the predictions and decide whether or not we can use them to calculate LogLoss.**\n", + "\n", + "`LinearRegressor` uses the L2 loss, which doesn't do a great job at penalizing misclassifications when the output is interpreted as a probability. For example, there should be a huge difference whether a negative example is classified as positive with a probability of 0.9 vs 0.9999, but L2 loss doesn't strongly differentiate these cases.\n", + "\n", + "In contrast, `LogLoss` penalizes these \"confidence errors\" much more heavily. Remember, `LogLoss` is defined as:\n", + "\n", + "$$Log Loss = \\sum_{(x,y)\\in D} -y \\cdot log(y_{pred}) - (1 - y) \\cdot log(1 - y_{pred})$$\n", + "\n", + "\n", + "But first, we'll need to obtain the prediction values. We could use `LinearRegressor.predict` to obtain these.\n", + "\n", + "Given the predictions and the targets, can we calculate `LogLoss`?" + ] + }, + { + "metadata": { + "id": "dPpJUV862FYI", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Solution\n", + "\n", + "Click below to display the solution." + ] + }, + { + "metadata": { + "id": "kXFQ5uig2RoP", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 347 + }, + "outputId": "a4417629-3d0a-4615-a586-689c5774f573" + }, + "cell_type": "code", + "source": [ + "predict_validation_input_fn = lambda: my_input_fn(validation_examples, \n", + " validation_targets[\"median_house_value_is_high\"], \n", + " num_epochs=1, \n", + " shuffle=False)\n", + "\n", + "validation_predictions = linear_regressor.predict(input_fn=predict_validation_input_fn)\n", + "validation_predictions = np.array([item['predictions'][0] for item in validation_predictions])\n", + "\n", + "_ = plt.hist(validation_predictions)" + ], + "execution_count": 11, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeoAAAFKCAYAAADScRzUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAGfVJREFUeJzt3X9M1Pfhx/HX/eB2ZT2q5+5sTV23\nLG44y0CipWKgA6VVkm20FSdEzVa6aUo7bdksc2Y1aTJQi1FTE3+NajTtiJf+wbcxYpws0UDZ5iUE\nmyXaLVmcdnJXaUXAcZLP949+d99ShcNPgXsDz8df8rnP8Xl/3veuTz6fs4fDsixLAADASM5kDwAA\nAAyNUAMAYDBCDQCAwQg1AAAGI9QAABiMUAMAYDB3sgdwN5FId1KPP316qrq6epM6homIebOHebOH\nebOHebNnrOctEPAN+RhX1HfhdruSPYQJiXmzh3mzh3mzh3mzJ5nzRqgBADAYoQYAwGCEGgAAgxFq\nAAAMRqgBADAYoQYAwGCEGgAAgxFqAAAMRqgBADAYoQYAwGCEGgAAgxFqAAAMZuRvz4J5nqs9k+wh\nJFRfXZjsIQDAqOOKGgAAgxFqAAAMRqgBADAYoQYAwGCEGgAAgxFqAAAMRqgBADAYoQYAwGCEGgAA\ngxFqAAAMlvAjRPv6+lRdXa2PP/5Y//nPf/TCCy8oPT1dmzZt0sDAgAKBgHbs2CGPx6PGxkYdOXJE\nTqdTK1euVGlpqWKxmKqrq3X16lW5XC7V1NRo9uzZ43FuAABMeAmvqJubm/Xoo4/q2LFj2rVrl2pr\na7Vnzx6Vl5fr7bff1iOPPKJQKKTe3l7t3btXhw8f1tGjR3XkyBF98skneu+995SWlqZ33nlH69ev\nV11d3XicFwAAk0LCUBcXF+tnP/uZJOmjjz7SzJkz1dbWpiVLlkiSCgoK1Nraqvb2dmVkZMjn88nr\n9So7O1vhcFitra0qKiqSJOXm5iocDo/h6QAAMLmM+LdnrVq1Sv/+97+1b98+/fSnP5XH45EkzZgx\nQ5FIRNFoVH6/P76/3++/Y7vT6ZTD4VB/f3/8+QAAYGgjDvUf/vAH/e1vf9OvfvUrWZYV3/75P3/e\nvW7/vOnTU+V2u0Y6tDERCPiSenzcu4n8mk3ksScT82YP82ZPsuYtYagvXLigGTNm6KGHHtLcuXM1\nMDCgr371q7p165a8Xq+uXbumYDCoYDCoaDQaf15nZ6eysrIUDAYViUSUnp6uWCwmy7ISXk13dfV+\n+TP7EgIBnyKR7qSOAfduor5mrDd7mDd7mDd7xnrehvshIOF71H/9619VX18vSYpGo+rt7VVubq6a\nmpokSadOnVJeXp4yMzPV0dGhGzduqKenR+FwWAsWLNDixYt18uRJSZ/9w7ScnJzROCcAAKaEhFfU\nq1at0m9+8xuVl5fr1q1b+u1vf6tHH31Ur776qhoaGjRr1iyVlJQoJSVFVVVVqqiokMPhUGVlpXw+\nn4qLi9XS0qKysjJ5PB7V1taOx3kBADApOKyRvGk8zpJ9W4ZbQ3d6rvZMsoeQUH11YbKHYAvrzR7m\nzR7mzR6jb30DAIDkIdQAABiMUAMAYDBCDQCAwQg1AAAGI9QAABiMUAMAYDBCDQCAwQg1AAAGI9QA\nABiMUAMAYDBCDQCAwQg1AAAGI9QAABiMUAMAYDBCDQCAwQg1AAAGI9QAABiMUAMAYDBCDQCAwQg1\nAAAGI9QAABiMUAMAYDBCDQCAwQg1AAAGI9QAABiMUAMAYDBCDQCAwQg1AAAGI9QAABiMUAMAYDBC\nDQCAwQg1AAAGI9QAABiMUAMAYDBCDQCAwdwj2Wn79u06f/68bt++rXXr1unMmTP64IMPNG3aNElS\nRUWFvv/976uxsVFHjhyR0+nUypUrVVpaqlgspurqal29elUul0s1NTWaPXv2mJ4UAACTRcJQv//+\n+7p06ZIaGhrU1dWlp59+Wo8//rheeeUVFRQUxPfr7e3V3r17FQqFlJKSohUrVqioqEjNzc1KS0tT\nXV2dzp07p7q6Ou3atWtMTwoAgMki4a3vhQsXavfu3ZKktLQ09fX1aWBg4I792tvblZGRIZ/PJ6/X\nq+zsbIXDYbW2tqqoqEiSlJubq3A4PMqnAADA5JUw1C6XS6mpqZKkUCik/Px8uVwuHTt2TGvXrtXL\nL7+s69evKxqNyu/3x5/n9/sViUQGbXc6nXI4HOrv7x+j0wEAYHIZ0XvUknT69GmFQiHV19frwoUL\nmjZtmubOnasDBw7ozTff1Pz58wftb1nWXb/PUNs/b/r0VLndrpEObUwEAr6kHh/3biK/ZhN57MnE\nvNnDvNmTrHkbUajPnj2rffv26dChQ/L5fFq0aFH8scLCQm3dulVPPfWUotFofHtnZ6eysrIUDAYV\niUSUnp6uWCwmy7Lk8XiGPV5XV6/N0xkdgYBPkUh3UseAezdRXzPWmz3Mmz3Mmz1jPW/D/RCQ8NZ3\nd3e3tm/frv3798f/lfdLL72ky5cvS5La2to0Z84cZWZmqqOjQzdu3FBPT4/C4bAWLFigxYsX6+TJ\nk5Kk5uZm5eTkjMY5AQAwJSS8oj5x4oS6urq0cePG+LZnnnlGGzdu1H333afU1FTV1NTI6/WqqqpK\nFRUVcjgcqqyslM/nU3FxsVpaWlRWViaPx6Pa2toxPSEAACYThzWSN43HWbJvy3Br6E7P1Z5J9hAS\nqq8uTPYQbGG92cO82cO82WP0rW8AAJA8hBoAAIMRagAADEaoAQAwGKEGAMBghBoAAIMRagAADEao\nAQAwGKEGAMBghBoAAIMRagAADEaoAQAwGKEGAMBghBoAAIMRagAADEaoAQAwGKEGAMBghBoAAIMR\nagAADEaoAQAwGKEGAMBghBoAAIMRagAADEaoAQAwGKEGAMBghBoAAIMRagAADEaoAQAwGKEGAMBg\nhBoAAIMRagAADEaoAQAwGKEGAMBghBoAAIMRagAADEaoAQAwmHskO23fvl3nz5/X7du3tW7dOmVk\nZGjTpk0aGBhQIBDQjh075PF41NjYqCNHjsjpdGrlypUqLS1VLBZTdXW1rl69KpfLpZqaGs2ePXus\nzwsAgEkhYajff/99Xbp0SQ0NDerq6tLTTz+tRYsWqby8XMuXL9fOnTsVCoVUUlKivXv3KhQKKSUl\nRStWrFBRUZGam5uVlpamuro6nTt3TnV1ddq1a9d4nBsAABNewlvfCxcu1O7duyVJaWlp6uvrU1tb\nm5YsWSJJKigoUGtrq9rb25WRkSGfzyev16vs7GyFw2G1traqqKhIkpSbm6twODyGpwMAwOSSMNQu\nl0upqamSpFAopPz8fPX19cnj8UiSZsyYoUgkomg0Kr/fH3+e3++/Y7vT6ZTD4VB/f/9YnAsAAJPO\niN6jlqTTp08rFAqpvr5eTz75ZHy7ZVl33f9et3/e9OmpcrtdIx3amAgEfEk9Pu7dRH7NJvLYk4l5\ns4d5sydZ8zaiUJ89e1b79u3ToUOH5PP5lJqaqlu3bsnr9eratWsKBoMKBoOKRqPx53R2diorK0vB\nYFCRSETp6emKxWKyLCt+NT6Urq7eL3dWX1Ig4FMk0p3UMeDeTdTXjPVmD/NmD/Nmz1jP23A/BCS8\n9d3d3a3t27dr//79mjZtmqTP3mtuamqSJJ06dUp5eXnKzMxUR0eHbty4oZ6eHoXDYS1YsECLFy/W\nyZMnJUnNzc3KyckZjXMCAGBKSHhFfeLECXV1dWnjxo3xbbW1tdqyZYsaGho0a9YslZSUKCUlRVVV\nVaqoqJDD4VBlZaV8Pp+Ki4vV0tKisrIyeTwe1dbWjukJAQAwmTiskbxpPM6SfVuGW0N3eq72TLKH\nkFB9dWGyh2AL680e5s0e5s0eo299AwCA5CHUAAAYjFADAGAwQg0AgMEINQAABiPUAAAYjFADAGAw\nQg0AgMEINQAABiPUAAAYjFADAGAwQg0AgMEINQAABiPUAAAYjFADAGAwQg0AgMEINQAABiPUAAAY\njFADAGAwQg0AgMEINQAABiPUAAAYjFADAGAwQg0AgMEINQAABiPUAAAYjFADAGAwQg0AgMEINQAA\nBiPUAAAYjFADAGAwQg0AgMEINQAABiPUAAAYjFADAGAwQg0AgMFGFOqLFy9q6dKlOnbsmCSpurpa\nP/jBD7RmzRqtWbNGf/rTnyRJjY2NevbZZ1VaWqrjx49LkmKxmKqqqlRWVqbVq1fr8uXLY3MmAABM\nQu5EO/T29ur111/XokWLBm1/5ZVXVFBQMGi/vXv3KhQKKSUlRStWrFBRUZGam5uVlpamuro6nTt3\nTnV1ddq1a9fonwkAAJNQwitqj8ejgwcPKhgMDrtfe3u7MjIy5PP55PV6lZ2drXA4rNbWVhUVFUmS\ncnNzFQ6HR2fkAABMAQlD7Xa75fV679h+7NgxrV27Vi+//LKuX7+uaDQqv98ff9zv9ysSiQza7nQ6\n5XA41N/fP4qnAADA5JXw1vfd/OhHP9K0adM0d+5cHThwQG+++abmz58/aB/Lsu763KG2f9706aly\nu112hjZqAgFfUo+PezeRX7OJPPZkYt7sYd7sSda82Qr159+vLiws1NatW/XUU08pGo3Gt3d2dior\nK0vBYFCRSETp6emKxWKyLEsej2fY79/V1WtnWKMmEPApEulO6hhw7ybqa8Z6s4d5s4d5s2es5224\nHwJshfqll17Spk2bNHv2bLW1tWnOnDnKzMzUli1bdOPGDblcLoXDYW3evFk3b97UyZMnlZeXp+bm\nZuXk5Ng+EWA4z9WeSfYQhlVfXZjsIQCYgBKG+sKFC9q2bZuuXLkit9utpqYmrV69Whs3btR9992n\n1NRU1dTUyOv1qqqqShUVFXI4HKqsrJTP51NxcbFaWlpUVlYmj8ej2tra8TgvAAAmBYc1kjeNx1my\nb8twa+hOpl+tTgRDXVGz3uxh3uxh3uxJ5q1vPpkMAACDEWoAAAxGqAEAMBihBgDAYIQaAACDEWoA\nAAxGqAEAMBihBgDAYIQaAACDEWoAAAxGqAEAMBihBgDAYIQaAACDEWoAAAxGqAEAMBihBgDAYIQa\nAACDEWoAAAxGqAEAMBihBgDAYIQaAACDEWoAAAxGqAEAMBihBgDAYIQaAACDEWoAAAxGqAEAMBih\nBgDAYIQaAACDEWoAAAxGqAEAMBihBgDAYIQaAACDEWoAAAxGqAEAMBihBgDAYCMK9cWLF7V06VId\nO3ZMkvTRRx9pzZo1Ki8v14YNG9Tf3y9Jamxs1LPPPqvS0lIdP35ckhSLxVRVVaWysjKtXr1aly9f\nHqNTAQBg8kkY6t7eXr3++utatGhRfNuePXtUXl6ut99+W4888ohCoZB6e3u1d+9eHT58WEePHtWR\nI0f0ySef6L333lNaWpreeecdrV+/XnV1dWN6QgAATCYJQ+3xeHTw4EEFg8H4tra2Ni1ZskSSVFBQ\noNbWVrW3tysjI0M+n09er1fZ2dkKh8NqbW1VUVGRJCk3N1fhcHiMTgUAgMknYajdbre8Xu+gbX19\nffJ4PJKkGTNmKBKJKBqNyu/3x/fx+/13bHc6nXI4HPFb5QAAYHjuL/sNLMsale2fN316qtxu15ca\n15cVCPiSenxMPsOtKdabPcybPcybPcmaN1uhTk1N1a1bt+T1enXt2jUFg0EFg0FFo9H4Pp2dncrK\nylIwGFQkElF6erpisZgsy4pfjQ+lq6vXzrBGTSDgUyTSndQxYPIZak2x3uxh3uxh3uwZ63kb7ocA\nW/97Vm5urpqamiRJp06dUl5enjIzM9XR0aEbN26op6dH4XBYCxYs0OLFi3Xy5ElJUnNzs3Jycuwc\nEgCAKSnhFfWFCxe0bds2XblyRW63W01NTXrjjTdUXV2thoYGzZo1SyUlJUpJSVFVVZUqKirkcDhU\nWVkpn8+n4uJitbS0qKysTB6PR7W1teNxXgAATAoOayRvGo+zZN+W4dbQnZ6rPZPsIUx49dWFd93O\nerOHebOHebNnwt36BgAA44NQAwBgMEINAIDBCDUAAAYj1AAAGIxQAwBgMEINAIDBCDUAAAYj1AAA\nGIxQAwBgMEINAIDBCDUAAAYj1AAAGIxQAwBgMEINAIDBCDUAAAYj1AAAGIxQAwBgMEINAIDBCDUA\nAAYj1AAAGIxQAwBgMEINAIDBCDUAAAYj1AAAGIxQAwBgMEINAIDBCDUAAAYj1AAAGIxQAwBgMEIN\nAIDBCDUAAAYj1AAAGIxQAwBgMEINAIDBCDUAAAZz23lSW1ubNmzYoDlz5kiSvv3tb+v555/Xpk2b\nNDAwoEAgoB07dsjj8aixsVFHjhyR0+nUypUrVVpaOqonAADAZGYr1JL02GOPac+ePfGvf/3rX6u8\nvFzLly/Xzp07FQqFVFJSor179yoUCiklJUUrVqxQUVGRpk2bNiqDBwBgshu1W99tbW1asmSJJKmg\noECtra1qb29XRkaGfD6fvF6vsrOzFQ6HR+uQAABMeravqD/88EOtX79en376qV588UX19fXJ4/FI\nkmbMmKFIJKJoNCq/3x9/jt/vVyQS+fKjBgBgirAV6m984xt68cUXtXz5cl2+fFlr167VwMBA/HHL\nsu76vKG2f9H06alyu112hjZqAgFfUo+PyWe4NcV6s4d5s4d5sydZ82Yr1DNnzlRxcbEk6etf/7q+\n9rWvqaOjQ7du3ZLX69W1a9cUDAYVDAYVjUbjz+vs7FRWVlbC79/V1WtnWKMmEPApEulO6hgw+Qy1\nplhv9jBv9jBv9oz1vA33Q4Ct96gbGxv1+9//XpIUiUT08ccf65lnnlFTU5Mk6dSpU8rLy1NmZqY6\nOjp048YN9fT0KBwOa8GCBXYOCQDAlGTrirqwsFC//OUv9cc//lGxWExbt27V3Llz9eqrr6qhoUGz\nZs1SSUmJUlJSVFVVpYqKCjkcDlVWVsrn45YLAAAjZSvU999/v/bt23fH9rfeeuuObcuWLdOyZcvs\nHAYAgCmPTyYDAMBghBoAAIMRagAADEaoAQAwGKEGAMBghBoAAIMRagAADEaoAQAwGKEGAMBghBoA\nAIMRagAADEaoAQAwGKEGAMBghBoAAIMRagAADEaoAQAwGKEGAMBghBoAAIMRagAADOZO9gCAqeK5\n2jPJHkJC9dWFyR4CgC/gihoAAIMRagAADEaoAQAwGKEGAMBghBoAAIMRagAADEaoAQAwGKEGAMBg\nhBoAAIMRagAADEaoAQAwGKEGAMBghBoAAIMRagAADMavuQQQZ/qv4uTXcGIq4ooaAACDjcsV9e9+\n9zu1t7fL4XBo8+bN+t73vjcehwUAYMIb81D/+c9/1j//+U81NDTo73//uzZv3qyGhoaxPiyAScj0\nW/MSt+cx+sY81K2trVq6dKkk6Vvf+pY+/fRT3bx5U/fff/9YH3pCmQh/AQEAxt+YhzoajWrevHnx\nr/1+vyKRCKEGMCnxQ/fUMJ53Tsb9X31blpVwn0DANw4jMWsM/1P3o3E9HgBgYhjzf/UdDAYVjUbj\nX3d2dioQCIz1YQEAmBTGPNSLFy9WU1OTJOmDDz5QMBjktjcAACM05re+s7OzNW/ePK1atUoOh0Ov\nvfbaWB8SAIBJw2GN5E1jAACQFHwyGQAABiPUAAAYjF/KISkWi6m6ulpXr16Vy+VSTU2NZs+ePWif\nefPmKTs7O/714cOH5XK5xnuoxhjuY2FbWlq0c+dOuVwu5efnq7KyMokjNctw81ZYWKgHH3wwvq7e\neOMNzZw5M1lDNcrFixf1wgsv6Cc/+YlWr1496DHW29CGmzfW29C2b9+u8+fP6/bt21q3bp2efPLJ\n+GNJWW8WrHfffdfaunWrZVmWdfbsWWvDhg137PPYY4+N97CM1dbWZv385z+3LMuyPvzwQ2vlypWD\nHl++fLl19epVa2BgwCorK7MuXbqUjGEaJ9G8FRQUWDdv3kzG0IzW09NjrV692tqyZYt19OjROx5n\nvd1donljvd1da2ur9fzzz1uWZVnXr1+3nnjiiUGPJ2O9cetbn33MaVFRkSQpNzdX4XA4ySMy21Af\nCytJly9f1gMPPKCHHnpITqdTTzzxhFpbW5M5XGMMN28Ymsfj0cGDBxUMBu94jPU2tOHmDUNbuHCh\ndu/eLUlKS0tTX1+fBgYGJCVvvRFqffYxp36/X5LkdDrlcDjU398/aJ/+/n5VVVVp1apVeuutt5Ix\nTGNEo1FNnz49/vV/PxZWkiKRSHwuv/jYVDfcvP3Xa6+9prKyMr3xxhsj+hS/qcDtdsvr9d71Mdbb\n0Iabt/9ivd3J5XIpNTVVkhQKhZSfnx9/eyBZ623KvUd9/PhxHT9+fNC29vb2QV/fbcFu2rRJP/zh\nD+VwOLR69WotWLBAGRkZYzrWiYL/wO354rz94he/UF5enh544AFVVlaqqalJy5YtS9LoMNmx3oZ3\n+vRphUIh1dfXJ3soUy/UpaWlKi0tHbSturpakUhE6enpisVisixLHo9n0D5lZWXxPz/++OO6ePHi\nlA31cB8L+8XHrl27xq23/5Po43RLSkrif87Pz9fFixf5izMB1pt9rLehnT17Vvv27dOhQ4fk8/3/\n731I1nrj1rc++5jTkydPSpKam5uVk5Mz6PF//OMfqqqqkmVZun37tsLhsObMmZOMoRphuI+Fffjh\nh3Xz5k3961//0u3bt9Xc3KzFixcnc7jGGG7euru7VVFREX/L5S9/+cuUXmMjxXqzh/U2tO7ubm3f\nvl379+/XtGnTBj2WrPU25a6o76a4uFgtLS0qKyuTx+NRbW2tJOnAgQNauHCh5s+frwcffFArVqyQ\n0+lUYWHhoP+tZqq528fCvvvuu/L5fCoqKtLWrVtVVVUl6bO5/eY3v5nkEZsh0bzl5+frxz/+sb7y\nla/ou9/9Llc3/+fChQvatm2brly5IrfbraamJhUWFurhhx9mvQ0j0byx3u7uxIkT6urq0saNG+Pb\ncnJy9J3vfCdp642PEAUAwGDc+gYAwGCEGgAAgxFqAAAMRqgBADAYoQYAwGCEGgAAgxFqAAAMRqgB\nADDY/wJYpPykm8lW1gAAAABJRU5ErkJggg==\n", + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "metadata": { + "id": "rYpy336F9wBg", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Task 2: Train a Logistic Regression Model and Calculate LogLoss on the Validation Set\n", + "\n", + "To use logistic regression, simply use [LinearClassifier](https://www.tensorflow.org/api_docs/python/tf/estimator/LinearClassifier) instead of `LinearRegressor`. Complete the code below.\n", + "\n", + "**NOTE**: When running `train()` and `predict()` on a `LinearClassifier` model, you can access the real-valued predicted probabilities via the `\"probabilities\"` key in the returned dict—e.g., `predictions[\"probabilities\"]`. Sklearn's [log_loss](http://scikit-learn.org/stable/modules/generated/sklearn.metrics.log_loss.html) function is handy for calculating LogLoss using these probabilities.\n" + ] + }, + { + "metadata": { + "id": "JElcb--E9wBm", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def train_linear_classifier_model(\n", + " learning_rate,\n", + " steps,\n", + " batch_size,\n", + " training_examples,\n", + " training_targets,\n", + " validation_examples,\n", + " validation_targets):\n", + " \"\"\"Trains a linear classification model.\n", + " \n", + " In addition to training, this function also prints training progress information,\n", + " as well as a plot of the training and validation loss over time.\n", + " \n", + " Args:\n", + " learning_rate: A `float`, the learning rate.\n", + " steps: A non-zero `int`, the total number of training steps. A training step\n", + " consists of a forward and backward pass using a single batch.\n", + " batch_size: A non-zero `int`, the batch size.\n", + " training_examples: A `DataFrame` containing one or more columns from\n", + " `california_housing_dataframe` to use as input features for training.\n", + " training_targets: A `DataFrame` containing exactly one column from\n", + " `california_housing_dataframe` to use as target for training.\n", + " validation_examples: A `DataFrame` containing one or more columns from\n", + " `california_housing_dataframe` to use as input features for validation.\n", + " validation_targets: A `DataFrame` containing exactly one column from\n", + " `california_housing_dataframe` to use as target for validation.\n", + " \n", + " Returns:\n", + " A `LinearClassifier` object trained on the training data.\n", + " \"\"\"\n", + "\n", + " periods = 10\n", + " steps_per_period = steps / periods\n", + " \n", + " # Create a linear classifier object.\n", + " my_optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n", + " my_optimizer = tf.contrib.estimator.clip_gradients_by_norm(my_optimizer, 5.0)\n", + " linear_classifier = linear_classifier = tf.estimator.LinearClassifier(\n", + " feature_columns=construct_feature_columns(training_examples),\n", + " optimizer=my_optimizer\n", + " )\n", + " \n", + " # Create input functions.\n", + " training_input_fn = lambda: my_input_fn(training_examples, \n", + " training_targets[\"median_house_value_is_high\"], \n", + " batch_size=batch_size)\n", + " predict_training_input_fn = lambda: my_input_fn(training_examples, \n", + " training_targets[\"median_house_value_is_high\"], \n", + " num_epochs=1, \n", + " shuffle=False)\n", + " predict_validation_input_fn = lambda: my_input_fn(validation_examples, \n", + " validation_targets[\"median_house_value_is_high\"], \n", + " num_epochs=1, \n", + " shuffle=False)\n", + " \n", + " # Train the model, but do so inside a loop so that we can periodically assess\n", + " # loss metrics.\n", + " print(\"Training model...\")\n", + " print(\"LogLoss (on training data):\")\n", + " training_log_losses = []\n", + " validation_log_losses = []\n", + " for period in range (0, periods):\n", + " # Train the model, starting from the prior state.\n", + " linear_classifier.train(\n", + " input_fn=training_input_fn,\n", + " steps=steps_per_period\n", + " )\n", + " # Take a break and compute predictions. \n", + " training_probabilities = linear_classifier.predict(input_fn=predict_training_input_fn)\n", + " training_probabilities = np.array([item['probabilities'] for item in training_probabilities])\n", + " \n", + " validation_probabilities = linear_classifier.predict(input_fn=predict_validation_input_fn)\n", + " validation_probabilities = np.array([item['probabilities'] for item in validation_probabilities])\n", + " \n", + " training_log_loss = metrics.log_loss(training_targets, training_probabilities)\n", + " validation_log_loss = metrics.log_loss(validation_targets, validation_probabilities)\n", + " # Occasionally print the current loss.\n", + " print(\" period %02d : %0.2f\" % (period, training_log_loss))\n", + " # Add the loss metrics from this period to our list.\n", + " training_log_losses.append(training_log_loss)\n", + " validation_log_losses.append(validation_log_loss)\n", + " print(\"Model training finished.\")\n", + " \n", + " # Output a graph of loss metrics over periods.\n", + " plt.ylabel(\"LogLoss\")\n", + " plt.xlabel(\"Periods\")\n", + " plt.title(\"LogLoss vs. Periods\")\n", + " plt.tight_layout()\n", + " plt.plot(training_log_losses, label=\"training\")\n", + " plt.plot(validation_log_losses, label=\"validation\")\n", + " plt.legend()\n", + "\n", + " return linear_classifier" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "VM0wmnFUIYH9", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 2581 + }, + "outputId": "48f36cae-5763-477f-8221-da3a877a0bec" + }, + "cell_type": "code", + "source": [ + "linear_classifier = train_linear_classifier_model(\n", + " learning_rate=0.000005,\n", + " steps=500,\n", + " batch_size=20,\n", + " training_examples=training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 15, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Training model...\n", + "LogLoss (on training data):\n" + ], + "name": "stdout" + }, + { + "output_type": "error", + "ename": "KeyboardInterrupt", + "evalue": "ignored", + "traceback": [ + "\u001b[0;31m\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0mtraining_targets\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtraining_targets\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0mvalidation_examples\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidation_examples\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m validation_targets=validation_targets)\n\u001b[0m", + "\u001b[0;32m\u001b[0m in \u001b[0;36mtrain_linear_classifier_model\u001b[0;34m(learning_rate, steps, batch_size, training_examples, training_targets, validation_examples, validation_targets)\u001b[0m\n\u001b[1;32m 68\u001b[0m \u001b[0;31m# Take a break and compute predictions.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 69\u001b[0m \u001b[0mtraining_probabilities\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlinear_classifier\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_fn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpredict_training_input_fn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 70\u001b[0;31m \u001b[0mtraining_probabilities\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'probabilities'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mitem\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtraining_probabilities\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 71\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 72\u001b[0m \u001b[0mvalidation_probabilities\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlinear_classifier\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_fn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpredict_validation_input_fn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/estimator/estimator.pyc\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, input_fn, predict_keys, hooks, checkpoint_path, yield_single_examples)\u001b[0m\n\u001b[1;32m 593\u001b[0m hooks=all_hooks) as mon_sess:\n\u001b[1;32m 594\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mmon_sess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_stop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 595\u001b[0;31m \u001b[0mpreds_evaluated\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmon_sess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpredictions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 596\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0myield_single_examples\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 597\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0mpreds_evaluated\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/monitored_session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 669\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 670\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 671\u001b[0;31m run_metadata=run_metadata)\n\u001b[0m\u001b[1;32m 672\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 673\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mrun_step_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstep_fn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/monitored_session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 1154\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1155\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1156\u001b[0;31m run_metadata=run_metadata)\n\u001b[0m\u001b[1;32m 1157\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0m_PREEMPTION_ERRORS\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1158\u001b[0m logging.info('An error was raised. This may be due to a preemption in '\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/monitored_session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1238\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1239\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1240\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1241\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0m_PREEMPTION_ERRORS\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1242\u001b[0m \u001b[0;32mraise\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/monitored_session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 1310\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1311\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1312\u001b[0;31m run_metadata=run_metadata)\n\u001b[0m\u001b[1;32m 1313\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1314\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_hooks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/monitored_session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1074\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1075\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1076\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1077\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1078\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mrun_step_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstep_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mraw_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun_with_hooks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 927\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 928\u001b[0m result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 929\u001b[0;31m run_metadata_ptr)\n\u001b[0m\u001b[1;32m 930\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 931\u001b[0m \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 1135\u001b[0m \u001b[0;31m# Create a fetch handler to take care of the structure of fetches.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1136\u001b[0m fetch_handler = _FetchHandler(\n\u001b[0;32m-> 1137\u001b[0;31m self._graph, fetches, feed_dict_tensor, feed_handles=feed_handles)\n\u001b[0m\u001b[1;32m 1138\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1139\u001b[0m \u001b[0;31m# Run request and get response.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, graph, fetches, feeds, feed_handles)\u001b[0m\n\u001b[1;32m 469\u001b[0m \"\"\"\n\u001b[1;32m 470\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mgraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_default\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 471\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetch_mapper\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_FetchMapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfor_fetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 472\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetches\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 473\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_targets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mfor_fetch\u001b[0;34m(fetch)\u001b[0m\n\u001b[1;32m 261\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0m_ListFetchMapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 262\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMapping\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 263\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0m_DictFetchMapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 264\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0m_is_attrs_instance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 265\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0m_AttrsFetchMapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, fetches)\u001b[0m\n\u001b[1;32m 401\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_keys\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 402\u001b[0m self._mappers = [\n\u001b[0;32m--> 403\u001b[0;31m \u001b[0m_FetchMapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfor_fetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mfetch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 404\u001b[0m ]\n\u001b[1;32m 405\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_unique_fetches\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_value_indices\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_uniquify_fetches\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mappers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mfor_fetch\u001b[0;34m(fetch)\u001b[0m\n\u001b[1;32m 261\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0m_ListFetchMapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 262\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMapping\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 263\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0m_DictFetchMapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 264\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0m_is_attrs_instance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 265\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0m_AttrsFetchMapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, fetches)\u001b[0m\n\u001b[1;32m 401\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_keys\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 402\u001b[0m self._mappers = [\n\u001b[0;32m--> 403\u001b[0;31m \u001b[0m_FetchMapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfor_fetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mfetch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 404\u001b[0m ]\n\u001b[1;32m 405\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_unique_fetches\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_value_indices\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_uniquify_fetches\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mappers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mfor_fetch\u001b[0;34m(fetch)\u001b[0m\n\u001b[1;32m 269\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensor_type\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 270\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontraction_fn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfetch_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 271\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0m_ElementFetchMapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetches\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontraction_fn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 272\u001b[0m \u001b[0;31m# Did not find anything.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 273\u001b[0m raise TypeError('Fetch argument %r has invalid type %r' % (fetch,\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, fetches, contraction_fn)\u001b[0m\n\u001b[1;32m 298\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 299\u001b[0m self._unique_fetches.append(ops.get_default_graph().as_graph_element(\n\u001b[0;32m--> 300\u001b[0;31m fetch, allow_tensor=True, allow_operation=True))\n\u001b[0m\u001b[1;32m 301\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 302\u001b[0m raise TypeError('Fetch argument %r has invalid type %r, '\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.pyc\u001b[0m in \u001b[0;36mas_graph_element\u001b[0;34m(self, obj, allow_tensor, allow_operation)\u001b[0m\n\u001b[1;32m 3485\u001b[0m \"\"\"\n\u001b[1;32m 3486\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_finalized\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3487\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_as_graph_element_locked\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mallow_tensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mallow_operation\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 3488\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3489\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.pyc\u001b[0m in \u001b[0;36m_as_graph_element_locked\u001b[0;34m(self, obj, allow_tensor, allow_operation)\u001b[0m\n\u001b[1;32m 3566\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mallow_tensor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3567\u001b[0m \u001b[0;31m# Actually obj is just the object it's referring to.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3568\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgraph\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 3569\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Tensor %s is not an element of this graph.\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3570\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.pyc\u001b[0m in \u001b[0;36mgraph\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 347\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mgraph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 348\u001b[0m \u001b[0;34m\"\"\"The `Graph` that contains this tensor.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 349\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_op\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgraph\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 350\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 351\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.pyc\u001b[0m in \u001b[0;36mgraph\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 2232\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mc_api\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_OperationOpType\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_c_op\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2233\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2234\u001b[0;31m \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2235\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mgraph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2236\u001b[0m \u001b[0;34m\"\"\"The `Graph` that contains this operation.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ] + }, + { + "metadata": { + "id": "i2e3TlyL57Qs", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Solution\n", + "\n", + "Click below to see the solution.\n", + "\n" + ] + }, + { + "metadata": { + "id": "5YxXd2hn6MuF", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def train_linear_classifier_model(\n", + " learning_rate,\n", + " steps,\n", + " batch_size,\n", + " training_examples,\n", + " training_targets,\n", + " validation_examples,\n", + " validation_targets):\n", + " \"\"\"Trains a linear classification model.\n", + " \n", + " In addition to training, this function also prints training progress information,\n", + " as well as a plot of the training and validation loss over time.\n", + " \n", + " Args:\n", + " learning_rate: A `float`, the learning rate.\n", + " steps: A non-zero `int`, the total number of training steps. A training step\n", + " consists of a forward and backward pass using a single batch.\n", + " batch_size: A non-zero `int`, the batch size.\n", + " training_examples: A `DataFrame` containing one or more columns from\n", + " `california_housing_dataframe` to use as input features for training.\n", + " training_targets: A `DataFrame` containing exactly one column from\n", + " `california_housing_dataframe` to use as target for training.\n", + " validation_examples: A `DataFrame` containing one or more columns from\n", + " `california_housing_dataframe` to use as input features for validation.\n", + " validation_targets: A `DataFrame` containing exactly one column from\n", + " `california_housing_dataframe` to use as target for validation.\n", + " \n", + " Returns:\n", + " A `LinearClassifier` object trained on the training data.\n", + " \"\"\"\n", + "\n", + " periods = 10\n", + " steps_per_period = steps / periods\n", + " \n", + " # Create a linear classifier object.\n", + " my_optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n", + " my_optimizer = tf.contrib.estimator.clip_gradients_by_norm(my_optimizer, 5.0) \n", + " linear_classifier = tf.estimator.LinearClassifier(\n", + " feature_columns=construct_feature_columns(training_examples),\n", + " optimizer=my_optimizer\n", + " )\n", + " \n", + " # Create input functions.\n", + " training_input_fn = lambda: my_input_fn(training_examples, \n", + " training_targets[\"median_house_value_is_high\"], \n", + " batch_size=batch_size)\n", + " predict_training_input_fn = lambda: my_input_fn(training_examples, \n", + " training_targets[\"median_house_value_is_high\"], \n", + " num_epochs=1, \n", + " shuffle=False)\n", + " predict_validation_input_fn = lambda: my_input_fn(validation_examples, \n", + " validation_targets[\"median_house_value_is_high\"], \n", + " num_epochs=1, \n", + " shuffle=False)\n", + " \n", + " # Train the model, but do so inside a loop so that we can periodically assess\n", + " # loss metrics.\n", + " print(\"Training model...\")\n", + " print(\"LogLoss (on training data):\")\n", + " training_log_losses = []\n", + " validation_log_losses = []\n", + " for period in range (0, periods):\n", + " # Train the model, starting from the prior state.\n", + " linear_classifier.train(\n", + " input_fn=training_input_fn,\n", + " steps=steps_per_period\n", + " )\n", + " # Take a break and compute predictions. \n", + " training_probabilities = linear_classifier.predict(input_fn=predict_training_input_fn)\n", + " training_probabilities = np.array([item['probabilities'] for item in training_probabilities])\n", + " \n", + " validation_probabilities = linear_classifier.predict(input_fn=predict_validation_input_fn)\n", + " validation_probabilities = np.array([item['probabilities'] for item in validation_probabilities])\n", + " \n", + " training_log_loss = metrics.log_loss(training_targets, training_probabilities)\n", + " validation_log_loss = metrics.log_loss(validation_targets, validation_probabilities)\n", + " # Occasionally print the current loss.\n", + " print(\" period %02d : %0.2f\" % (period, training_log_loss))\n", + " # Add the loss metrics from this period to our list.\n", + " training_log_losses.append(training_log_loss)\n", + " validation_log_losses.append(validation_log_loss)\n", + " print(\"Model training finished.\")\n", + " \n", + " # Output a graph of loss metrics over periods.\n", + " plt.ylabel(\"LogLoss\")\n", + " plt.xlabel(\"Periods\")\n", + " plt.title(\"LogLoss vs. Periods\")\n", + " plt.tight_layout()\n", + " plt.plot(training_log_losses, label=\"training\")\n", + " plt.plot(validation_log_losses, label=\"validation\")\n", + " plt.legend()\n", + "\n", + " return linear_classifier" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "UPM_T1FXsTaL", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "linear_classifier = train_linear_classifier_model(\n", + " learning_rate=0.000005,\n", + " steps=500,\n", + " batch_size=20,\n", + " training_examples=training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "i-Xo83_aR6s_", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Task 3: Calculate Accuracy and plot a ROC Curve for the Validation Set\n", + "\n", + "A few of the metrics useful for classification are the model [accuracy](https://en.wikipedia.org/wiki/Accuracy_and_precision#In_binary_classification), the [ROC curve](https://en.wikipedia.org/wiki/Receiver_operating_characteristic) and the area under the ROC curve (AUC). We'll examine these metrics.\n", + "\n", + "`LinearClassifier.evaluate` calculates useful metrics like accuracy and AUC." + ] + }, + { + "metadata": { + "id": "DKSQ87VVIYIA", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "evaluation_metrics = linear_classifier.evaluate(input_fn=predict_validation_input_fn)\n", + "\n", + "print(\"AUC on the validation set: %0.2f\" % evaluation_metrics['auc'])\n", + "print(\"Accuracy on the validation set: %0.2f\" % evaluation_metrics['accuracy'])" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "47xGS2uNIYIE", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "You may use class probabilities, such as those calculated by `LinearClassifier.predict`,\n", + "and Sklearn's [roc_curve](http://scikit-learn.org/stable/modules/model_evaluation.html#roc-metrics) to\n", + "obtain the true positive and false positive rates needed to plot a ROC curve." + ] + }, + { + "metadata": { + "id": "xaU7ttj8IYIF", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "validation_probabilities = linear_classifier.predict(input_fn=predict_validation_input_fn)\n", + "# Get just the probabilities for the positive class.\n", + "validation_probabilities = np.array([item['probabilities'][1] for item in validation_probabilities])\n", + "\n", + "false_positive_rate, true_positive_rate, thresholds = metrics.roc_curve(\n", + " validation_targets, validation_probabilities)\n", + "plt.plot(false_positive_rate, true_positive_rate, label=\"our model\")\n", + "plt.plot([0, 1], [0, 1], label=\"random classifier\")\n", + "_ = plt.legend(loc=2)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "PIdhwfgzIYII", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "**See if you can tune the learning settings of the model trained at Task 2 to improve AUC.**\n", + "\n", + "Often times, certain metrics improve at the detriment of others, and you'll need to find the settings that achieve a good compromise.\n", + "\n", + "**Verify if all metrics improve at the same time.**" + ] + }, + { + "metadata": { + "id": "XKIqjsqcCaxO", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# TUNE THE SETTINGS BELOW TO IMPROVE AUC\n", + "linear_classifier = train_linear_classifier_model(\n", + " learning_rate=0.000003,\n", + " steps=20000,\n", + " batch_size=500,\n", + " training_examples=training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=validation_examples,\n", + " validation_targets=validation_targets)\n", + "\n", + "evaluation_metrics = linear_classifier.evaluate(input_fn=predict_validation_input_fn)\n", + "\n", + "print(\"AUC on the validation set: %0.2f\" % evaluation_metrics['auc'])\n", + "print(\"Accuracy on the validation set: %0.2f\" % evaluation_metrics['accuracy'])" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "wCugvl0JdWYL", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Solution\n", + "\n", + "Click below for a possible solution." + ] + }, + { + "metadata": { + "id": "VHosS1g2aetf", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "One possible solution that works is to just train for longer, as long as we don't overfit. \n", + "\n", + "We can do this by increasing the number the steps, the batch size, or both.\n", + "\n", + "All metrics improve at the same time, so our loss metric is a good proxy\n", + "for both AUC and accuracy.\n", + "\n", + "Notice how it takes many, many more iterations just to squeeze a few more \n", + "units of AUC. This commonly happens. But often even this small gain is worth \n", + "the costs." + ] + }, + { + "metadata": { + "id": "dWgTEYMddaA-", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "linear_classifier = train_linear_classifier_model(\n", + " learning_rate=0.000003,\n", + " steps=20000,\n", + " batch_size=500,\n", + " training_examples=training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=validation_examples,\n", + " validation_targets=validation_targets)\n", + "\n", + "evaluation_metrics = linear_classifier.evaluate(input_fn=predict_validation_input_fn)\n", + "\n", + "print(\"AUC on the validation set: %0.2f\" % evaluation_metrics['auc'])\n", + "print(\"Accuracy on the validation set: %0.2f\" % evaluation_metrics['accuracy'])" + ], + "execution_count": 0, + "outputs": [] + } + ] +} \ No newline at end of file diff --git a/multi_class_classification_of_handwritten_digits.ipynb b/multi_class_classification_of_handwritten_digits.ipynb new file mode 100644 index 0000000..1a336d4 --- /dev/null +++ b/multi_class_classification_of_handwritten_digits.ipynb @@ -0,0 +1,1120 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "multi-class_classification_of_handwritten_digits.ipynb", + "version": "0.3.2", + "provenance": [], + "collapsed_sections": [ + "JndnmDMp66FL", + "266KQvZoMxMv", + "6sfw3LH0Oycm" + ], + "include_colab_link": true + }, + "kernelspec": { + "name": "python2", + "display_name": "Python 2" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "metadata": { + "id": "JndnmDMp66FL", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "#### Copyright 2017 Google LLC." + ] + }, + { + "metadata": { + "id": "hMqWDc_m6rUC", + "colab_type": "code", + "cellView": "both", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "mPa95uXvcpcn", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Classifying Handwritten Digits with Neural Networks" + ] + }, + { + "metadata": { + "id": "Fdpn8b90u8Tp", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "![img](https://www.tensorflow.org/versions/r0.11/images/MNIST.png)" + ] + }, + { + "metadata": { + "id": "c7HLCm66Cs2p", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "**Learning Objectives:**\n", + " * Train both a linear model and a neural network to classify handwritten digits from the classic [MNIST](http://yann.lecun.com/exdb/mnist/) data set\n", + " * Compare the performance of the linear and neural network classification models\n", + " * Visualize the weights of a neural-network hidden layer" + ] + }, + { + "metadata": { + "id": "HSEh-gNdu8T0", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Our goal is to map each input image to the correct numeric digit. We will create a NN with a few hidden layers and a Softmax layer at the top to select the winning class." + ] + }, + { + "metadata": { + "id": "2NMdE1b-7UIH", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Setup\n", + "\n", + "First, let's download the data set, import TensorFlow and other utilities, and load the data into a *pandas* `DataFrame`. Note that this data is a sample of the original MNIST training data; we've taken 20000 rows at random." + ] + }, + { + "metadata": { + "id": "4LJ4SD8BWHeh", + "colab_type": "code", + "cellView": "both", + "colab": {} + }, + "cell_type": "code", + "source": [ + "from __future__ import print_function\n", + "\n", + "import glob\n", + "import math\n", + "import os\n", + "\n", + "from IPython import display\n", + "from matplotlib import cm\n", + "from matplotlib import gridspec\n", + "from matplotlib import pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "import seaborn as sns\n", + "from sklearn import metrics\n", + "import tensorflow as tf\n", + "from tensorflow.python.data import Dataset\n", + "\n", + "tf.logging.set_verbosity(tf.logging.ERROR)\n", + "pd.options.display.max_rows = 10\n", + "pd.options.display.float_format = '{:.1f}'.format\n", + "\n", + "mnist_dataframe = pd.read_csv(\n", + " \"https://download.mlcc.google.com/mledu-datasets/mnist_train_small.csv\",\n", + " sep=\",\",\n", + " header=None)\n", + "\n", + "# Use just the first 10,000 records for training/validation.\n", + "mnist_dataframe = mnist_dataframe.head(10000)\n", + "\n", + "mnist_dataframe = mnist_dataframe.reindex(np.random.permutation(mnist_dataframe.index))\n", + "mnist_dataframe.head()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "kg0-25p2mOi0", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Each row represents one labeled example. Column 0 represents the label that a human rater has assigned for one handwritten digit. For example, if Column 0 contains '6', then a human rater interpreted the handwritten character as the digit '6'. The ten digits 0-9 are each represented, with a unique class label for each possible digit. Thus, this is a multi-class classification problem with 10 classes." + ] + }, + { + "metadata": { + "id": "PQ7vuOwRCsZ1", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "![img](https://www.tensorflow.org/versions/r0.11/images/MNIST-Matrix.png)" + ] + }, + { + "metadata": { + "id": "dghlqJPIu8UM", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Columns 1 through 784 contain the feature values, one per pixel for the 28×28=784 pixel values. The pixel values are on a gray scale in which 0 represents white, 255 represents black, and values between 0 and 255 represent shades of gray. Most of the pixel values are 0; you may want to take a minute to confirm that they aren't all 0. For example, adjust the following text block to print out the values in column 72." + ] + }, + { + "metadata": { + "id": "2ZkrL5MCqiJI", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "mnist_dataframe.loc[:, 72:72]" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "vLNg2VxqhUZ", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Now, let's parse out the labels and features and look at a few examples. Note the use of `loc` which allows us to pull out columns based on original location, since we don't have a header row in this data set." + ] + }, + { + "metadata": { + "id": "JfFWWvMWDFrR", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def parse_labels_and_features(dataset):\n", + " \"\"\"Extracts labels and features.\n", + " \n", + " This is a good place to scale or transform the features if needed.\n", + " \n", + " Args:\n", + " dataset: A Pandas `Dataframe`, containing the label on the first column and\n", + " monochrome pixel values on the remaining columns, in row major order.\n", + " Returns:\n", + " A `tuple` `(labels, features)`:\n", + " labels: A Pandas `Series`.\n", + " features: A Pandas `DataFrame`.\n", + " \"\"\"\n", + " labels = dataset[0]\n", + "\n", + " # DataFrame.loc index ranges are inclusive at both ends.\n", + " features = dataset.loc[:,1:784]\n", + " # Scale the data to [0, 1] by dividing out the max value, 255.\n", + " features = features / 255\n", + "\n", + " return labels, features" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "mFY_-7vZu8UU", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "training_targets, training_examples = parse_labels_and_features(mnist_dataframe[:7500])\n", + "training_examples.describe()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "4-Vgg-1zu8Ud", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "validation_targets, validation_examples = parse_labels_and_features(mnist_dataframe[7500:10000])\n", + "validation_examples.describe()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "wrnAI1v6u8Uh", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Show a random example and its corresponding label." + ] + }, + { + "metadata": { + "id": "s-euVJVtu8Ui", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "rand_example = np.random.choice(training_examples.index)\n", + "_, ax = plt.subplots()\n", + "ax.matshow(training_examples.loc[rand_example].values.reshape(28, 28))\n", + "ax.set_title(\"Label: %i\" % training_targets.loc[rand_example])\n", + "ax.grid(False)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "ScmYX7xdZMXE", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Task 1: Build a Linear Model for MNIST\n", + "\n", + "First, let's create a baseline model to compare against. The `LinearClassifier` provides a set of *k* one-vs-all classifiers, one for each of the *k* classes.\n", + "\n", + "You'll notice that in addition to reporting accuracy, and plotting Log Loss over time, we also display a [**confusion matrix**](https://en.wikipedia.org/wiki/Confusion_matrix). The confusion matrix shows which classes were misclassified as other classes. Which digits get confused for each other?\n", + "\n", + "Also note that we track the model's error using the `log_loss` function. This should not be confused with the loss function internal to `LinearClassifier` that is used for training." + ] + }, + { + "metadata": { + "id": "cpoVC4TSdw5Z", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def construct_feature_columns():\n", + " \"\"\"Construct the TensorFlow Feature Columns.\n", + "\n", + " Returns:\n", + " A set of feature columns\n", + " \"\"\" \n", + " \n", + " # There are 784 pixels in each image.\n", + " return set([tf.feature_column.numeric_column('pixels', shape=784)])" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "kMmL89yGeTfz", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Here, we'll make separate input functions for training and for prediction. We'll nest them in `create_training_input_fn()` and `create_predict_input_fn()`, respectively, so we can invoke these functions to return the corresponding `_input_fn`s to pass to our `.train()` and `.predict()` calls." + ] + }, + { + "metadata": { + "id": "OeS47Bmn5Ms2", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def create_training_input_fn(features, labels, batch_size, num_epochs=None, shuffle=True):\n", + " \"\"\"A custom input_fn for sending MNIST data to the estimator for training.\n", + "\n", + " Args:\n", + " features: The training features.\n", + " labels: The training labels.\n", + " batch_size: Batch size to use during training.\n", + "\n", + " Returns:\n", + " A function that returns batches of training features and labels during\n", + " training.\n", + " \"\"\"\n", + " def _input_fn(num_epochs=None, shuffle=True):\n", + " # Input pipelines are reset with each call to .train(). To ensure model\n", + " # gets a good sampling of data, even when number of steps is small, we \n", + " # shuffle all the data before creating the Dataset object\n", + " idx = np.random.permutation(features.index)\n", + " raw_features = {\"pixels\":features.reindex(idx)}\n", + " raw_targets = np.array(labels[idx])\n", + " \n", + " ds = Dataset.from_tensor_slices((raw_features,raw_targets)) # warning: 2GB limit\n", + " ds = ds.batch(batch_size).repeat(num_epochs)\n", + " \n", + " if shuffle:\n", + " ds = ds.shuffle(10000)\n", + " \n", + " # Return the next batch of data.\n", + " feature_batch, label_batch = ds.make_one_shot_iterator().get_next()\n", + " return feature_batch, label_batch\n", + "\n", + " return _input_fn" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "8zoGWAoohrwS", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def create_predict_input_fn(features, labels, batch_size):\n", + " \"\"\"A custom input_fn for sending mnist data to the estimator for predictions.\n", + "\n", + " Args:\n", + " features: The features to base predictions on.\n", + " labels: The labels of the prediction examples.\n", + "\n", + " Returns:\n", + " A function that returns features and labels for predictions.\n", + " \"\"\"\n", + " def _input_fn():\n", + " raw_features = {\"pixels\": features.values}\n", + " raw_targets = np.array(labels)\n", + " \n", + " ds = Dataset.from_tensor_slices((raw_features, raw_targets)) # warning: 2GB limit\n", + " ds = ds.batch(batch_size)\n", + " \n", + " \n", + " # Return the next batch of data.\n", + " feature_batch, label_batch = ds.make_one_shot_iterator().get_next()\n", + " return feature_batch, label_batch\n", + "\n", + " return _input_fn" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "G6DjSLZMu8Um", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def train_linear_classification_model(\n", + " learning_rate,\n", + " steps,\n", + " batch_size,\n", + " training_examples,\n", + " training_targets,\n", + " validation_examples,\n", + " validation_targets):\n", + " \"\"\"Trains a linear classification model for the MNIST digits dataset.\n", + " \n", + " In addition to training, this function also prints training progress information,\n", + " a plot of the training and validation loss over time, and a confusion\n", + " matrix.\n", + " \n", + " Args:\n", + " learning_rate: A `float`, the learning rate to use.\n", + " steps: A non-zero `int`, the total number of training steps. A training step\n", + " consists of a forward and backward pass using a single batch.\n", + " batch_size: A non-zero `int`, the batch size.\n", + " training_examples: A `DataFrame` containing the training features.\n", + " training_targets: A `DataFrame` containing the training labels.\n", + " validation_examples: A `DataFrame` containing the validation features.\n", + " validation_targets: A `DataFrame` containing the validation labels.\n", + " \n", + " Returns:\n", + " The trained `LinearClassifier` object.\n", + " \"\"\"\n", + "\n", + " periods = 10\n", + "\n", + " steps_per_period = steps / periods \n", + " # Create the input functions.\n", + " predict_training_input_fn = create_predict_input_fn(\n", + " training_examples, training_targets, batch_size)\n", + " predict_validation_input_fn = create_predict_input_fn(\n", + " validation_examples, validation_targets, batch_size)\n", + " training_input_fn = create_training_input_fn(\n", + " training_examples, training_targets, batch_size)\n", + " \n", + " # Create a LinearClassifier object.\n", + " my_optimizer = tf.train.AdagradOptimizer(learning_rate=learning_rate)\n", + " my_optimizer = tf.contrib.estimator.clip_gradients_by_norm(my_optimizer, 5.0)\n", + " classifier = tf.estimator.LinearClassifier(\n", + " feature_columns=construct_feature_columns(),\n", + " n_classes=10,\n", + " optimizer=my_optimizer,\n", + " config=tf.estimator.RunConfig(keep_checkpoint_max=1)\n", + " )\n", + "\n", + " # Train the model, but do so inside a loop so that we can periodically assess\n", + " # loss metrics.\n", + " print(\"Training model...\")\n", + " print(\"LogLoss error (on validation data):\")\n", + " training_errors = []\n", + " validation_errors = []\n", + " for period in range (0, periods):\n", + " # Train the model, starting from the prior state.\n", + " classifier.train(\n", + " input_fn=training_input_fn,\n", + " steps=steps_per_period\n", + " )\n", + " \n", + " # Take a break and compute probabilities.\n", + " training_predictions = list(classifier.predict(input_fn=predict_training_input_fn))\n", + " training_probabilities = np.array([item['probabilities'] for item in training_predictions])\n", + " training_pred_class_id = np.array([item['class_ids'][0] for item in training_predictions])\n", + " training_pred_one_hot = tf.keras.utils.to_categorical(training_pred_class_id,10)\n", + " \n", + " validation_predictions = list(classifier.predict(input_fn=predict_validation_input_fn))\n", + " validation_probabilities = np.array([item['probabilities'] for item in validation_predictions]) \n", + " validation_pred_class_id = np.array([item['class_ids'][0] for item in validation_predictions])\n", + " validation_pred_one_hot = tf.keras.utils.to_categorical(validation_pred_class_id,10) \n", + " \n", + " # Compute training and validation errors.\n", + " training_log_loss = metrics.log_loss(training_targets, training_pred_one_hot)\n", + " validation_log_loss = metrics.log_loss(validation_targets, validation_pred_one_hot)\n", + " # Occasionally print the current loss.\n", + " print(\" period %02d : %0.2f\" % (period, validation_log_loss))\n", + " # Add the loss metrics from this period to our list.\n", + " training_errors.append(training_log_loss)\n", + " validation_errors.append(validation_log_loss)\n", + " print(\"Model training finished.\")\n", + " # Remove event files to save disk space.\n", + " _ = map(os.remove, glob.glob(os.path.join(classifier.model_dir, 'events.out.tfevents*')))\n", + " \n", + " # Calculate final predictions (not probabilities, as above).\n", + " final_predictions = classifier.predict(input_fn=predict_validation_input_fn)\n", + " final_predictions = np.array([item['class_ids'][0] for item in final_predictions])\n", + " \n", + " \n", + " accuracy = metrics.accuracy_score(validation_targets, final_predictions)\n", + " print(\"Final accuracy (on validation data): %0.2f\" % accuracy)\n", + "\n", + " # Output a graph of loss metrics over periods.\n", + " plt.ylabel(\"LogLoss\")\n", + " plt.xlabel(\"Periods\")\n", + " plt.title(\"LogLoss vs. Periods\")\n", + " plt.plot(training_errors, label=\"training\")\n", + " plt.plot(validation_errors, label=\"validation\")\n", + " plt.legend()\n", + " plt.show()\n", + " \n", + " # Output a plot of the confusion matrix.\n", + " cm = metrics.confusion_matrix(validation_targets, final_predictions)\n", + " # Normalize the confusion matrix by row (i.e by the number of samples\n", + " # in each class).\n", + " cm_normalized = cm.astype(\"float\") / cm.sum(axis=1)[:, np.newaxis]\n", + " ax = sns.heatmap(cm_normalized, cmap=\"bone_r\")\n", + " ax.set_aspect(1)\n", + " plt.title(\"Confusion matrix\")\n", + " plt.ylabel(\"True label\")\n", + " plt.xlabel(\"Predicted label\")\n", + " plt.show()\n", + "\n", + " return classifier" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "ItHIUyv2u8Ur", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "**Spend 5 minutes seeing how well you can do on accuracy with a linear model of this form. For this exercise, limit yourself to experimenting with the hyperparameters for batch size, learning rate and steps.**\n", + "\n", + "Stop if you get anything above about 0.9 accuracy." + ] + }, + { + "metadata": { + "id": "yaiIhIQqu8Uv", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "classifier = train_linear_classification_model(\n", + " learning_rate=0.03,\n", + " steps=1000,\n", + " batch_size=30,\n", + " training_examples=training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "266KQvZoMxMv", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Solution\n", + "\n", + "Click below for one possible solution." + ] + }, + { + "metadata": { + "id": "lRWcn24DM3qa", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Here is a set of parameters that should attain roughly 0.9 accuracy." + ] + }, + { + "metadata": { + "id": "TGlBMrUoM1K_", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "_ = train_linear_classification_model(\n", + " learning_rate=0.03,\n", + " steps=1000,\n", + " batch_size=30,\n", + " training_examples=training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "mk095OfpPdOx", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Task 2: Replace the Linear Classifier with a Neural Network\n", + "\n", + "**Replace the LinearClassifier above with a [`DNNClassifier`](https://www.tensorflow.org/api_docs/python/tf/estimator/DNNClassifier) and find a parameter combination that gives 0.95 or better accuracy.**\n", + "\n", + "You may wish to experiment with additional regularization methods, such as dropout. These additional regularization methods are documented in the comments for the `DNNClassifier` class." + ] + }, + { + "metadata": { + "id": "rm8P_Ttwu8U4", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#\n", + "# YOUR CODE HERE: Replace the linear classifier with a neural network.\n", + "#\n", + "def train_nn_classification_model(\n", + " learning_rate,\n", + " steps,\n", + " batch_size,\n", + " hidden_units,\n", + " training_examples,\n", + " training_targets,\n", + " validation_examples,\n", + " validation_targets):\n", + "\n", + "\n", + " periods = 10.\n", + " steps_per_period = steps / periods \n", + " \n", + " predict_training_input_fn = create_predict_input_fn(\n", + " training_examples, training_targets, batch_size)\n", + " predict_validation_input_fn = create_predict_input_fn(\n", + " validation_examples, validation_targets, batch_size)\n", + " training_input_fn = create_training_input_fn(\n", + " training_examples, training_targets, batch_size)\n", + " \n", + " \n", + " predict_training_input_fn = create_predict_input_fn(\n", + " training_examples, training_targets, batch_size)\n", + " predict_validation_input_fn = create_predict_input_fn(\n", + " validation_examples, validation_targets, batch_size)\n", + " training_input_fn = create_training_input_fn(\n", + " training_examples, training_targets, batch_size)\n", + " \n", + " \n", + " feature_columns = [tf.feature_column.numeric_column('pixels', shape=784)]\n", + "\n", + " \n", + " my_optimizer = tf.train.AdagradOptimizer(learning_rate=learning_rate)\n", + " my_optimizer = tf.contrib.estimator.clip_gradients_by_norm(my_optimizer, 5.0)\n", + " classifier = tf.estimator.DNNClassifier(\n", + " feature_columns=feature_columns,\n", + " n_classes=10,\n", + " hidden_units=hidden_units,\n", + " optimizer=my_optimizer,\n", + " config=tf.contrib.learn.RunConfig(keep_checkpoint_max=1)\n", + " )\n", + "\n", + " \n", + " print(\"Training model...\")\n", + " print(\"LogLoss error (on validation data):\")\n", + " training_errors = []\n", + " validation_errors = []\n", + " for period in range (0, periods):\n", + " \n", + " classifier.train(\n", + " input_fn=training_input_fn,\n", + " steps=steps_per_period\n", + " )\n", + " \n", + " \n", + " training_predictions = list(classifier.predict(input_fn=predict_training_input_fn))\n", + " training_probabilities = np.array([item['probabilities'] for item in training_predictions])\n", + " training_pred_class_id = np.array([item['class_ids'][0] for item in training_predictions])\n", + " training_pred_one_hot = tf.keras.utils.to_categorical(training_pred_class_id,10)\n", + " \n", + " validation_predictions = list(classifier.predict(input_fn=predict_validation_input_fn))\n", + " validation_probabilities = np.array([item['probabilities'] for item in validation_predictions]) \n", + " validation_pred_class_id = np.array([item['class_ids'][0] for item in validation_predictions])\n", + " validation_pred_one_hot = tf.keras.utils.to_categorical(validation_pred_class_id,10) \n", + " \n", + " \n", + " training_log_loss = metrics.log_loss(training_targets, training_pred_one_hot)\n", + " validation_log_loss = metrics.log_loss(validation_targets, validation_pred_one_hot)\n", + " \n", + " print(\" period %02d : %0.2f\" % (period, validation_log_loss))\n", + " \n", + " training_errors.append(training_log_loss)\n", + " validation_errors.append(validation_log_loss)\n", + " print(\"Model training finished.\")\n", + " \n", + " _ = map(os.remove, glob.glob(os.path.join(classifier.model_dir, 'events.out.tfevents*')))\n", + " \n", + " \n", + " final_predictions = classifier.predict(input_fn=predict_validation_input_fn)\n", + " final_predictions = np.array([item['class_ids'][0] for item in final_predictions])\n", + " \n", + " \n", + " accuracy = metrics.accuracy_score(validation_targets, final_predictions)\n", + " print(\"Final accuracy (on validation data): %0.2f\" % accuracy)\n", + "\n", + " \n", + " plt.ylabel(\"LogLoss\")\n", + " plt.xlabel(\"Periods\")\n", + " plt.title(\"LogLoss vs. Periods\")\n", + " plt.plot(training_errors, label=\"training\")\n", + " plt.plot(validation_errors, label=\"validation\")\n", + " plt.legend()\n", + " plt.show()\n", + " \n", + " \n", + " cm = metrics.confusion_matrix(validation_targets, final_predictions)\n", + " \n", + " cm_normalized = cm.astype(\"float\") / cm.sum(axis=1)[:, np.newaxis]\n", + " ax = sns.heatmap(cm_normalized, cmap=\"bone_r\")\n", + " ax.set_aspect(1)\n", + " plt.title(\"Confusion matrix\")\n", + " plt.ylabel(\"True label\")\n", + " plt.xlabel(\"Predicted label\")\n", + " plt.show()\n", + "\n", + " return classifier" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "TOfmiSvqu8U9", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Once you have a good model, double check that you didn't overfit the validation set by evaluating on the test data that we'll load below.\n" + ] + }, + { + "metadata": { + "id": "evlB5ubzu8VJ", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "mnist_test_dataframe = pd.read_csv(\n", + " \"https://download.mlcc.google.com/mledu-datasets/mnist_test.csv\",\n", + " sep=\",\",\n", + " header=None)\n", + "\n", + "test_targets, test_examples = parse_labels_and_features(mnist_test_dataframe)\n", + "test_examples.describe()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "PDuLd2Hcu8VL", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#\n", + "# YOUR CODE HERE: Calculate accuracy on the test set.\n", + "#\n", + "classifier = train_nn_classification_model(\n", + " learning_rate=0.05,\n", + " steps=1000,\n", + " batch_size=30,\n", + " hidden_units=[100, 100],\n", + " training_examples=training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "6sfw3LH0Oycm", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Solution\n", + "\n", + "Click below for a possible solution." + ] + }, + { + "metadata": { + "id": "XatDGFKEO374", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "The code below is almost identical to the original `LinearClassifer` training code, with the exception of the NN-specific configuration, such as the hyperparameter for hidden units." + ] + }, + { + "metadata": { + "id": "kdNTx8jkPQUx", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def train_nn_classification_model(\n", + " learning_rate,\n", + " steps,\n", + " batch_size,\n", + " hidden_units,\n", + " training_examples,\n", + " training_targets,\n", + " validation_examples,\n", + " validation_targets):\n", + " \"\"\"Trains a neural network classification model for the MNIST digits dataset.\n", + " \n", + " In addition to training, this function also prints training progress information,\n", + " a plot of the training and validation loss over time, as well as a confusion\n", + " matrix.\n", + " \n", + " Args:\n", + " learning_rate: A `float`, the learning rate to use.\n", + " steps: A non-zero `int`, the total number of training steps. A training step\n", + " consists of a forward and backward pass using a single batch.\n", + " batch_size: A non-zero `int`, the batch size.\n", + " hidden_units: A `list` of int values, specifying the number of neurons in each layer.\n", + " training_examples: A `DataFrame` containing the training features.\n", + " training_targets: A `DataFrame` containing the training labels.\n", + " validation_examples: A `DataFrame` containing the validation features.\n", + " validation_targets: A `DataFrame` containing the validation labels.\n", + " \n", + " Returns:\n", + " The trained `DNNClassifier` object.\n", + " \"\"\"\n", + "\n", + " periods = 10\n", + " # Caution: input pipelines are reset with each call to train. \n", + " # If the number of steps is small, your model may never see most of the data. \n", + " # So with multiple `.train` calls like this you may want to control the length \n", + " # of training with num_epochs passed to the input_fn. Or, you can do a really-big shuffle, \n", + " # or since it's in-memory data, shuffle all the data in the `input_fn`.\n", + " steps_per_period = steps / periods \n", + " # Create the input functions.\n", + " predict_training_input_fn = create_predict_input_fn(\n", + " training_examples, training_targets, batch_size)\n", + " predict_validation_input_fn = create_predict_input_fn(\n", + " validation_examples, validation_targets, batch_size)\n", + " training_input_fn = create_training_input_fn(\n", + " training_examples, training_targets, batch_size)\n", + " \n", + " # Create the input functions.\n", + " predict_training_input_fn = create_predict_input_fn(\n", + " training_examples, training_targets, batch_size)\n", + " predict_validation_input_fn = create_predict_input_fn(\n", + " validation_examples, validation_targets, batch_size)\n", + " training_input_fn = create_training_input_fn(\n", + " training_examples, training_targets, batch_size)\n", + " \n", + " # Create feature columns.\n", + " feature_columns = [tf.feature_column.numeric_column('pixels', shape=784)]\n", + "\n", + " # Create a DNNClassifier object.\n", + " my_optimizer = tf.train.AdagradOptimizer(learning_rate=learning_rate)\n", + " my_optimizer = tf.contrib.estimator.clip_gradients_by_norm(my_optimizer, 5.0)\n", + " classifier = tf.estimator.DNNClassifier(\n", + " feature_columns=feature_columns,\n", + " n_classes=10,\n", + " hidden_units=hidden_units,\n", + " optimizer=my_optimizer,\n", + " config=tf.contrib.learn.RunConfig(keep_checkpoint_max=1)\n", + " )\n", + "\n", + " # Train the model, but do so inside a loop so that we can periodically assess\n", + " # loss metrics.\n", + " print(\"Training model...\")\n", + " print(\"LogLoss error (on validation data):\")\n", + " training_errors = []\n", + " validation_errors = []\n", + " for period in range (0, periods):\n", + " # Train the model, starting from the prior state.\n", + " classifier.train(\n", + " input_fn=training_input_fn,\n", + " steps=steps_per_period\n", + " )\n", + " \n", + " # Take a break and compute probabilities.\n", + " training_predictions = list(classifier.predict(input_fn=predict_training_input_fn))\n", + " training_probabilities = np.array([item['probabilities'] for item in training_predictions])\n", + " training_pred_class_id = np.array([item['class_ids'][0] for item in training_predictions])\n", + " training_pred_one_hot = tf.keras.utils.to_categorical(training_pred_class_id,10)\n", + " \n", + " validation_predictions = list(classifier.predict(input_fn=predict_validation_input_fn))\n", + " validation_probabilities = np.array([item['probabilities'] for item in validation_predictions]) \n", + " validation_pred_class_id = np.array([item['class_ids'][0] for item in validation_predictions])\n", + " validation_pred_one_hot = tf.keras.utils.to_categorical(validation_pred_class_id,10) \n", + " \n", + " # Compute training and validation errors.\n", + " training_log_loss = metrics.log_loss(training_targets, training_pred_one_hot)\n", + " validation_log_loss = metrics.log_loss(validation_targets, validation_pred_one_hot)\n", + " # Occasionally print the current loss.\n", + " print(\" period %02d : %0.2f\" % (period, validation_log_loss))\n", + " # Add the loss metrics from this period to our list.\n", + " training_errors.append(training_log_loss)\n", + " validation_errors.append(validation_log_loss)\n", + " print(\"Model training finished.\")\n", + " # Remove event files to save disk space.\n", + " _ = map(os.remove, glob.glob(os.path.join(classifier.model_dir, 'events.out.tfevents*')))\n", + " \n", + " # Calculate final predictions (not probabilities, as above).\n", + " final_predictions = classifier.predict(input_fn=predict_validation_input_fn)\n", + " final_predictions = np.array([item['class_ids'][0] for item in final_predictions])\n", + " \n", + " \n", + " accuracy = metrics.accuracy_score(validation_targets, final_predictions)\n", + " print(\"Final accuracy (on validation data): %0.2f\" % accuracy)\n", + "\n", + " # Output a graph of loss metrics over periods.\n", + " plt.ylabel(\"LogLoss\")\n", + " plt.xlabel(\"Periods\")\n", + " plt.title(\"LogLoss vs. Periods\")\n", + " plt.plot(training_errors, label=\"training\")\n", + " plt.plot(validation_errors, label=\"validation\")\n", + " plt.legend()\n", + " plt.show()\n", + " \n", + " # Output a plot of the confusion matrix.\n", + " cm = metrics.confusion_matrix(validation_targets, final_predictions)\n", + " # Normalize the confusion matrix by row (i.e by the number of samples\n", + " # in each class).\n", + " cm_normalized = cm.astype(\"float\") / cm.sum(axis=1)[:, np.newaxis]\n", + " ax = sns.heatmap(cm_normalized, cmap=\"bone_r\")\n", + " ax.set_aspect(1)\n", + " plt.title(\"Confusion matrix\")\n", + " plt.ylabel(\"True label\")\n", + " plt.xlabel(\"Predicted label\")\n", + " plt.show()\n", + "\n", + " return classifier" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "ZfzsTYGPPU8I", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "classifier = train_nn_classification_model(\n", + " learning_rate=0.05,\n", + " steps=1000,\n", + " batch_size=30,\n", + " hidden_units=[100, 100],\n", + " training_examples=training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "qXvrOgtUR-zD", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Next, we verify the accuracy on the test set." + ] + }, + { + "metadata": { + "id": "scQNpDePSFjt", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "mnist_test_dataframe = pd.read_csv(\n", + " \"https://download.mlcc.google.com/mledu-datasets/mnist_test.csv\",\n", + " sep=\",\",\n", + " header=None)\n", + "\n", + "test_targets, test_examples = parse_labels_and_features(mnist_test_dataframe)\n", + "test_examples.describe()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "EVaWpWKvSHmu", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "predict_test_input_fn = create_predict_input_fn(\n", + " test_examples, test_targets, batch_size=100)\n", + "\n", + "test_predictions = classifier.predict(input_fn=predict_test_input_fn)\n", + "test_predictions = np.array([item['class_ids'][0] for item in test_predictions])\n", + " \n", + "accuracy = metrics.accuracy_score(test_targets, test_predictions)\n", + "print(\"Accuracy on test data: %0.2f\" % accuracy)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "WX2mQBAEcisO", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Task 3: Visualize the weights of the first hidden layer.\n", + "\n", + "Let's take a few minutes to dig into our neural network and see what it has learned by accessing the `weights_` attribute of our model.\n", + "\n", + "The input layer of our model has `784` weights corresponding to the `28×28` pixel input images. The first hidden layer will have `784×N` weights where `N` is the number of nodes in that layer. We can turn those weights back into `28×28` images by *reshaping* each of the `N` `1×784` arrays of weights into `N` arrays of size `28×28`.\n", + "\n", + "Run the following cell to plot the weights. Note that this cell requires that a `DNNClassifier` called \"classifier\" has already been trained." + ] + }, + { + "metadata": { + "id": "eUC0Z8nbafgG", + "colab_type": "code", + "cellView": "both", + "colab": {} + }, + "cell_type": "code", + "source": [ + "print(classifier.get_variable_names())\n", + "\n", + "weights0 = classifier.get_variable_value(\"dnn/hiddenlayer_0/kernel\")\n", + "\n", + "print(\"weights0 shape:\", weights0.shape)\n", + "\n", + "num_nodes = weights0.shape[1]\n", + "num_rows = int(math.ceil(num_nodes / 10.0))\n", + "fig, axes = plt.subplots(num_rows, 10, figsize=(20, 2 * num_rows))\n", + "for coef, ax in zip(weights0.T, axes.ravel()):\n", + " # Weights in coef is reshaped from 1x784 to 28x28.\n", + " ax.matshow(coef.reshape(28, 28), cmap=plt.cm.pink)\n", + " ax.set_xticks(())\n", + " ax.set_yticks(())\n", + "\n", + "plt.show()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "kL8MEhNgrx9N", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "The first hidden layer of the neural network should be modeling some pretty low level features, so visualizing the weights will probably just show some fuzzy blobs or possibly a few parts of digits. You may also see some neurons that are essentially noise -- these are either unconverged or they are being ignored by higher layers.\n", + "\n", + "It can be interesting to stop training at different numbers of iterations and see the effect.\n", + "\n", + "**Train the classifier for 10, 100 and respectively 1000 steps. Then run this visualization again.**\n", + "\n", + "What differences do you see visually for the different levels of convergence?" + ] + } + ] +} \ No newline at end of file diff --git a/sparsity_and_l1_regularization.ipynb b/sparsity_and_l1_regularization.ipynb new file mode 100644 index 0000000..33eba58 --- /dev/null +++ b/sparsity_and_l1_regularization.ipynb @@ -0,0 +1,598 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "sparsity_and_l1_regularization.ipynb", + "version": "0.3.2", + "provenance": [], + "collapsed_sections": [ + "JndnmDMp66FL", + "yjUCX5LAkxAX" + ], + "include_colab_link": true + }, + "kernelspec": { + "name": "python2", + "display_name": "Python 2" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "metadata": { + "id": "JndnmDMp66FL", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "#### Copyright 2017 Google LLC." + ] + }, + { + "metadata": { + "id": "hMqWDc_m6rUC", + "colab_type": "code", + "cellView": "both", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "g4T-_IsVbweU", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Sparsity and L1 Regularization" + ] + }, + { + "metadata": { + "id": "g8ue2FyFIjnQ", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "**Learning Objectives:**\n", + " * Calculate the size of a model\n", + " * Apply L1 regularization to reduce the size of a model by increasing sparsity" + ] + }, + { + "metadata": { + "id": "ME_WXE7cIjnS", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "One way to reduce complexity is to use a regularization function that encourages weights to be exactly zero. For linear models such as regression, a zero weight is equivalent to not using the corresponding feature at all. In addition to avoiding overfitting, the resulting model will be more efficient.\n", + "\n", + "L1 regularization is a good way to increase sparsity.\n", + "\n" + ] + }, + { + "metadata": { + "id": "fHRzeWkRLrHF", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Setup\n", + "\n", + "Run the cells below to load the data and create feature definitions." + ] + }, + { + "metadata": { + "id": "pb7rSrLKIjnS", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "from __future__ import print_function\n", + "\n", + "import math\n", + "\n", + "from IPython import display\n", + "from matplotlib import cm\n", + "from matplotlib import gridspec\n", + "from matplotlib import pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "from sklearn import metrics\n", + "import tensorflow as tf\n", + "from tensorflow.python.data import Dataset\n", + "\n", + "tf.logging.set_verbosity(tf.logging.ERROR)\n", + "pd.options.display.max_rows = 10\n", + "pd.options.display.float_format = '{:.1f}'.format\n", + "\n", + "california_housing_dataframe = pd.read_csv(\"https://download.mlcc.google.com/mledu-datasets/california_housing_train.csv\", sep=\",\")\n", + "\n", + "california_housing_dataframe = california_housing_dataframe.reindex(\n", + " np.random.permutation(california_housing_dataframe.index))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "3V7q8jk0IjnW", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def preprocess_features(california_housing_dataframe):\n", + " \"\"\"Prepares input features from California housing data set.\n", + "\n", + " Args:\n", + " california_housing_dataframe: A Pandas DataFrame expected to contain data\n", + " from the California housing data set.\n", + " Returns:\n", + " A DataFrame that contains the features to be used for the model, including\n", + " synthetic features.\n", + " \"\"\"\n", + " selected_features = california_housing_dataframe[\n", + " [\"latitude\",\n", + " \"longitude\",\n", + " \"housing_median_age\",\n", + " \"total_rooms\",\n", + " \"total_bedrooms\",\n", + " \"population\",\n", + " \"households\",\n", + " \"median_income\"]]\n", + " processed_features = selected_features.copy()\n", + " # Create a synthetic feature.\n", + " processed_features[\"rooms_per_person\"] = (\n", + " california_housing_dataframe[\"total_rooms\"] /\n", + " california_housing_dataframe[\"population\"])\n", + " return processed_features\n", + "\n", + "def preprocess_targets(california_housing_dataframe):\n", + " \"\"\"Prepares target features (i.e., labels) from California housing data set.\n", + "\n", + " Args:\n", + " california_housing_dataframe: A Pandas DataFrame expected to contain data\n", + " from the California housing data set.\n", + " Returns:\n", + " A DataFrame that contains the target feature.\n", + " \"\"\"\n", + " output_targets = pd.DataFrame()\n", + " # Create a boolean categorical feature representing whether the\n", + " # median_house_value is above a set threshold.\n", + " output_targets[\"median_house_value_is_high\"] = (\n", + " california_housing_dataframe[\"median_house_value\"] > 265000).astype(float)\n", + " return output_targets" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "pAG3tmgwIjnY", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# Choose the first 12000 (out of 17000) examples for training.\n", + "training_examples = preprocess_features(california_housing_dataframe.head(12000))\n", + "training_targets = preprocess_targets(california_housing_dataframe.head(12000))\n", + "\n", + "# Choose the last 5000 (out of 17000) examples for validation.\n", + "validation_examples = preprocess_features(california_housing_dataframe.tail(5000))\n", + "validation_targets = preprocess_targets(california_housing_dataframe.tail(5000))\n", + "\n", + "# Double-check that we've done the right thing.\n", + "print(\"Training examples summary:\")\n", + "display.display(training_examples.describe())\n", + "print(\"Validation examples summary:\")\n", + "display.display(validation_examples.describe())\n", + "\n", + "print(\"Training targets summary:\")\n", + "display.display(training_targets.describe())\n", + "print(\"Validation targets summary:\")\n", + "display.display(validation_targets.describe())" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "gHkniRI1Ijna", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def my_input_fn(features, targets, batch_size=1, shuffle=True, num_epochs=None):\n", + " \"\"\"Trains a linear regression model.\n", + " \n", + " Args:\n", + " features: pandas DataFrame of features\n", + " targets: pandas DataFrame of targets\n", + " batch_size: Size of batches to be passed to the model\n", + " shuffle: True or False. Whether to shuffle the data.\n", + " num_epochs: Number of epochs for which data should be repeated. None = repeat indefinitely\n", + " Returns:\n", + " Tuple of (features, labels) for next data batch\n", + " \"\"\"\n", + " \n", + " # Convert pandas data into a dict of np arrays.\n", + " features = {key:np.array(value) for key,value in dict(features).items()} \n", + " \n", + " # Construct a dataset, and configure batching/repeating.\n", + " ds = Dataset.from_tensor_slices((features,targets)) # warning: 2GB limit\n", + " ds = ds.batch(batch_size).repeat(num_epochs)\n", + " \n", + " # Shuffle the data, if specified.\n", + " if shuffle:\n", + " ds = ds.shuffle(10000)\n", + " \n", + " # Return the next batch of data.\n", + " features, labels = ds.make_one_shot_iterator().get_next()\n", + " return features, labels" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "bLzK72jkNJPf", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def get_quantile_based_buckets(feature_values, num_buckets):\n", + " quantiles = feature_values.quantile(\n", + " [(i+1.)/(num_buckets + 1.) for i in range(num_buckets)])\n", + " return [quantiles[q] for q in quantiles.keys()]" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "al2YQpKyIjnd", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def construct_feature_columns():\n", + " \"\"\"Construct the TensorFlow Feature Columns.\n", + "\n", + " Returns:\n", + " A set of feature columns\n", + " \"\"\"\n", + "\n", + " bucketized_households = tf.feature_column.bucketized_column(\n", + " tf.feature_column.numeric_column(\"households\"),\n", + " boundaries=get_quantile_based_buckets(training_examples[\"households\"], 10))\n", + " bucketized_longitude = tf.feature_column.bucketized_column(\n", + " tf.feature_column.numeric_column(\"longitude\"),\n", + " boundaries=get_quantile_based_buckets(training_examples[\"longitude\"], 50))\n", + " bucketized_latitude = tf.feature_column.bucketized_column(\n", + " tf.feature_column.numeric_column(\"latitude\"),\n", + " boundaries=get_quantile_based_buckets(training_examples[\"latitude\"], 50))\n", + " bucketized_housing_median_age = tf.feature_column.bucketized_column(\n", + " tf.feature_column.numeric_column(\"housing_median_age\"),\n", + " boundaries=get_quantile_based_buckets(\n", + " training_examples[\"housing_median_age\"], 10))\n", + " bucketized_total_rooms = tf.feature_column.bucketized_column(\n", + " tf.feature_column.numeric_column(\"total_rooms\"),\n", + " boundaries=get_quantile_based_buckets(training_examples[\"total_rooms\"], 10))\n", + " bucketized_total_bedrooms = tf.feature_column.bucketized_column(\n", + " tf.feature_column.numeric_column(\"total_bedrooms\"),\n", + " boundaries=get_quantile_based_buckets(training_examples[\"total_bedrooms\"], 10))\n", + " bucketized_population = tf.feature_column.bucketized_column(\n", + " tf.feature_column.numeric_column(\"population\"),\n", + " boundaries=get_quantile_based_buckets(training_examples[\"population\"], 10))\n", + " bucketized_median_income = tf.feature_column.bucketized_column(\n", + " tf.feature_column.numeric_column(\"median_income\"),\n", + " boundaries=get_quantile_based_buckets(training_examples[\"median_income\"], 10))\n", + " bucketized_rooms_per_person = tf.feature_column.bucketized_column(\n", + " tf.feature_column.numeric_column(\"rooms_per_person\"),\n", + " boundaries=get_quantile_based_buckets(\n", + " training_examples[\"rooms_per_person\"], 10))\n", + "\n", + " long_x_lat = tf.feature_column.crossed_column(\n", + " set([bucketized_longitude, bucketized_latitude]), hash_bucket_size=1000)\n", + "\n", + " feature_columns = set([\n", + " long_x_lat,\n", + " bucketized_longitude,\n", + " bucketized_latitude,\n", + " bucketized_housing_median_age,\n", + " bucketized_total_rooms,\n", + " bucketized_total_bedrooms,\n", + " bucketized_population,\n", + " bucketized_households,\n", + " bucketized_median_income,\n", + " bucketized_rooms_per_person])\n", + " \n", + " return feature_columns" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "hSBwMrsrE21n", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Calculate the Model Size\n", + "\n", + "To calculate the model size, we simply count the number of parameters that are non-zero. We provide a helper function below to do that. The function uses intimate knowledge of the Estimators API - don't worry about understanding how it works." + ] + }, + { + "metadata": { + "id": "e6GfTI0CFhB8", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def model_size(estimator):\n", + " variables = estimator.get_variable_names()\n", + " size = 0\n", + " for variable in variables:\n", + " if not any(x in variable \n", + " for x in ['global_step',\n", + " 'centered_bias_weight',\n", + " 'bias_weight',\n", + " 'Ftrl']\n", + " ):\n", + " size += np.count_nonzero(estimator.get_variable_value(variable))\n", + " return size" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "XabdAaj67GfF", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Reduce the Model Size\n", + "\n", + "Your team needs to build a highly accurate Logistic Regression model on the *SmartRing*, a ring that is so smart it can sense the demographics of a city block ('median_income', 'avg_rooms', 'households', ..., etc.) and tell you whether the given city block is high cost city block or not.\n", + "\n", + "Since the SmartRing is small, the engineering team has determined that it can only handle a model that has **no more than 600 parameters**. On the other hand, the product management team has determined that the model is not launchable unless the **LogLoss is less than 0.35** on the holdout test set.\n", + "\n", + "Can you use your secret weapon—L1 regularization—to tune the model to satisfy both the size and accuracy constraints?" + ] + }, + { + "metadata": { + "id": "G79hGRe7qqej", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Task 1: Find a good regularization coefficient.\n", + "\n", + "**Find an L1 regularization strength parameter which satisfies both constraints — model size is less than 600 and log-loss is less than 0.35 on validation set.**\n", + "\n", + "The following code will help you get started. There are many ways to apply regularization to your model. Here, we chose to do it using `FtrlOptimizer`, which is designed to give better results with L1 regularization than standard gradient descent.\n", + "\n", + "Again, the model will train on the entire data set, so expect it to run slower than normal." + ] + }, + { + "metadata": { + "id": "1Fcdm0hpIjnl", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def train_linear_classifier_model(\n", + " learning_rate,\n", + " regularization_strength,\n", + " steps,\n", + " batch_size,\n", + " feature_columns,\n", + " training_examples,\n", + " training_targets,\n", + " validation_examples,\n", + " validation_targets):\n", + " \"\"\"Trains a linear regression model.\n", + " \n", + " In addition to training, this function also prints training progress information,\n", + " as well as a plot of the training and validation loss over time.\n", + " \n", + " Args:\n", + " learning_rate: A `float`, the learning rate.\n", + " regularization_strength: A `float` that indicates the strength of the L1\n", + " regularization. A value of `0.0` means no regularization.\n", + " steps: A non-zero `int`, the total number of training steps. A training step\n", + " consists of a forward and backward pass using a single batch.\n", + " feature_columns: A `set` specifying the input feature columns to use.\n", + " training_examples: A `DataFrame` containing one or more columns from\n", + " `california_housing_dataframe` to use as input features for training.\n", + " training_targets: A `DataFrame` containing exactly one column from\n", + " `california_housing_dataframe` to use as target for training.\n", + " validation_examples: A `DataFrame` containing one or more columns from\n", + " `california_housing_dataframe` to use as input features for validation.\n", + " validation_targets: A `DataFrame` containing exactly one column from\n", + " `california_housing_dataframe` to use as target for validation.\n", + " \n", + " Returns:\n", + " A `LinearClassifier` object trained on the training data.\n", + " \"\"\"\n", + "\n", + " periods = 7\n", + " steps_per_period = steps / periods\n", + "\n", + " # Create a linear classifier object.\n", + " my_optimizer = tf.train.FtrlOptimizer(learning_rate=learning_rate, l1_regularization_strength=regularization_strength)\n", + " my_optimizer = tf.contrib.estimator.clip_gradients_by_norm(my_optimizer, 5.0)\n", + " linear_classifier = tf.estimator.LinearClassifier(\n", + " feature_columns=feature_columns,\n", + " optimizer=my_optimizer\n", + " )\n", + " \n", + " # Create input functions.\n", + " training_input_fn = lambda: my_input_fn(training_examples, \n", + " training_targets[\"median_house_value_is_high\"], \n", + " batch_size=batch_size)\n", + " predict_training_input_fn = lambda: my_input_fn(training_examples, \n", + " training_targets[\"median_house_value_is_high\"], \n", + " num_epochs=1, \n", + " shuffle=False)\n", + " predict_validation_input_fn = lambda: my_input_fn(validation_examples, \n", + " validation_targets[\"median_house_value_is_high\"], \n", + " num_epochs=1, \n", + " shuffle=False)\n", + " \n", + " # Train the model, but do so inside a loop so that we can periodically assess\n", + " # loss metrics.\n", + " print(\"Training model...\")\n", + " print(\"LogLoss (on validation data):\")\n", + " training_log_losses = []\n", + " validation_log_losses = []\n", + " for period in range (0, periods):\n", + " # Train the model, starting from the prior state.\n", + " linear_classifier.train(\n", + " input_fn=training_input_fn,\n", + " steps=steps_per_period\n", + " )\n", + " # Take a break and compute predictions.\n", + " training_probabilities = linear_classifier.predict(input_fn=predict_training_input_fn)\n", + " training_probabilities = np.array([item['probabilities'] for item in training_probabilities])\n", + " \n", + " validation_probabilities = linear_classifier.predict(input_fn=predict_validation_input_fn)\n", + " validation_probabilities = np.array([item['probabilities'] for item in validation_probabilities])\n", + " \n", + " # Compute training and validation loss.\n", + " training_log_loss = metrics.log_loss(training_targets, training_probabilities)\n", + " validation_log_loss = metrics.log_loss(validation_targets, validation_probabilities)\n", + " # Occasionally print the current loss.\n", + " print(\" period %02d : %0.2f\" % (period, validation_log_loss))\n", + " # Add the loss metrics from this period to our list.\n", + " training_log_losses.append(training_log_loss)\n", + " validation_log_losses.append(validation_log_loss)\n", + " print(\"Model training finished.\")\n", + "\n", + " # Output a graph of loss metrics over periods.\n", + " plt.ylabel(\"LogLoss\")\n", + " plt.xlabel(\"Periods\")\n", + " plt.title(\"LogLoss vs. Periods\")\n", + " plt.tight_layout()\n", + " plt.plot(training_log_losses, label=\"training\")\n", + " plt.plot(validation_log_losses, label=\"validation\")\n", + " plt.legend()\n", + "\n", + " return linear_classifier" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "9H1CKHSzIjno", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "linear_classifier = train_linear_classifier_model(\n", + " learning_rate=0.1,\n", + " # TWEAK THE REGULARIZATION VALUE BELOW\n", + " regularization_strength=0.1,\n", + " steps=300,\n", + " batch_size=100,\n", + " feature_columns=construct_feature_columns(),\n", + " training_examples=training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=validation_examples,\n", + " validation_targets=validation_targets)\n", + "print(\"Model size:\", model_size(linear_classifier))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "yjUCX5LAkxAX", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Solution\n", + "\n", + "Click below to see a possible solution." + ] + }, + { + "metadata": { + "id": "hgGhy-okmkWL", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "A regularization strength of 0.1 should be sufficient. Note that there is a compromise to be struck:\n", + "stronger regularization gives us smaller models, but can affect the classification loss." + ] + }, + { + "metadata": { + "id": "_rV8YQWZIjns", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "linear_classifier = train_linear_classifier_model(\n", + " learning_rate=0.1,\n", + " regularization_strength=0.1,\n", + " steps=300,\n", + " batch_size=100,\n", + " feature_columns=construct_feature_columns(),\n", + " training_examples=training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=validation_examples,\n", + " validation_targets=validation_targets)\n", + "print(\"Model size:\", model_size(linear_classifier))" + ], + "execution_count": 0, + "outputs": [] + } + ] +} \ No newline at end of file diff --git a/synthetic_features_and_outliers.ipynb b/synthetic_features_and_outliers.ipynb new file mode 100644 index 0000000..1eb4442 --- /dev/null +++ b/synthetic_features_and_outliers.ipynb @@ -0,0 +1,604 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "synthetic_features_and_outliers.ipynb", + "version": "0.3.2", + "provenance": [], + "collapsed_sections": [ + "JndnmDMp66FL", + "i5Ul3zf5QYvW", + "jByCP8hDRZmM", + "WvgxW0bUSC-c" + ], + "include_colab_link": true + }, + "kernelspec": { + "name": "python2", + "display_name": "Python 2" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "metadata": { + "id": "JndnmDMp66FL", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "#### Copyright 2017 Google LLC." + ] + }, + { + "metadata": { + "id": "hMqWDc_m6rUC", + "colab_type": "code", + "cellView": "both", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "4f3CKqFUqL2-", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Synthetic Features and Outliers" + ] + }, + { + "metadata": { + "id": "jnKgkN5fHbGy", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "**Learning Objectives:**\n", + " * Create a synthetic feature that is the ratio of two other features\n", + " * Use this new feature as an input to a linear regression model\n", + " * Improve the effectiveness of the model by identifying and clipping (removing) outliers out of the input data" + ] + }, + { + "metadata": { + "id": "VOpLo5dcHbG0", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Let's revisit our model from the previous First Steps with TensorFlow exercise. \n", + "\n", + "First, we'll import the California housing data into a *pandas* `DataFrame`:" + ] + }, + { + "metadata": { + "id": "S8gm6BpqRRuh", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Setup" + ] + }, + { + "metadata": { + "id": "9D8GgUovHbG0", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "from __future__ import print_function\n", + "\n", + "import math\n", + "\n", + "from IPython import display\n", + "from matplotlib import cm\n", + "from matplotlib import gridspec\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "import sklearn.metrics as metrics\n", + "import tensorflow as tf\n", + "from tensorflow.python.data import Dataset\n", + "\n", + "tf.logging.set_verbosity(tf.logging.ERROR)\n", + "pd.options.display.max_rows = 10\n", + "pd.options.display.float_format = '{:.1f}'.format\n", + "\n", + "california_housing_dataframe = pd.read_csv(\"https://download.mlcc.google.com/mledu-datasets/california_housing_train.csv\", sep=\",\")\n", + "\n", + "california_housing_dataframe = california_housing_dataframe.reindex(\n", + " np.random.permutation(california_housing_dataframe.index))\n", + "california_housing_dataframe[\"median_house_value\"] /= 1000.0\n", + "california_housing_dataframe" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "I6kNgrwCO_ms", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Next, we'll set up our input function, and define the function for model training:" + ] + }, + { + "metadata": { + "id": "5RpTJER9XDub", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def my_input_fn(features, targets, batch_size=1, shuffle=True, num_epochs=None):\n", + " \"\"\"Trains a linear regression model of one feature.\n", + " \n", + " Args:\n", + " features: pandas DataFrame of features\n", + " targets: pandas DataFrame of targets\n", + " batch_size: Size of batches to be passed to the model\n", + " shuffle: True or False. Whether to shuffle the data.\n", + " num_epochs: Number of epochs for which data should be repeated. None = repeat indefinitely\n", + " Returns:\n", + " Tuple of (features, labels) for next data batch\n", + " \"\"\"\n", + " \n", + " # Convert pandas data into a dict of np arrays.\n", + " features = {key:np.array(value) for key,value in dict(features).items()} \n", + " \n", + " # Construct a dataset, and configure batching/repeating.\n", + " ds = Dataset.from_tensor_slices((features,targets)) # warning: 2GB limit\n", + " ds = ds.batch(batch_size).repeat(num_epochs)\n", + " \n", + " # Shuffle the data, if specified.\n", + " if shuffle:\n", + " ds = ds.shuffle(buffer_size=10000)\n", + " \n", + " # Return the next batch of data.\n", + " features, labels = ds.make_one_shot_iterator().get_next()\n", + " return features, labels" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "VgQPftrpHbG3", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def train_model(learning_rate, steps, batch_size, input_feature):\n", + " \"\"\"Trains a linear regression model.\n", + " \n", + " Args:\n", + " learning_rate: A `float`, the learning rate.\n", + " steps: A non-zero `int`, the total number of training steps. A training step\n", + " consists of a forward and backward pass using a single batch.\n", + " batch_size: A non-zero `int`, the batch size.\n", + " input_feature: A `string` specifying a column from `california_housing_dataframe`\n", + " to use as input feature.\n", + " \n", + " Returns:\n", + " A Pandas `DataFrame` containing targets and the corresponding predictions done\n", + " after training the model.\n", + " \"\"\"\n", + " \n", + " periods = 10\n", + " steps_per_period = steps / periods\n", + "\n", + " my_feature = input_feature\n", + " my_feature_data = california_housing_dataframe[[my_feature]].astype('float32')\n", + " my_label = \"median_house_value\"\n", + " targets = california_housing_dataframe[my_label].astype('float32')\n", + "\n", + " # Create input functions.\n", + " training_input_fn = lambda: my_input_fn(my_feature_data, targets, batch_size=batch_size)\n", + " predict_training_input_fn = lambda: my_input_fn(my_feature_data, targets, num_epochs=1, shuffle=False)\n", + " \n", + " # Create feature columns.\n", + " feature_columns = [tf.feature_column.numeric_column(my_feature)]\n", + " \n", + " # Create a linear regressor object.\n", + " my_optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n", + " my_optimizer = tf.contrib.estimator.clip_gradients_by_norm(my_optimizer, 5.0)\n", + " linear_regressor = tf.estimator.LinearRegressor(\n", + " feature_columns=feature_columns,\n", + " optimizer=my_optimizer\n", + " )\n", + "\n", + " # Set up to plot the state of our model's line each period.\n", + " plt.figure(figsize=(15, 6))\n", + " plt.subplot(1, 2, 1)\n", + " plt.title(\"Learned Line by Period\")\n", + " plt.ylabel(my_label)\n", + " plt.xlabel(my_feature)\n", + " sample = california_housing_dataframe.sample(n=300)\n", + " plt.scatter(sample[my_feature], sample[my_label])\n", + " colors = [cm.coolwarm(x) for x in np.linspace(-1, 1, periods)]\n", + "\n", + " # Train the model, but do so inside a loop so that we can periodically assess\n", + " # loss metrics.\n", + " print(\"Training model...\")\n", + " print(\"RMSE (on training data):\")\n", + " root_mean_squared_errors = []\n", + " for period in range (0, periods):\n", + " # Train the model, starting from the prior state.\n", + " linear_regressor.train(\n", + " input_fn=training_input_fn,\n", + " steps=steps_per_period,\n", + " )\n", + " # Take a break and compute predictions.\n", + " predictions = linear_regressor.predict(input_fn=predict_training_input_fn)\n", + " predictions = np.array([item['predictions'][0] for item in predictions])\n", + " \n", + " # Compute loss.\n", + " root_mean_squared_error = math.sqrt(\n", + " metrics.mean_squared_error(predictions, targets))\n", + " # Occasionally print the current loss.\n", + " print(\" period %02d : %0.2f\" % (period, root_mean_squared_error))\n", + " # Add the loss metrics from this period to our list.\n", + " root_mean_squared_errors.append(root_mean_squared_error)\n", + " # Finally, track the weights and biases over time.\n", + " # Apply some math to ensure that the data and line are plotted neatly.\n", + " y_extents = np.array([0, sample[my_label].max()])\n", + " \n", + " weight = linear_regressor.get_variable_value('linear/linear_model/%s/weights' % input_feature)[0]\n", + " bias = linear_regressor.get_variable_value('linear/linear_model/bias_weights')\n", + " \n", + " x_extents = (y_extents - bias) / weight\n", + " x_extents = np.maximum(np.minimum(x_extents,\n", + " sample[my_feature].max()),\n", + " sample[my_feature].min())\n", + " y_extents = weight * x_extents + bias\n", + " plt.plot(x_extents, y_extents, color=colors[period]) \n", + " print(\"Model training finished.\")\n", + "\n", + " # Output a graph of loss metrics over periods.\n", + " plt.subplot(1, 2, 2)\n", + " plt.ylabel('RMSE')\n", + " plt.xlabel('Periods')\n", + " plt.title(\"Root Mean Squared Error vs. Periods\")\n", + " plt.tight_layout()\n", + " plt.plot(root_mean_squared_errors)\n", + "\n", + " # Create a table with calibration data.\n", + " calibration_data = pd.DataFrame()\n", + " calibration_data[\"predictions\"] = pd.Series(predictions)\n", + " calibration_data[\"targets\"] = pd.Series(targets)\n", + " display.display(calibration_data.describe())\n", + "\n", + " print(\"Final RMSE (on training data): %0.2f\" % root_mean_squared_error)\n", + " \n", + " return calibration_data" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "FJ6xUNVRm-do", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Task 1: Try a Synthetic Feature\n", + "\n", + "Both the `total_rooms` and `population` features count totals for a given city block.\n", + "\n", + "But what if one city block were more densely populated than another? We can explore how block density relates to median house value by creating a synthetic feature that's a ratio of `total_rooms` and `population`.\n", + "\n", + "In the cell below, create a feature called `rooms_per_person`, and use that as the `input_feature` to `train_model()`.\n", + "\n", + "What's the best performance you can get with this single feature by tweaking the learning rate? (The better the performance, the better your regression line should fit the data, and the lower\n", + "the final RMSE should be.)" + ] + }, + { + "metadata": { + "id": "isONN2XK32Wo", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "**NOTE**: You may find it helpful to add a few code cells below so you can try out several different learning rates and compare the results. To add a new code cell, hover your cursor directly below the center of this cell, and click **CODE**." + ] + }, + { + "metadata": { + "id": "5ihcVutnnu1D", + "colab_type": "code", + "cellView": "both", + "colab": {} + }, + "cell_type": "code", + "source": [ + "#\n", + "# YOUR CODE HERE\n", + "#\n", + "california_housing_dataframe[\"rooms_per_person\"] = (\n", + " california_housing_dataframe[\"total_rooms\"] / california_housing_dataframe[\"population\"])\n", + "\n", + "calibration_data = train_model(\n", + " learning_rate=0.05,\n", + " steps=500,\n", + " batch_size=5,\n", + " input_feature=\"rooms_per_person\")" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "i5Ul3zf5QYvW", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Solution\n", + "\n", + "Click below for a solution." + ] + }, + { + "metadata": { + "id": "Leaz2oYMQcBf", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "california_housing_dataframe[\"rooms_per_person\"] = (\n", + " california_housing_dataframe[\"total_rooms\"] / california_housing_dataframe[\"population\"])\n", + "\n", + "calibration_data = train_model(\n", + " learning_rate=0.05,\n", + " steps=500,\n", + " batch_size=5,\n", + " input_feature=\"rooms_per_person\")" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "ZjQrZ8mcHFiU", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Task 2: Identify Outliers\n", + "\n", + "We can visualize the performance of our model by creating a scatter plot of predictions vs. target values. Ideally, these would lie on a perfectly correlated diagonal line.\n", + "\n", + "Use Pyplot's [`scatter()`](https://matplotlib.org/gallery/shapes_and_collections/scatter.html) to create a scatter plot of predictions vs. targets, using the rooms-per-person model you trained in Task 1.\n", + "\n", + "Do you see any oddities? Trace these back to the source data by looking at the distribution of values in `rooms_per_person`." + ] + }, + { + "metadata": { + "id": "P0BDOec4HbG_", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# YOUR CODE HERE\n", + "plt.figure(figsize=(15, 6))\n", + "plt.subplot(1, 2, 1)\n", + "plt.scatter(calibration_data[\"predictions\"], calibration_data[\"targets\"])\n", + "plt.subplot(1, 2, 2)\n", + "_ = california_housing_dataframe[\"rooms_per_person\"].hist()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "jByCP8hDRZmM", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Solution\n", + "\n", + "Click below for the solution." + ] + }, + { + "metadata": { + "id": "s0tiX2gdRe-S", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "plt.figure(figsize=(15, 6))\n", + "plt.subplot(1, 2, 1)\n", + "plt.scatter(calibration_data[\"predictions\"], calibration_data[\"targets\"])" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "kMQD0Uq3RqTX", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "The calibration data shows most scatter points aligned to a line. The line is almost vertical, but we'll come back to that later. Right now let's focus on the ones that deviate from the line. We notice that they are relatively few in number.\n", + "\n", + "If we plot a histogram of `rooms_per_person`, we find that we have a few outliers in our input data:" + ] + }, + { + "metadata": { + "id": "POTM8C_ER1Oc", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "plt.subplot(1, 2, 2)\n", + "_ = california_housing_dataframe[\"rooms_per_person\"].hist()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "9l0KYpBQu8ed", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Task 3: Clip Outliers\n", + "\n", + "See if you can further improve the model fit by setting the outlier values of `rooms_per_person` to some reasonable minimum or maximum.\n", + "\n", + "For reference, here's a quick example of how to apply a function to a Pandas `Series`:\n", + "\n", + " clipped_feature = my_dataframe[\"my_feature_name\"].apply(lambda x: max(x, 0))\n", + "\n", + "The above `clipped_feature` will have no values less than `0`." + ] + }, + { + "metadata": { + "id": "rGxjRoYlHbHC", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# YOUR CODE HERE\n", + "california_housing_dataframe[\"rooms_per_person\"] = (\n", + " california_housing_dataframe[\"rooms_per_person\"]).apply(lambda x: min(x, 5))\n", + "\n", + "_ = california_housing_dataframe[\"rooms_per_person\"].hist()\n", + "calibration_data = train_model(\n", + " learning_rate=0.05,\n", + " steps=500,\n", + " batch_size=5,\n", + " input_feature=\"rooms_per_person\")\n", + "_ = plt.scatter(calibration_data[\"predictions\"], calibration_data[\"targets\"])" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "WvgxW0bUSC-c", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Solution\n", + "\n", + "Click below for the solution." + ] + }, + { + "metadata": { + "id": "8YGNjXPaSMPV", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "The histogram we created in Task 2 shows that the majority of values are less than `5`. Let's clip `rooms_per_person` to 5, and plot a histogram to double-check the results." + ] + }, + { + "metadata": { + "id": "9YyARz6gSR7Q", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "california_housing_dataframe[\"rooms_per_person\"] = (\n", + " california_housing_dataframe[\"rooms_per_person\"]).apply(lambda x: min(x, 5))\n", + "\n", + "_ = california_housing_dataframe[\"rooms_per_person\"].hist()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "vO0e1p_aSgKA", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "To verify that clipping worked, let's train again and print the calibration data once more:" + ] + }, + { + "metadata": { + "id": "ZgSP2HKfSoOH", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "calibration_data = train_model(\n", + " learning_rate=0.05,\n", + " steps=500,\n", + " batch_size=5,\n", + " input_feature=\"rooms_per_person\")" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "gySE-UgfSony", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "_ = plt.scatter(calibration_data[\"predictions\"], calibration_data[\"targets\"])" + ], + "execution_count": 0, + "outputs": [] + } + ] +} \ No newline at end of file diff --git a/validation.ipynb b/validation.ipynb new file mode 100644 index 0000000..423ce3d --- /dev/null +++ b/validation.ipynb @@ -0,0 +1,1538 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "validation.ipynb", + "version": "0.3.2", + "provenance": [], + "collapsed_sections": [ + "JndnmDMp66FL", + "4Xp9NhOCYSuz", + "pECTKgw5ZvFK", + "dER2_43pWj1T", + "I-La4N9ObC1x", + "yTghc_5HkJDW" + ], + "include_colab_link": true + }, + "kernelspec": { + "name": "python2", + "display_name": "Python 2" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "metadata": { + "id": "JndnmDMp66FL", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "#### Copyright 2017 Google LLC." + ] + }, + { + "metadata": { + "id": "hMqWDc_m6rUC", + "colab_type": "code", + "cellView": "both", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "zbIgBK-oXHO7", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Validation" + ] + }, + { + "metadata": { + "id": "WNX0VyBpHpCX", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "**Learning Objectives:**\n", + " * Use multiple features, instead of a single feature, to further improve the effectiveness of a model\n", + " * Debug issues in model input data\n", + " * Use a test data set to check if a model is overfitting the validation data" + ] + }, + { + "metadata": { + "id": "za0m1T8CHpCY", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "As in the prior exercises, we're working with the [California housing data set](https://developers.google.com/machine-learning/crash-course/california-housing-data-description), to try and predict `median_house_value` at the city block level from 1990 census data." + ] + }, + { + "metadata": { + "id": "r2zgMfWDWF12", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Setup" + ] + }, + { + "metadata": { + "id": "8jErhkLzWI1B", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "First off, let's load up and prepare our data. This time, we're going to work with multiple features, so we'll modularize the logic for preprocessing the features a bit:" + ] + }, + { + "metadata": { + "id": "PwS5Bhm6HpCZ", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "from __future__ import print_function\n", + "\n", + "import math\n", + "\n", + "from IPython import display\n", + "from matplotlib import cm\n", + "from matplotlib import gridspec\n", + "from matplotlib import pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "from sklearn import metrics\n", + "import tensorflow as tf\n", + "from tensorflow.python.data import Dataset\n", + "\n", + "tf.logging.set_verbosity(tf.logging.ERROR)\n", + "pd.options.display.max_rows = 10\n", + "pd.options.display.float_format = '{:.1f}'.format\n", + "\n", + "california_housing_dataframe = pd.read_csv(\"https://download.mlcc.google.com/mledu-datasets/california_housing_train.csv\", sep=\",\")\n", + "\n", + "# california_housing_dataframe = california_housing_dataframe.reindex(\n", + "# np.random.permutation(california_housing_dataframe.index))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "J2ZyTzX0HpCc", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def preprocess_features(california_housing_dataframe):\n", + " \"\"\"Prepares input features from California housing data set.\n", + "\n", + " Args:\n", + " california_housing_dataframe: A Pandas DataFrame expected to contain data\n", + " from the California housing data set.\n", + " Returns:\n", + " A DataFrame that contains the features to be used for the model, including\n", + " synthetic features.\n", + " \"\"\"\n", + " selected_features = california_housing_dataframe[\n", + " [\"latitude\",\n", + " \"longitude\",\n", + " \"housing_median_age\",\n", + " \"total_rooms\",\n", + " \"total_bedrooms\",\n", + " \"population\",\n", + " \"households\",\n", + " \"median_income\"]]\n", + " processed_features = selected_features.copy()\n", + " # Create a synthetic feature.\n", + " processed_features[\"rooms_per_person\"] = (\n", + " california_housing_dataframe[\"total_rooms\"] /\n", + " california_housing_dataframe[\"population\"])\n", + " return processed_features\n", + "\n", + "def preprocess_targets(california_housing_dataframe):\n", + " \"\"\"Prepares target features (i.e., labels) from California housing data set.\n", + "\n", + " Args:\n", + " california_housing_dataframe: A Pandas DataFrame expected to contain data\n", + " from the California housing data set.\n", + " Returns:\n", + " A DataFrame that contains the target feature.\n", + " \"\"\"\n", + " output_targets = pd.DataFrame()\n", + " # Scale the target to be in units of thousands of dollars.\n", + " output_targets[\"median_house_value\"] = (\n", + " california_housing_dataframe[\"median_house_value\"] / 1000.0)\n", + " return output_targets" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "sZSIaDiaHpCf", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "For the **training set**, we'll choose the first 12000 examples, out of the total of 17000." + ] + }, + { + "metadata": { + "id": "P9wejvw7HpCf", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 284 + }, + "outputId": "adb58bf4-9238-402e-dcb1-3b6b27d15d41" + }, + "cell_type": "code", + "source": [ + "training_examples = preprocess_features(california_housing_dataframe.head(12000))\n", + "training_examples.describe()" + ], + "execution_count": 3, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
latitudelongitudehousing_median_agetotal_roomstotal_bedroomspopulationhouseholdsmedian_incomerooms_per_person
count12000.012000.012000.012000.012000.012000.012000.012000.012000.0
mean34.6-118.527.52655.7547.11476.0505.43.81.9
std1.61.212.12258.1434.31174.3391.71.91.3
min32.5-121.41.02.02.03.02.00.50.0
25%33.8-118.917.01451.8299.0815.0283.02.51.4
50%34.0-118.228.02113.5438.01207.0411.03.51.9
75%34.4-117.836.03146.0653.01777.0606.04.62.3
max41.8-114.352.037937.05471.035682.05189.015.055.2
\n", + "
" + ], + "text/plain": [ + " latitude longitude housing_median_age total_rooms total_bedrooms \\\n", + "count 12000.0 12000.0 12000.0 12000.0 12000.0 \n", + "mean 34.6 -118.5 27.5 2655.7 547.1 \n", + "std 1.6 1.2 12.1 2258.1 434.3 \n", + "min 32.5 -121.4 1.0 2.0 2.0 \n", + "25% 33.8 -118.9 17.0 1451.8 299.0 \n", + "50% 34.0 -118.2 28.0 2113.5 438.0 \n", + "75% 34.4 -117.8 36.0 3146.0 653.0 \n", + "max 41.8 -114.3 52.0 37937.0 5471.0 \n", + "\n", + " population households median_income rooms_per_person \n", + "count 12000.0 12000.0 12000.0 12000.0 \n", + "mean 1476.0 505.4 3.8 1.9 \n", + "std 1174.3 391.7 1.9 1.3 \n", + "min 3.0 2.0 0.5 0.0 \n", + "25% 815.0 283.0 2.5 1.4 \n", + "50% 1207.0 411.0 3.5 1.9 \n", + "75% 1777.0 606.0 4.6 2.3 \n", + "max 35682.0 5189.0 15.0 55.2 " + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 3 + } + ] + }, + { + "metadata": { + "id": "JlkgPR-SHpCh", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 284 + }, + "outputId": "1c160054-ce2a-42f5-c50f-47deaef651ce" + }, + "cell_type": "code", + "source": [ + "training_targets = preprocess_targets(california_housing_dataframe.head(12000))\n", + "training_targets.describe()" + ], + "execution_count": 4, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
median_house_value
count12000.0
mean198.0
std111.9
min15.0
25%117.1
50%170.5
75%244.4
max500.0
\n", + "
" + ], + "text/plain": [ + " median_house_value\n", + "count 12000.0\n", + "mean 198.0\n", + "std 111.9\n", + "min 15.0\n", + "25% 117.1\n", + "50% 170.5\n", + "75% 244.4\n", + "max 500.0" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 4 + } + ] + }, + { + "metadata": { + "id": "5l1aA2xOHpCj", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "For the **validation set**, we'll choose the last 5000 examples, out of the total of 17000." + ] + }, + { + "metadata": { + "id": "fLYXLWAiHpCk", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 284 + }, + "outputId": "7e627bd6-3555-44c5-fd9d-360a9e29bbbe" + }, + "cell_type": "code", + "source": [ + "validation_examples = preprocess_features(california_housing_dataframe.tail(5000))\n", + "validation_examples.describe()" + ], + "execution_count": 5, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
latitudelongitudehousing_median_agetotal_roomstotal_bedroomspopulationhouseholdsmedian_incomerooms_per_person
count5000.05000.05000.05000.05000.05000.05000.05000.05000.0
mean38.1-122.231.32614.8521.11318.1491.24.12.1
std0.90.513.41979.6388.51073.7366.52.00.6
min36.1-124.31.08.01.08.01.00.50.1
25%37.5-122.420.01481.0292.0731.0278.02.71.7
50%37.8-122.131.02164.0424.01074.0403.03.72.1
75%38.4-121.942.03161.2635.01590.2603.05.12.4
max42.0-121.452.032627.06445.028566.06082.015.018.3
\n", + "
" + ], + "text/plain": [ + " latitude longitude housing_median_age total_rooms total_bedrooms \\\n", + "count 5000.0 5000.0 5000.0 5000.0 5000.0 \n", + "mean 38.1 -122.2 31.3 2614.8 521.1 \n", + "std 0.9 0.5 13.4 1979.6 388.5 \n", + "min 36.1 -124.3 1.0 8.0 1.0 \n", + "25% 37.5 -122.4 20.0 1481.0 292.0 \n", + "50% 37.8 -122.1 31.0 2164.0 424.0 \n", + "75% 38.4 -121.9 42.0 3161.2 635.0 \n", + "max 42.0 -121.4 52.0 32627.0 6445.0 \n", + "\n", + " population households median_income rooms_per_person \n", + "count 5000.0 5000.0 5000.0 5000.0 \n", + "mean 1318.1 491.2 4.1 2.1 \n", + "std 1073.7 366.5 2.0 0.6 \n", + "min 8.0 1.0 0.5 0.1 \n", + "25% 731.0 278.0 2.7 1.7 \n", + "50% 1074.0 403.0 3.7 2.1 \n", + "75% 1590.2 603.0 5.1 2.4 \n", + "max 28566.0 6082.0 15.0 18.3 " + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 5 + } + ] + }, + { + "metadata": { + "id": "oVPcIT3BHpCm", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 284 + }, + "outputId": "590e532a-5aae-42fd-b623-67a88f3fc2d9" + }, + "cell_type": "code", + "source": [ + "validation_targets = preprocess_targets(california_housing_dataframe.tail(5000))\n", + "validation_targets.describe()" + ], + "execution_count": 6, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
median_house_value
count5000.0
mean229.5
std122.5
min15.0
25%130.4
50%213.0
75%303.2
max500.0
\n", + "
" + ], + "text/plain": [ + " median_house_value\n", + "count 5000.0\n", + "mean 229.5\n", + "std 122.5\n", + "min 15.0\n", + "25% 130.4\n", + "50% 213.0\n", + "75% 303.2\n", + "max 500.0" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 6 + } + ] + }, + { + "metadata": { + "id": "z3TZV1pgfZ1n", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Task 1: Examine the Data\n", + "Okay, let's look at the data above. We have `9` input features that we can use.\n", + "\n", + "Take a quick skim over the table of values. Everything look okay? See how many issues you can spot. Don't worry if you don't have a background in statistics; common sense will get you far.\n", + "\n", + "After you've had a chance to look over the data yourself, check the solution for some additional thoughts on how to verify data." + ] + }, + { + "metadata": { + "id": "4Xp9NhOCYSuz", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Solution\n", + "\n", + "Click below for the solution." + ] + }, + { + "metadata": { + "id": "gqeRmK57YWpy", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Let's check our data against some baseline expectations:\n", + "\n", + "* For some values, like `median_house_value`, we can check to see if these values fall within reasonable ranges (keeping in mind this was 1990 data — not today!).\n", + "\n", + "* For other values, like `latitude` and `longitude`, we can do a quick check to see if these line up with expected values from a quick Google search.\n", + "\n", + "If you look closely, you may see some oddities:\n", + "\n", + "* `median_income` is on a scale from about 3 to 15. It's not at all clear what this scale refers to—looks like maybe some log scale? It's not documented anywhere; all we can assume is that higher values correspond to higher income.\n", + "\n", + "* The maximum `median_house_value` is 500,001. This looks like an artificial cap of some kind.\n", + "\n", + "* Our `rooms_per_person` feature is generally on a sane scale, with a 75th percentile value of about 2. But there are some very large values, like 18 or 55, which may show some amount of corruption in the data.\n", + "\n", + "We'll use these features as given for now. But hopefully these kinds of examples can help to build a little intuition about how to check data that comes to you from an unknown source." + ] + }, + { + "metadata": { + "id": "fXliy7FYZZRm", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Task 2: Plot Latitude/Longitude vs. Median House Value" + ] + }, + { + "metadata": { + "id": "aJIWKBdfsDjg", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Let's take a close look at two features in particular: **`latitude`** and **`longitude`**. These are geographical coordinates of the city block in question.\n", + "\n", + "This might make a nice visualization — let's plot `latitude` and `longitude`, and use color to show the `median_house_value`." + ] + }, + { + "metadata": { + "id": "5_LD23bJ06TW", + "colab_type": "code", + "cellView": "both", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 498 + }, + "outputId": "9b7b9acc-17ed-432b-b745-92469070817c" + }, + "cell_type": "code", + "source": [ + "plt.figure(figsize=(13, 8))\n", + "\n", + "ax = plt.subplot(1, 2, 1)\n", + "ax.set_title(\"Validation Data\")\n", + "\n", + "ax.set_autoscaley_on(False)\n", + "ax.set_ylim([32, 43])\n", + "ax.set_autoscalex_on(False)\n", + "ax.set_xlim([-126, -112])\n", + "plt.scatter(validation_examples[\"longitude\"],\n", + " validation_examples[\"latitude\"],\n", + " cmap=\"coolwarm\",\n", + " c=validation_targets[\"median_house_value\"] / validation_targets[\"median_house_value\"].max())\n", + "\n", + "ax = plt.subplot(1,2,2)\n", + "ax.set_title(\"Training Data\")\n", + "\n", + "ax.set_autoscaley_on(False)\n", + "ax.set_ylim([32, 43])\n", + "ax.set_autoscalex_on(False)\n", + "ax.set_xlim([-126, -112])\n", + "plt.scatter(training_examples[\"longitude\"],\n", + " training_examples[\"latitude\"],\n", + " cmap=\"coolwarm\",\n", + " c=training_targets[\"median_house_value\"] / training_targets[\"median_house_value\"].max())\n", + "_ = plt.plot()" + ], + "execution_count": 7, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAwEAAAHhCAYAAAA2xLK+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3Xec3VWd+P/X+ZTbprdk0jupJJSQ\nhBBaCCkIIsgqRZTV3fVnd3dF3VUerq6u7vLgqw99iOt+97uIIixqqBKqoUkJ6ZDek8lkJtNnbr+f\ncn5/3Gk3cyeZNBDu+/l4wGNyy+dz7p3knPM+5X2U1lojhBBCCCGEKBjGe10AIYQQQgghxLtLggAh\nhBBCCCEKjAQBQgghhBBCFBgJAoQQQgghhCgwEgQIIYQQQghRYCQIEEIIIYQQosBIECDOiltuuYXf\n/va3Ax5/+OGHueWWW4773p/97Gd861vfAuBTn/oUW7duHfCadevWsXjx4hOWY/PmzezYsQOABx54\ngJ/85CdDKf6QLF68mCuvvJLly5dz2WWX8clPfpKXX355SO/dt28fa9euPWNlEUKID4rvfOc7LF++\nnOXLlzNz5szeenb58uXEYrGTutby5ctpaWk57mvuueceHnroodMpco6pU6dy9dVXs2zZMi677DI+\n+9nPsnHjxiG9t3+bJcTZZr3XBRAfTDfeeCMrV67ktttuy3n88ccf58Ybbxzyde6///7TKsfKlSu5\n8MILmTZtGp/4xCdO61r53H333cydOxeAN954g3/6p3/ia1/7Gtdee+1x3/fCCy/gui4XXXTRGS+T\nEEK8n333u9/t/Xnx4sX8x3/8R289e7KeeeaZE77mH//xH0/p2sfzm9/8htraWrTWPPPMM3z+85/n\npz/96Qnr/P5tlhBnm8wEiLNixYoV7Nixg7q6ut7HDh8+zPbt21mxYgUAv//971mxYgVLly7ltttu\no76+fsB1Fi9ezLp16wC49957ufzyy/nIRz7C66+/3vuaZDLJV7/6VZYtW8bixYv593//dwAeeugh\nHn/8ce6++27uu+++nBmGI0eO8JnPfIZly5Zx7bXX8thjj/WWcdGiRfz617/muuuu49JLL2XVqlVD\n+swXX3wx//Zv/8Z//Md/oLXG932++93v9pbrzjvvxHEcVq9ezS9/+Ut+/etf86Mf/QiAn//85yxb\ntowlS5bw2c9+lq6urpP9yoUQoiDcfvvt/PjHP2bFihVs2LCBlpYWPvOZz7B8+XIWL17Mfffd1/va\nqVOn0tjYyJo1a/j4xz/OPffcw4oVK1i8eDFvvfUWAN/85je59957gWyb87//+7/cdNNNLFq0qLeO\nBvjP//xPLr74Yj760Y/y29/+dkiz0UopVqxYwT/8wz9wzz33AENvswZrQ4Q4UyQIEGdFcXExS5Ys\n4fHHH+997Mknn+Sqq66iuLiY1tZWvve973Hffffx3HPPMXbs2N5KOJ89e/bwq1/9ipUrV7Jy5Up2\n7tzZ+9xDDz1EPB7nmWee4dFHH+WRRx5h3bp13HLLLcyePZs777yTv/7rv8653l133cW8efN49tln\n+eUvf8n3v/99Dh8+DEB7ezuGYfDkk0/yz//8zye1hGjBggVEo1H279/P888/z7p16/jjH//I008/\nzdatW1m1ahWLFy/m6quv5pOf/CTf/OY32bJlC7/97W9ZuXIlzz33HJlMhgceeGDI9xRCiEKzZcsW\nnnrqKS644AJ+8YtfMHr0aJ555hnuv/9+7rnnHhoaGga8Z9u2bcyZM4enn36aW2+9lV/84hd5r712\n7VoefvhhVq5cyQMPPEBjYyO7d+/mv//7v3n88cd58MEHhzTD0N/ixYvZvHkzqVRqyG3WYG2IEGeK\nBAHirLnxxht58skne//8xBNP9C4FqqqqYv369dTW1gIwd+7cnFmDY61du5aLLrqI6upqTNPkwx/+\ncO9zn/70p7n33ntRSlFWVsaUKVN6O/T5OI7D66+/zq233grAqFGjmD9/Pm+++SYAruv2lnPmzJkc\nOXJkyJ/ZMAwikQixWIxly5axcuVKbNsmGAxy7rnn5v2Ms2bN4qWXXqK4uBjDMDj//POP+10IIUSh\nu/zyyzGMbBfm29/+NnfddRcAY8aMoaamJm8bUFRUxJIlS4Dj1+3XXXcdpmkyfPhwqqqqaGhoYO3a\ntcybN49hw4YRDAb56Ec/elLlLS4uxvd94vH4kNusobYhQpwq2RMgzpoFCxaQTqfZvHkzhmGQTCZZ\nsGABAJ7n8dOf/pTVq1fjeR7xeJwJEyYMeq3Ozk5KSkp6/1xaWtr784EDB/jRj37Evn37MAyDxsbG\n4+476OjoQGs94HptbW0AmKZJJBIBsp163/eH/JlTqRStra1UVlbS1tbGv/7rv7Jt2zaUUrS0tPCp\nT31qwHuSySQ//OEPWbNmTe9nveKKK4Z8TyGEKDRlZWW9P7/zzju9o/+GYdDc3Jy33u5f5x+vbi8u\nLu792TRNPM+jq6sr557Dhw8/qfIePnwY27YpKSkZcps11DZEiFMlQYA4awzD4Prrr+ePf/wjpmly\n/fXX947crFq1itWrV/PAAw9QWVnJ7373u5xZg2OVlpYSjUZ7/9ze3t778/e+9z1mzpzJz3/+c0zT\n5Oabbz5uuSoqKjAMg87Ozt5KvaOjg6qqqtP5uAA8++yzjBs3jtGjR3PXXXdhWRZPPvkkgUBg0M1n\n999/PwcOHOCRRx6hqKiIH//4xxw9evS0yyKEEIXgzjvv5FOf+hS33HILSikuvfTSM36P4uJiEolE\n75+bmppO6v3PPvss8+bNIxAIDLnN+vGPfzykNkSIUyXLgcRZdeONN7J69Wr+9Kc/5Yx0tLa2MmrU\nKCorK2lvb+fpp58mHo8Pep3zzz+f9evX09bWhud5PPHEEznXmj59OqZp8tprr3Hw4MHeytqyrJzg\noeexRYsW8fDDDwNw6NAh1q1bx8KFC0/rs65Zs4a7776br3/9673lOueccwgEAuzYsYONGzfmLVdr\naysTJ06kqKiI+vp6Xn755ZzGRgghxOBaW1uZNWsWSikeffRRksnkGa9DZ8+ezZo1a2hrayOTyfQm\nkziRnuxA999/P3//93/fW96htFnHa0OEOBNkJkCcVePGjWPYsGG9P/e49tpreeqpp7j66qsZM2YM\nX/3qV/nc5z7Hj370I4qKigZcZ/r06dx8883ccMMNlJeX86EPfYhdu3YB8LnPfY4f/vCH3HvvvVx1\n1VV88Ytf5Kc//SnTp09nyZIl3H333dTV1eVM8X73u9/l29/+No888gi2bfP973+fESNGHHcvQT53\n3nknwWCQeDzOiBEj+MEPfsDll18OZPcqfOMb3+CRRx5h7ty5fOMb3+Bb3/oWs2fP5sorr+RrX/sa\n9fX1fOUrX+HLX/4yy5YtY+rUqXzzm9/kS1/6Er/61a+44447TvYrF0KIgvKVr3yFL3zhC5SXl3Pz\nzTfz8Y9/nLvuuosHH3zwjN1j9uzZ3HDDDdxwww2MGDGCa665hl/96leDvv7222/HNE1isRiTJk3i\nv/7rvzj33HOBobdZx2tDerLsCXE6lNZav9eFEEIIIYT4S6a1RikFwEsvvcRPfvKTIc8ICPGXSJYD\nCSGEEEIcR1tbGwsWLKC+vh6tNU8//TTnnXfee10sIU6LzAQIIYQQQpzAQw89xP/8z/+glGLixIn8\n4Ac/OCMJJYR4r0gQIIQQQgghRIGR5UBCCCGEEEIUGAkChBBCCCGEKDDvSopQ1/Vob3//5ratqIi8\nb8svZX9vSNnfG+/nstfUlJz4RQXg/dxevJ///knZ3zvv5/JL2d8bZ6q9eFdmAizLfDduc9a8n8sv\nZX9vSNnfG+/nsous9/PvUMr+3ng/lx3e3+WXsr+/yXIgIYQQQgghCowEAUIIIYQQQhQYCQKEEEII\nIYQoMBIECCGEEEIIUWAkCBBCCCGEEKLASBAghBBCCCFEgZEgQAghhBBCiAIjQYAQQgghhBAFRoIA\nIYQQQgghCowEAUIIIYQQQhQYCQKEEEIIIYQoMBIECCGEEEIIUWAkCBBCCCGEEKLASBAghBBCCCFE\ngZEgQAghhBBCiAIjQYAQQgghhBAFRoIAIYQQQgghCowEAUIIIYQQQhQYCQKEEEIIIYQoMBIECCGE\nEEIIUWAkCBBCCCGEEKLASBAghBBCCCFEgZEgQAghhBBCiAIjQYAQQgghhBAFRoIAIYQQQgghCowE\nAUIIIYQQQhQYCQKEEEIIIYQoMBIECCGEEEIIUWAkCBBCCCGEEKLASBAghBBCCCFEgZEgQAghhBBC\niAIjQYAQQgghhBAFRoIAIYQQQgghCowEAUIIIYQQQhSYIQUBqVSKJUuW8Mgjj9DQ0MAdd9zBJz7x\nCe644w6am5vPdhmFEEK8T0h7IYQQ7w9DCgJ+8YtfUFZWBsBPfvITPvaxj/HAAw9w9dVXc999953V\nAr7bOuOKhnaF673XJYFEymfXwQzt0b+AwgghxBAUUnshhBDvZ9aJXrB371727NnDFVdcAcB3vvMd\ngsEgABUVFWzduvWsFvDd0pWEV7cHaOgwcH1Fadhn2kiPCya473pZfF/z++djbNyeoTPmEwnB9IlB\nbvtQMeGgrOASQvxlKpT2QgghPghOGAT8+7//O3fddRePPfYYAJFIBADP83jwwQf5whe+MKQb1dSU\nnEYxzy6tNX98weNwW99jXUmDdfsMhlUFqal5d8v/wBOtvLQ21fvnRArWb0sTDFj8w18PP+nr/SV/\n9yciZX9vSNnFqSiE9uJEpOzvjfdz2eH9XX4p+/vXcYOAxx57jPPOO48xY8bkPO55Hl//+tdZsGAB\nF1988ZBu1NwcPfVSnmX7mwwOtwQAlfO4r2HjHofzJwVpaoqyt1HRGjMoDWumjvQxzsKgvO9r3twU\ny/vcph1xdu5pp7LshLFbr5qakr/o7/54pOzvDSn7e+P93hgVSntxPO/3v39S9vfG+7n8Uvb3xplq\nL47bm3zppZeoq6vjpZdeorGxkUAgQG1tLY899hjjxo3ji1/84hkpxHutI2FwbADQI5mGWNLn0TU2\nR9r7XvfOQZ8lcxwqi/Wg1/W1Zt02h12HXJSCaeMszp9mY6j89wJIO5po3M/7XCIFjS0eZcUmr21O\nsb/ewzRh9uQAs88JDHj9vsNpfv98jLa2NCNqLK5eWEI4JMuJhBBnXqG0F0II8UFx3CDgJz/5Se/P\nP/vZzxg1ahQtLS3Yts2Xv/zls164s23nYdh6yKA9pkkmHayAiW3ndpJLwpqn1/ocaTdzHm/qMnh1\nm8X185y81/a15jerkmzc2benYO02l+0HPG5bHkINEggEA4rKMpN4auBehNIixYgak1+ujLJtf7/r\nbs1w6QVBbrqqqPexF9+K8YfnOkmm+oKUDdtTfPHWKqorhj6TIIQQQ/FBby/Eu8dxNJ6vCckeOCHO\nqpPuDT744IOk02luv/12ACZNmsS//Mu/nOlynXWb9ileetsgldF4no9pKlwXMoaHYShMSxGJKCbX\nery+08x7jYZ2g464orwodzbA15rHX0ywbksGwzJyOvzrtzvMmmhx3lQ77zUNpbhoVpD6Jhf/mEmG\nQEDx21Vxth9wc67p+fDa5jQXTgsyYZRFOuOz6tVoTgAAcKjB4fEXu/jMjZUn81UJIcQp+aC0F+Ld\n0dru8vvnu9hzyMF1NWNH2CxbWMTMKaH3umhCfCANOQj40pe+BMCNN9541grzbvE1rN+l6Yw6eF5f\nR9k0FQHbQGuFHTCpjPiMqvBxBkkQ5PqK1DETATsPZnj0hRgHG7rTejpgWiZWIPtVa2D7QWfQIABg\nyYIwAG9tSdHY4vWmK23p8GntzN5QK43Rb1OC48CmnWkmjLJYszlBa3v+tKL76zLZ78DXvPl2giNH\nHUpLTC6fW0QwIKMuQojT90FqL8TJaW53eW1DHNeDOdNCTBkbHNL7XE/zy993sL++r1Hdvi/DkWaX\nL95iMH5U35LXzbvSvLE5TVunR2mxweIFMGP8mf4kQnzwFeS6kM441Lf4OQEAgOdpMvjEo0mCoQBa\nh9hzRDGsHA41DbxORZFPTWnfNdIZzUOrohxtzV3T77keylCYVnZGQQ2y/wCgsR027FHEnQjTpgVp\nW9eBm/AHXT6Uo+c1J3htV8zl5w+1sftgpvexl9+Kc8cN5UwZJyMuQgghTt5zr0f540tRYolsG/j8\nG1Eunh3hUx+pOGEb9vrGRE4A0KMz6vPS2gR3dAcBa95O8fBzMVI9zVeTx+5DLVx3WYQlCyJn9PMI\n8UFXkEO/vq8HBAA9PC/7XDyaIpV0qGuG+dNMQlbu6y1DM2usi9nvG3x1Q3JAANB7Tzf7uFIwfUL+\n2Gt7HTy4Gjbszu5X2F5nUlpdTlFZBDt4zMzBsUuFLLhgWraSnD87TE1F/iVMk8YE+N0znTkBAEBD\ni8vvnulE68E3OgshhBD5NLY4PPliV28AANkZ6lfWJ3hlXXwI7x/8UMy2zuxzWmte3pDqCwB67uPC\n65tTOK60X0KcjIIMAgCO19c1TQMNpJIZNDBrvMGy8zNMrnUZXuYxYZjHktkOc8bndvgHy+oDoNEo\nYN4Mm9mT+4KAjKN54+0Ub7yd5rV3NIn0sWUxCYdtfN8fGAh0s0y49IIQ40ZkrxsMGFxzWQlFodyR\nl3EjbT68uIRdBzL5LsP+eoe9h/I/J4QQQgzmzxsSxJP5G9Z3dqXyPt5fecng3ZHS4uxzsYSmsSX/\n+tzGVp/6pnf/cE8h3s8KcjlQWRFUlEB7nvSwvuejNViWied64HvsqIe2pM24WphQ7RCy81d0Y2oH\n/zqHVZp8ZHGYOVOs3mnRP29K8ae30rR2ZoMHw0xRVBIkXJS7JMcOWGit8b2+IGPWZJuSIhPLhDnn\nBJgxMTdF6OUXFTNhdID12xxa21OMHGZz1YJiTEORGWS0xPez6VCFEEKIk+EeZxTeGWTmvb/LL4rw\n6obkgE5+OAgLz8vukwsGFKGAQSozsJ0K2lBSNIRls0KIXgU5E2CZivMmKY5dU6O1znb8uynT4FAL\nvLwV9rYE2d4Y5MWdYRo68y+1uWBGkHPGDQwEKkoNPvORYs47x+4NAA42ujz5arI3AIBsABLrTOJk\ncitB3f0/3/exAiajam3+5oZSrpgbAqV4eUOa3z4dH7CecuyIAH93cy1/c1MV11xWSjBgYFmKsSPy\nzygMrzSZOUn2BAghhDg50ycFBz1Ac7A2p79gwOCvbyhjyjgbs7uJHTXM4qZlpczobpcCtmJKnjYW\nYPJYm6qTOEhTCFGgMwEAV8xR7K7zOdgESqnuAMDHc/s65eGwjX/MV5R0TLYeCTK8NIFxzKCDoRSf\nvamUR/4UZ/chB8eB0bUWVy8IM6Y2txJc806aZJ4ZUq0hmUhjB/ru66Td3rX62tdEIja7Djo89FyC\naLwvkNmyJ8NNSyJcOP342RiWLyrhcKNDR7TvswYCcOX8YmxbRlKEEEKcnNnnhLhwRpi1W5I5j08Y\nZbPskqGdbjpxdIA7/7qK+qMuqbRmwmgb08xtk/5qaRHRhGb3QQfPzx7fOXVCkI8vlU3BQpysgg0C\nlFJ8crnFv/x3Ci/PChjDAO37DK8dWLF0pUyOtFuMrhy4/jAcMrhgZpiRw4OMGW4wZWz+EZBEavDp\nUd2vPE7GJdrZt6lKGQZNbZrn3kznBAAA8RSsXpvm/GmB455KPGNSiC9/oprVa2I0t7uUFBlcPDvC\n+TOkEhVCCHHylFJ89mOVjB8VZce+NK6nGTcywDWXllAcyT973t+uA2n+vCFBR9SnotTgsrlFAwIA\ngOKwyZduLmX7Poe6Rpfh1SZXLayitTV2Nj6WEB9oBRsEAIRsg6XzLZ5d4+L3dLxVNoNPSRgmTAzn\nrYQAnDyBQ1uXx2+fTrHviJfdV2DAlLEZPnlNmHAod560tsoE8p82HDB94tEUnueRiKZ7ZwGUUtgB\ni2AQjrTm3wB1+KjH0VaPEdXH/9WOHxXg03JomBBCiDPEMBQrLi1lxaUn9741byd5aFVnzsbit3en\nuf3aMi6YER7weqUUMyYFmDEp0HtfIcTJK8g9Af0tviDAHdcEuWCazcTRJuNrTZbONfnm7WEmj8k/\nih+2PUZXDOyEr1ydYm+915t5yPVh+wGPR14cuO7n8gtDjKoZ+PWPHm7ytdsijC7PEO9K9QUAhiIY\nDqCUYuJIE3uQ35xpQcCSClEIIcRfHq11Tipq39c8/0ZsQGahWFzz/BtxSVstxFlU0DMBPaaPM5k+\nbuB05dThaTqSBvF033Om0kyqcbCPeXlb1GNPXf48x7vrPDKOJtBvvX04qPjbG4pZ9VqKgw3ZgGL8\nSIsPXRKivNTkCx8vpaHF5eEXUhxpAV8rLBNqqxRJR2FFwoQtjet6OP2OLZ440qSq/MRTr0IIIcS7\n5WB9msf/1M6+ujSGAVPGhfirayrxXKhryD+zfajBoTPqU14qbZoQZ4MEAcdRHtEsmpSkPlZMc7uD\nbcGYCocRZQM7+7G4Jp1/dQ/JjB4QBABUlpl84pqiQe8/otriKx8vYu9hnyOtHr4Pr7wDTV0ACtMC\n0zIxDEU6kWF4lcF1lw2cOu3huppX13fR2eUxbVKYaRMHf60QQghxJrR3uvzsN4056T+bWmPUNzl8\n6fbh2BZ520/bUtgysy3EWSNBwAkUBTWXjIbm5uMfdjKi2qSm3KC5Y+BmgeGVBkXhgRWZ42re3ObR\n2JrNznPBZIMxw3PX+SilmDzGZPIYk/ufcUhlBk6NBoMWC2eZXHNxkGAgf4W5c3+S+/7QzOGj2Zo2\nYLUzZ3oRn79tuFSyQgghzppnXu3Ie8jXgcNp1m+JMWlsgG17Bx5UOWlMgKJIwa9aFuKskX9dZ4ht\nKebPsrGO+UYDNiw8N9B7PkCPeMrn/z3l8vQan417fNZs8/nPJxxWvjz4ib2tXYMc8qUVwyqtQQMA\n39f85rGW3gAAIOPC2nfi/P6Z1iF+QiGEEOLkHR3klF+AhiaHm5aWMGp47pKfsbUWNy0tPdtFE6Kg\nyUzAGbRkXpDisGLDToeuuKai1GD+TJvzzhm4wXj1Bp9DTcccVoZi3Q6fEeVpFs7JzfW/bb9HPOnj\ne5BJZUgnMyjDIFQUIBgMUFU2+Gj+ui1xDtTnDy627U7mfVwIIYQ4E4qLBh9vLI6YjKkN8M9/W8Or\n6xO0dnjUVJosuiAis9RCnGUSBJxhC84NsODcwAlfd+honhyjZM8BeHGjw8Wz+2YPnnzN4Y0tHn53\nzBAIBbACFq7jkYgmKQm4TB1TNui9OqL5NywDpNL5yyGEEEKcCZfNLWHN5hjJY87HKS81WbIwO9of\nsBVXLRh8j1wPX2s2bk9z+KhDWbHBJefL+TZCnCoJAt4jzuCzo3TFfKIJTWmR4uBRj7e29wUAPQzD\nIBAyCARtWtrjrNkJF50DZp4Bl4tmRXj0OYNofGCHf/SIEwcsQgghxKk6Z0KYWz5UxdOvdNDQnG38\nxtQGuHFpBVUV+VNx5xNNePzXHzrZfdChp0l8eX2KL99uU3Hi+EEIcQwJAo6xp87h1U0Zmtt9ImHF\nrEkWNy0tPuP3GVkNTR0DH/c9H1N5vZmEtu7zBw8YNBimQbAoxB+ejXK0s5JrL3IHBAIVZTaLLizh\nmVc76Z9yubLMZMVl5WfmAwkhhBCDWHxxGYvmlrBxWwLbUsyZFhn0MM7BrHwhxq6DuWmEjjS7/Oqx\nFr56a9mAvXdCiOOTIKCfnQcdHliVoCvR99ieQx5pt5OrLjTYsDVOOqOZN7uIYOD09lRfs8Bi6/4M\njtd3Ha01mbTDrAkWoUE2+eZj2SapdIb9TQavbgFbpxlbazJmeN+v99brqqiptNmwLU4i4VFbE2DZ\nolImjZM0oUIIIc6+gG0wf86pDar5vmb3wfx723YfSLO/3mXi6KHPKgghJAjI8fKGTE4AAKCBF9+K\nsfqVGA1HsxXQo891sGRhCWUVYZrafcqLFZfMCQ44B+B4isMGf/shi/tWpemKZ+/jZlzG1yo+urhv\njeO5kwze2Orlnw3ovp3WGsM08TyfN7b5NNansC2YMsbiK5/IXkspxdJFZSxdNPjeASGEEOJUbduX\n4dVNKZrbPIojBrOn2Fw5N3xGRuh9nc1ql4/nQywh+9uEOFkSBPTT0JJ/A20yDfGuvj+3tLs8vKqd\nojIHO5RdU//mOw63Lg8zbsTQv9LRwy3++VMm67dnaGn3qa0Oc95UG6NfhTlmmMmCmT6vv+PhHVPH\n9VSsTsaltCJCIp7piQtwXNi23+V/Huvg1qWy7l8IIcTZ8/buNA88FSPee6SOz55DLp1RzQ2LT3/B\nvmUqRg+z2BYbOBtQW2MxbYK0c0KcLAkC+gkFFTAwF7/WGv+YHrjW2VSdPUFAY5vPE6+k+NLHB5/q\ndD3N06+n2HXIJZWBkdUGV1wQYN7M4KDvAfjQxTaTRxls2OWxdb+P64EyugOAtEMwaBEI2fieTzKe\nW0Fu2ZMitsiiWA5cEUIIcZa8vCHVLwDI0sBbW9NcvSBEccTM+76TsWRBhMNNDl2xvnY6YMHShWUn\nNRMvhMiSIKCfqWMtjjQPHGXwHA83zzyk7+cGBgcaPBrbPGor81d2v1mVZNOuvk1NTW0+B454fOZ6\nxdja4/8qpo41mTrWJJbwuft3LumUB1pTXB7CtrPvNUwDx82dzYgnNR1RX4IAIYQQZ4XWmsaW/Mtx\nuuKarfsc5s86/SBg5qQg/99N5byyPklLh0txxOCimWFWXFFBc3P0tK8vRKGRIKCfaxeFaOvy2bbf\n7V2DX1YEdS3xvK83rdxKzfUgnc5/qu++epete50Bj3fENC9vyHD7NUP7VYSDCu37ZFIOruuRSjqE\nIjYl5dm1/246NwgYXmUybJCgRAghhDhdSinCAciT8A7TgKrSMzcINWlMgEljZOmPEGeCBAH9WJbi\n0x8uYv8Rlz11LuUlBrMnW/yf+1Js35O7Y9gwDYKRUM5jI6sNRg/P3+HeU+fiDHJm19G2wQ/zOtY7\nezJ0tMbxut/i4eNkXHzPp6g0jNdv2ZJScMl5EZkmFUIIcVZNmxCgoTU14PHxIy0mjZGsPUL8JZIg\nII8JIy0mjOz7au768jj+8zckBvNsAAAgAElEQVR17D6QwvOgssKmI2WT9vo6/OEAXHZBANPI3+Eu\njgzeEQ8Hc59zXM26XZqWLogEYO5UKOs+dv2VjaneAKC/VCLDonNN9tsm7V0+ZcWKOefYfHRJKS0t\nsZP5+EIIIcRJuf6KCB1Rn637MmS6J73HjTD5+NIiyd8vxF8oCQKO4Wt456DJ4VYTT0N1ic+SC03+\n5mM1Oa+ra3T58+YMbV0+JRHF/FkBpo4bfLRj3owAL6/P0NiWu25SAbMm9v0aOuM+D78EDW19r9m8\nD1Zc5DN9nMHR1vzrLj1PU1Ou+PBluRuTh1r5ZhzNpr2ajAMzxkNliewhEEIIMTS2pfibG0o4cMRl\nT12G6nKT2ecEMJTiaKvL27sdQkHF/Fknl05bCHH2SBBwjNVbbHY39H0t9W0mR6M+y2ZnR+V7+Non\nFndo6/BpaIaumEcm43PulPyZfixLcdOSEI++mKK+OduRj4Rg7nSbyy7oe8/qTbkBAEA0CS+9DVPH\naIpCio48+59MA2oqTq3j/s4+nz9t1HR0Txj8eQucP9ln6VwlIzhCCCGGbPxIi/HdM+laa373fJy1\nW9Mk09nnV69Ncf3lYc6bevyseEKIs0+CgH4Otyr2Ng5c09/YBpv2Wyyc6nLgSIY/PBfNObrcsAw6\nopq6RpdPXaeYMTH/pqUpY2z+8RMWm3Y6RBOa2VMsKkv77qe1pq4pf9maOmB3vWb6RJv65oHrgSaO\nsph0CqclxpI+z63XRPtteUhl4M3tmmEVcP5kCQKEEEKcvFc3pnl1Qzon8XZzu88jq5NMHR8YsBRW\nCPHukjUf3VwPNh0KUlxsUlpiEAmDZfVVXS1Rg4yj+dXjXTkBAIDv+nieRzwFr25KH/c+pqG4cHqA\nKy4M5gQAvdfKn1yot4wfvizCpDE2dsDEsi0sy6Sq3OJjSyOnNGq/fhc5AUAPrWFn3XEKI4QQQhzH\n1r2ZPCfvQFuXz2ubBm4iFkK8uyQI6PbG3hBH232aGhO0NiexLEUkbOB178K1DHh1fYK2uElpZTFl\nVSUUlUUwjOxXqL1sVdfSPvRMP8dSSjGyKv9zlSUwdYxiw06ne7mQgVIKZRh0JRQvbRjkPPUTSDuD\nd/QTUkcLIYQ4RamBx+70Sg6STlsI8e6RIAA42qF4fWOc3Ts7aWxIUH84zo6t7XR1OYRDinTaZXSV\nx6b9iqLSCIFQADtoEy4KUVpVjGEYaJ2t0IrCp/eVXnZutsPfX9CGi2dkj01/c4uDM/C4AbbsdWjr\nPPkAZOyw/KckA3RETz2gEUIIUdhqq/K3h5YJU8bJamQh3mvyrxB4aaNLW2vukEUm43OkLsbEyWUE\nTJ+KsENH3Byw5MayLcIlIZLRFAqYPeX08iHXVhrcsdTnje3QHoOwDedNhrHDspVpa2f+7ECJFOyu\n85hflrvEKJH0ee71GIk0TB1nM31i7masc0aDZWhcP/dz+Z5PR5dPc4dPTbnEikIIIU7O4nlhdh9y\naWrPbbdmTbaZNu74B36tfSfOaxvjdHR5VJaZXHphMefPiJzN4gpRcCQIAI62ZpfSBEIWppkd1Xcy\n2Ww/ra0pTFOx+7CPp/Ovubcsk5Jik4Xnhbhybijva05GScRg6YX5nysOKzqiA0fubROGVyre2uaw\n57Df/ZjHph2ttLRlP99zr8Psc4L87Y1lmGb2s/haoX0Px9EY3WccaF/jej5oONqmqSk/7Y8khBCi\nwAyvNPnbG4p54a0UR5o8bFtxzjiLFQvDx33f6je7eHhVO+nuWe8D9bBtT4pbr6vgsrklx32vEGLo\nJAgA8DWRkiCW3TeKbgct0kmHRNzBDtrsaghQWp4dIU+lXFynb2Sjoszgyx8tpziS/7Tgodh12GfD\nHuiIQVEIZo2H8ycPHIGfNcnicNPAhZYTR5n8+R2fzXtyl/A4aQvIBgGeDxt3pPnjqzGWX1LMG29n\niCY0lvJxHT1glqMoDGOHS/YGIYQQp2ZEjcXtHyo+8Qu7+b7mxTWx3gCgRyqjefHNGIsuKO4dsBJC\nnB4JAgAzaGP5uR14pRSBkEU6mSGdSVNRXoxp+ti2jW2bxKIZHCfb4b7kXPu0AoAtB3yeehNS/Sq9\nA43Z9J2XnpsbCCxdECSW1Gze5RBNZM8HGD/S5NzJFo+/NnANvxWwsB0LJ9O3cXjTjgzrdsZJZMCy\nDdAKJ+1iBczejc4AM8cblBbJUiAhhBDvjqOtDnWNeTa+AXVHM7R3eVSVS9dFiDNB/iUBgYAJyYGP\nG4ZBIGTjpF0M5eK6EAgYGKZBKGyB9pg5XlFdDvevStPalT3Ma9ZEk4XnDtw/kI/WmrU7cwMAyKYK\n3bgHFkzX2FbfdQylWDw3SFOnItPgo5WiPWnwxvb811dKYdq5QUBDq4vZL2YxLYPisjCm7xIIGhSF\nYNo4k2Xz5a+HEEKId09R2CQcUiRTA5e9hoMGoaAMTAlxpkgvD7CP8y1oHzSKRMIjGfcwTYVpmkTC\nBrdcbhFL+Pz+T07vaYig2d/g0xXXrLi4b5Ow72vW7tTsO+KjyW70XTBD4fnQ3JH/3u0xqGvWTByh\n8DW8vV9xqEWxvwE6YgEMKxs5JNPZ/5SRLW+eT3HMH3ODE8/1iXYkqK4K8/VbA9i2wpCTgoUQQpyi\nLfs9tu7XpB3N8ArFotkGRaETd+BLi02mTQixcfvAkbmpE4KnnYFPCNFHggDAdx0810RrjWn1jeB7\nrkcimsQO2qTSHo7jk0n7hCMmJWEYN9zg/z7RPwDI0ho27HK5/HyTSMjA15rfv+Sz5UBfZ3z7QZ+9\nR+DmxQZBG5J58ilbJpREstdbtc5kZ31P5WdSXJKdwejq7EvmbygDj9woQGuNm8k9Q0DlWU/pe5pk\nymPjXk17DErCmoumKoK2BANCCCGG7rm1Lq++7eN1N0c7Dml2Hfb55DKLsiEsMb3tukpiiWZ2H8w2\njErBOeOC3HZt5dksthAFp+CDgCPNLjt2tNPRla2tDMPACpgEIwFM08RxPAzLJBQO0dGexOuu1cbW\naLTWA1Kf9eiKw57DPrMnG2w7oHMCgB576mHDLhhfC5v2DrzG2GFQU2aw+4hiZ/3AznggaBEKW6SS\n3dmNbEXKzwYNkK04J4ww8VImqYwiEjLYss/NWfffn+P6PLuu7z6b9mo+slAzukZGXoQQQpxYe9Tn\nre19AUCPhlZ4aaPP9YtO3J5UV1j809/Vsm5rgoYmh9HDbS6YGRnSElshxNAVdBDguJr/90hHbwAA\n4Ps+mZSPYZj4tgY0ShlEwhaGoTANmDjc5/JZPkpBKKCIJvIfthXtXtO4t37wkxFf2OgTCBhYZjZ7\nT08HfnQ1rLgo+/OBJgXkr/xs2+wNAsYMUyycFWD7QQ90dl3/5ReV8dq6DrYecPE9CB5O4AxyuPCx\nFWxrF7ywEe5YOmjxhRBCiF5v7/NJpPM/d7g5/6BZPoahmHdu0RkqlRAin4IOAv68IUFdY/4eseu6\nmHYQw1QYJvjaoLI6zKUzPOZP68nCo5gy2qC5I//Jumu2+sybphlk4L37PgrVc+Kwymb7KQ6kGV1h\nUBqx2VvvsXV3hvZOME1FqChAIND3a+sJGoI2zJtuMHOCxcwJVvdzmv9+tJM/b0j2jcoYNobh4fu5\nlbFhQKR44BkHh5uhpdOnukxmA4QQQhyfeZzBelNSewrxF6Wgg4CW9vyddwDf9bO5iLXOHiCGYtwI\nm/nTckf1r1lo8/Y+j1jimAsY0NwJ63Z4zBxvsGGXxs0zCNKT71ip7L18DV2pAM+vTbJ5j0s8BalM\n9jUOkE67lJSFCYVt0Brb9Jg4QjF/hsnsSblpStdtd3h5XerYW2JaBrg+PXFAOKiIlEYwrYFpTj2f\nQWcOhBBCiP4uOMfgz+/4dB3bJgLjayUIEOIvSUEHAdUV2U5vMBxAGYpMysHvGTJXABqUIhDM5s+v\nLh0YNNiWoqrcJJbqe04p1bu0Jp6CiSMN5s/QvLVd4/S7hGEojH7DJkoptNYYpoFlGzS1Z5cc9T8Y\nRfuQiGcIBE3SSQfDVASKQpjWwAhj+4HBeu+KBbPDjKw2sC2YNzPIQy9qDhwd+MraChheIRW3EEKI\nE4uEDBZfaPDcW7nLgiaPUiy+8NTP0xFCnHkFHQRUVAQpq1a9JwWHi33SyQyJrmS2I28YhIsswiET\n388ur8mnpkxR1zRwuYxpwIQR2Q708nkm08f6bD2g2dcArTGFYagB6/AV2U6/ZZu4jk++W7oZj+bG\nKKlEBu37HK1XdMSqKIn4jK3pe50/+EQHaQeunNu3/OeSmZrWLoj2y8oWCsCCGcjpjEIIIYZs3jSL\niSN81u3wybgwpkYxZ7IhbYkQf2EKNghIO5rn13m9AQBkMwOFIkG0r3EdD9PKdsZNO5vvPzPIwPol\n55rsa/DpiOU+Pm2sYtKovuuPqzUYVwvbDsIjr2nybvZV2Udd5zg9eCBcHMAOWsQ6kmjf5+D+dlbZ\nVXz8UpeK7hPaJ4wy2bQ7f6HbYtl9CD1ByORRBrcu9lm7K5vZqDgM50+CscNlL4AQQoiTU11msHz+\nwPZDa9heb3CgySTtQHmRZvY4j6qS/INs/bPdCSHOrIINAtZu92iLDnxcKYUdzHb6i4qDVFQGjz1q\na4CR1Qa3LrF59W2PxlafgK2YNEpx9dz8X++kEdkzANw8/XyFwnE8XMfvLk82r78yFH73pgLTyh5Y\nZpomqlIRbU/gZnxiKc0Tb1l88koXpeCSOQGee8slnsi9kRUwiaYtXt7sc+FURUn34Su1lQbXLTjB\nhxVCCCFO0Zu7TTbtt9Ddg2ANHXC41WDZeQ7DyvpaW9eDI10msYzC14qw5VNd7FMWOlGLLIQYqoIN\nAvIdztXDMBTFpWGCQYtg0CKdyVY6w8sGr3zGDDO4dcnxR80dV2Mo2HJQ4Xr5hzW0r4l2JLEtmDjK\npClq4ZM9wMzzfFzHw7D67mPbJqGwTTrpYJnZEf43d5pcPM3DMhXDhoVoOJrBdX1QYFsmgZCFUorn\n13m8vgWmj1V8eJElmRuEEEKcNfEU7KjvCwB6RFMGmw6YLJ2TnbnWGg60W8QyfW1dNGOS6DCYUOFS\nHJRAQIgzoWCDgMmjFK9szj8abwcsikuDeJ7GcTwScY/qMsXYKpfB8vUfz646j5c3Ohxp8bFMKC0y\n8Twb0xwYNISDmvPPz6b5XP22iVZ9dzRNA9M0sG1FOtMzU6AIhQN4nk9JiUU6ozjQAheT/WAjqy3a\n88x4aK3RGmJJWLtTE7A9PnRxwf51EEIIcZbtPWqSzORvQ1uife1hV0oRy/M6z1e0xA2Kg8dfLiuE\nGJqCXfA9qsakpMTGDpqY/TL0GIaitDxIIGBhWQb1dVEyGY8jzS4P/AkOHD25EYj6Zo/fPJVg844E\nLS0pWjs9Dh31SMZTA3L1A0wba3DtJUHiGZOGtvzXtG2DSROKKCvLdto9z6esIoJpKiJhRTQFv/6T\nwR/XKmZPsSk6Jv2/1rr35OMeO+t8PF9GV4QQQpwdIXvwNsYy+p5LOIMfkJkZZBZdCHHyCnLod18j\nPL0WHN8kGDTRAY3vabTvEykOYFlGNoe+Uihl4Lk+lm3SmdC88jaMv3po99Fa8z+PdXH0aF+etHQy\nQzASJBQJEu2IU1pR3Ls5d3gFXDorOxXaGjeZON7CMCAW8zja7PZukHJdjR0wGV4TAlI0JR2S8QxN\nzQbDasJoFA3t2f/2HvGorjQpSmlSKZ/2qI/WmmPjj1gSMg6Eg6f//QohhBAAR1p9Gls1E0YoJtXC\nhv0ebbGBqUJHVvY1SrY5tGBBCHF6Ci4I8H340yboiOfm5zcthWVZ2LbZmwrUdbMdZjfTl0XocCtE\nExrTyJ6yGwoMPiqxdkuKuiMDz09PJ9LYAQutNUVGknGjI1SXwbyp2U74jqMBHNtmWHX22tWVNuVl\nLjv3pNAaAoHsBI5hGlSWB4l2OXS1J+lsS1FWFsLrN1OaciDZkX19JGgQcTJE4wPLWlECwcDJfZdC\nCCFEPrGEzx9edtnboHHdbNs2fZzBvOnwxk5FZ7JnIYIGz2P3wQy2r5g3XVEZgZa4T8o9drGCpiKc\n59RNIcQpKbggYMdhaOrI33HPLs8xu3/WpJLZw8O0YfSOwmuteeA5l6NtPqYBY4crls+3GV45cGXV\nxu0DT+vtkYylCEYC+K7LRy/te7wjYVDfaXPsVGh5mcWI4TaNTU7vMiDoy+FvmApPa3zPJx7Pv14y\nkVZEIhbRuJPzuKHgvMkGhuRgE0IIcQY88orLzrq+UftkGjbs8gkHHD62EF7eavDOfognPDLde9wO\nNGg64rDsIsWYcpcjnRbx7qVBtuFTVeRTEZGZACHOlIILAhKD98t7O/q+n11n72Q8lKFw0hn8iJ3N\n0OP6HGzsG4nYflDTHnX4/A0BAnZuJ/pI82An9oLnefiuj3PMSEdTfGDmhB5lpSZamRQV2X1lRpOM\nOyjDoKjEJhZzsGyrt1I9VsoxmD/dYH+jTzwJ5cXZAOCScwvur4IQQoizoKXTZ++R/J31nXU+1yzQ\nNDQ5tHfkPqeBTXs1i2b5FIUNJle7xDMK14eSYHYGXghx5hRcz2/6WPjzNk08NbCjrVR2BgAgk3bR\nvo9lG8SSDqaVpqw8QDQxcJS9sU3z5laPy87Lfp1aa576c4KOuMIKWKCzswx+92ZcrTWqu6M/subY\nX8HgoxzhsIV3bGq1qJM9R8DXaE/T0Z6kqqZ40GsoBZfOsbjukuwUrW0x4NRiIYQQ4lS1delBD9dM\npMHxoKkj//PxFPzyiQwfWWQzcZQp6UCFOIsKLq4uCsGcCWCogRWLaWaX/biOh+t42EGLjuYYvq/x\nXJ+wctBe/gqpLdo38v70a0mefi2F6yuUyh70ZVomRr9hDKWyJyouvqhvJ25ju2LPYUVdg8/hoz5t\nHX7v/gQA55hK1XV96g9HsYMmyjCIRzPEo2kS3ct9smlAc8s7sgoqisFQioCtJAAQQghxRo2uUZQU\n5X+uqkQRsCBo539ea019s8tDz6c40OCRcSQIEOJsKbiZAIAr50BlCew8rElmwPcVnTGfWMLD9XxS\niQzptEs6kT1RTHWvu/e1ZrC0ZaVF2Q6+72s27hi4GRjAMAw8x8M04IJZET58WYTa6uyvoLFT8cKW\nIIl+h6NkMuB4muFVoNCknb7nXNenqTGBFbDx/ey8gutmDxOL2B4zJyr2Nii6En3lLYtoFs3Qcvy6\nEEKIsyYSMpg9weC1LbnLUi0TLpxqoJTCMjyUMrEsI5uAw8129n3Px0k6tOsA//m4Q02VzdTRsGK+\nIQdaCnGGFWQQADBnYva/LM2WvQ73PtyV97U9h3pNGGEQT+sB+wqqy2DBDIP12zPsOuTSkv8yKEMx\neVyAGxYXc8743FycWw5ZOQFAj2RSM6o4wzm1Lg+8bJLxLTxPE406ZByfQCB7mnA07eA6HkqBh8Gy\nCyCZ1uxoCHCkOU3A1GQyHpt3Q0MLXDTNxDKlQhVCCHHmLZlrsn13nLrGDK4LxcUml1wYYf6MIL6G\n9rhJUXEAw1DdZ9doEvEMmZSLYZq4jotpGXTG4a2d4GufDy8cmFpUCHHqCjYIONb0CTaVFTZt7cdk\nzjEN7KBNRQksX2AzcZTPK5s9jrTo3uxASy60+M3TSbbv79kvYGLZBr7n5xwIFrA0E4d5VJcP7Hx3\nJvKvzPK1Ip0B0wDfyXDkqIPvZ0fzTcvAsrJLeiw7O5oSigR7R0vCQVh6UYBX1iV44jWPzn6pQTfu\n1tx2tUlZUcGtCBNCCHGGeR5sPZhd8z95pObBP3aw+0Cm9/mODp/X1kWZM8Wkri2IHeobCFNKYVmK\nUNgiEU13z2x7vXv0AHbWZQe2wkEZvBLiTJEgAHBczdE2n6vmh3j2DUUi4aE1GJaBZZlUlBh8+NIA\nrgfVZYrPXm/T1qmxLEVNucFTr6X6BQBZSikM08gJAjpa4zz4SCtPvXCUZVdUc8tHRvY+FxhkfSRA\nOKBpj/ocadK4/dZHeq6Hb2uCIQvDMLCDFqGwTXGob9mS72teWJ8bAAAcbtY895bHX10pQYAQQohT\nt78Rnlmnae3KtjsvbARlVDJldvawzXhXiqb6Dlo7ff70ZgK7LJT3OrZtEgiZxDIOSqmcPW3RJLRF\nNaMkCBDijCnoIEBrzfNrXTbv8WjtglAAxo0MEgloosns9OSUsSbXXl7O/Y938IcXHVIZqCqF886x\nuPqi7Ne3rz5/Xn6lFIZh4DoOyViCzqY2ADq7XB5d1ci40WEWzq0AYGyVR0O7wbF7DiqKfKbUejzx\nuiaVOfYO4Do+ppVNZWpZ2fJksxBlO/fb9jk0tOb//AePZjcOy+ZgIYQQp8L14PHXfY62ZJekAli2\nSThi42QUkUiAUCiAHbQ4vLeFzTszTJ7ukS8vSU+b6bs+hqF6z8EBKAlDLOayanuGsmKD+eeGsSxp\nu4Q4HQUdBLz6tseLG73e8wFSmWzHeOoYxRdv6hupuP/JKFv2943ot3bB6nUu2/elqSrRtHd6aJ0/\n005tucv6t+rRfm6GA8eFN9Z19AYBs8e6RJOKvUct0q4CNFXFmkvOSWMY0NA2eIYEz9XZDES2wvPA\n9frKmnaP8z4/m5BUqlEhhBCnYuMen/qGVG8AAH0Z9krKgoSCAVJpTUlpmHBRgGTK5Uh9nPJhZQPa\nTM/ziXUmegOAcHEQ3d2c+W6Gnz3QQbp7xe7zr8e55ZpSpk7I3V8nhBi6gg4C3tnXFwD0t++I5s0t\nDnvrPRrbfI626ezyIKNv5EIDh45qduzJ7hJWpsIO2DmVmmlCSTA9IADokUj1VZpKwaJpDnPGuuxv\nMSgKwIThHj0DIdZxVu2YZncGI626X9tXhtmTA1SVZgOXY42uUXJKsBBCiFO2cWcmJwDo4ToeqYRD\nUVG2jUmloagkhOsmicU9ipIpApFwznsSsTS+p1EGREpCmKZBIOBTZDq8vSX3YIH6Jpf/faaLb/9d\n9dn7cEJ8wBX0gvBoIn/n3PHgydcybNzl0dCi8f3sacL91/dDX+pQAO1pPDe3IvQ8iHklg95/9IiB\n6yJLIprZYz0m1fYFAAATavN31pUBwZCNaSpCYZNwxCKe8vG6Aw/bUlxyrjEgJ3NlKVxxnmRaEEII\nceoGO50espt7K4o1kUi2q1E9rAjLzo49jq92mTzCp7JYY+CTiKdJJjKEi4OUV5cQKsqO8F822yDW\nEct7/cONLmu3JM/wJxKicBT0TEBFkaIjmi8Q0KTzrL/Xmpw19P4xI/za09Cvs60MRXOXYsq0Gg7W\nRUGDk0yjtWbMyBDXLxs25LJecb7BvkaPuqZ+11cQClnZA8mUoqUxSihikwla3P9Mhk9fk61EF8yw\nGFbus2G3TyKlqSxRLJxlUFla0DGgEEKI01Rdpth7OP9ztqmoKFV0JsC0oKwkwCGVbUvPGWsyd3o2\ngHj8Dc3mfQbFZZGc95uG5q0tGTrSAQJhyCSdAfeIxgcPQoQQx1fQQcD5U03qml2OGcBHMfg6+p4g\nQGuN7x5T+ajc2YEeSR0mXJx9baQkzOgaxec/UUtleWDIZbVMxV9dbvJ/n1GkMxoUBAJm7xkG2c3A\nikQsQ1FRgJ0HMrRFbWpqsu+fONJg4kjp9AshhDhz5kyxWLvNJd+q12lTQr1LWYsiFpYNtp19YFil\nSTLtc6ARpo2GhjZFU/8VP1oTj7l0ZHxQFuFiE8MwScX7DuqJhBSzp+bPNCSEOLGCDgLmTbdwXVi3\n06OlQxMJZTvL2/Z5OAMHHHIoUxEMB0inMvTEDIaZv5Pten21o8agNabw9ckvxakoUUwZbbC3Mc+h\nYolsgQ3DoPVoFDMY4MnXPKZOHPBSIYQQ4oyYPt7msvN9/rw50zugphRMnhBi+jkRsk2kwrYNPM8j\nnXZBKVa+6oMy6Upkz8EZXa05fzIk04rDTR5NrQ5ev8QWSikCIZt0sm+f3dxZIYZXFXQ3RojTUvD/\nehaea7Fglkk8mU0RaluK36R9Nu8ZuNEpWwlZvT/7vk9VbRntTV2MHmbSGjPxjhkN0b4mk8xdW5RI\nal5dH+XmaypPurxLL/B56i040JTt8HuuTyqZobMt0fsa19cYWrP3iE9Diyu/ZCGEEGfN9ZcFGTU6\nyKZdLmgYPzZEeZmF1tCVMNA6O3CVSrp43TPoh4+kKK0oArKZ6g42ZZcJ3bEM7vnf3ACgh2Ea1FQH\nKY/4zJ4S5OqFRe/ehxTiA0j6h4ChFCXdSxG1hgtn/f/s3XmMZMd94PlvRLwr77qrurv6Yl88xEsU\nKZEUrVuibMuybHg8NmyPbQwMrBfCLLAL24vFYoHB/GN4F2vYwIwxM4Zn1iNjNLZnLMuSrNOSKIki\nJd5Uk+y7u/qouyrvfEdE7B8vK6uqq4rdzUMS1fEBCDYzqzJfZTUjXkT8jiI9aVlaNSwvdgcdev1A\nbar+k9c0Ftxz9wi/9RHBP3434ZtPJ2wslJClKTrLtrxnku4ccvRqSiH8s0cM/+ufNAlCj7ibofXW\nmEiTabQR/Ncvd/i1D+5cASjTliePpyyuGsaGJA/c7uMpVzHIcRzHuX73HYSJkZDFliLRkm4MjbZk\nuZUvAKy1XJ5ZL1OXJlvnxZkFOHXJEnh5mezt/MIHy9x37FW6azqOc93cImCDJIOvvxwxW1eEZcHu\nMkxPF5m50KLV1ltqGq91BU6Mh7WGkSGfQlkg4nwVEAQKiLBG025srmBweP/rq23sk9Fpbb35tybP\nVUiNJU0yzs8pGm1FdZsNk9klzae+2OXi/Ppg+90XUn79oxHjw65ykOM4jnN9hIB9Q5rpqubbJ0PO\nLSnWChBaa1mab7O0sH5ibU3e9V5563ONBRabcHhaMjO/dX6bHBHcc9jdtjjOG8Vlim7w9PmA2brH\nxvZZBsWuPeUdu+oqJUk1xKnl6RMWIRWFQkChEKBUnrhbGSpv+p67jka8667Xd4z5f/5OFXtVyVJr\nLVma5ddqwWhLt2d44dFyE6cAACAASURBVNz2r/H3j8WbFgAAM/OGz3wzfl3X5jiO49xc0szy2HMp\nn388QSUt3nWox/6RlKX5FqdeXuTUK1e1rheQxJvDbn0PDkzAhx/wufMWxcY0u7Ga4GMP5+WwHcd5\nY1zXkrrX6/GzP/uz/O7v/i4PPvggv/d7v4fWmvHxcf7oj/6IILj+Kjc/zubq2+9+SyUplT3arc3H\nl2shQpWCpd62LG7TkAugUPQ5vD9ESTh6IOJj76ttaof+WpQixXve7vGV7/YGCck60+sNzQQYrWk3\nu9TbEtj8szXbhjOXt+Y9AJy5rGl1DeWCWyM6jnNjbpb5wll3aUHz6a+mzG3obL9nTPMrHw747vfa\nLC9uzosLQo9qrUDvqkXAkT2wazSfd37joyGnL2pOXdKUCoIHbvMIfLcAcJw30nXd5f27f/fvqNVq\nAPzJn/wJv/qrv8pf/dVfsX//fv7mb/7mTb3AH6ZtQuv7BMXC5ptoISEs5DX6J4cspQj8HSJoykXJ\n7/3LSf7335niFz889IYNZKWCHOz65/kJ679OIQRxLyVLMuYXtpY6ijN2rICUpDs/5ziO82pulvnC\nWff5xzcvAAAuLVo+/52U4dEyU9MjVIeKVGoRIxP5fxcrBUZqknIEo1V44Bh84uHNc+OhacVH3hnw\n7rt8twBwnDfBNRcBp0+f5tSpU7z3ve8F4IknnuADH/gAAO973/t4/PHH39QL/GEaKW2/CihHMD5k\nKZZ8glARRh6lcojvewgst+2FobJk/9T2r3tgkn6i0xtrdDigPFxiaKLK8ESV2liFsJjvsq31D7DW\nslTf+nONVAV7xrf/9QsB7Z5rwOI4zo25meYLJ7fSMJy7sn0S7/lZgybvZl+pRQyNlpiYqlAseSgl\neOjOkP/lFwX/888JPvqAdEUpHOeH7JqLgD/8wz/kD/7gDwb/3e12B8e5o6OjLCwsvHlX90N2x3RC\nKdx8PCmF5a4D8La9lsCXhJFPEHqDHIHpMcst/Zv/n3mnYN/Exu+FW3bBR9/5+ga2Rsswu5ihN3Rj\nsdby1ClJVAhQKj8R8AOPUrWIH3mI/mAqhKSwTQ6yFIJj+7ePBjMIvv3C9qFCjuM4O7mZ5gsnF6d2\nS8PNNWkGcwsZzWZKkhi6nYzF+Q6ddkIQSpablnrrxirlGWP5/Le7/N9/2eBf/4c6f/a3TZ55qXPt\nb3QcZ4tXzQn4u7/7O+655x727t277fPWXv//vOPjlRu7sh+B8XGYGLM8fRpW2xD6cGyP4NZpAVQQ\nfsL3XtIs1C2eglrB8p67PSYmosH3/95By3MnU+ZWDHsnFLcd8HZMKr6W+eWUv/gfSxw/1aPTs+zb\n5fPBByt89JEaz59KuLK8deCTUhAVQlqNDp6flzS9744S4+N5cvLJixkX5g27RgTVqkCqFGss1uYn\nAELmYUUrLfFj8Tv7cbiG18pd+4/GW/na38putvliJzfbtY+OWvbvWuH8la0lP6USmP7cAgzmwlYz\nxfMVz522PH/asn+X5KcfjLjtwLVLf/77v17in7633jV4cdVwYXaJ3/3lUe69rXjD1//j4mb7e/Pj\n4q187W+EV10EfP3rX2dmZoavf/3rzM7OEgQBxWKRXq9HFEXMzc0xMTHxai8xsLDQfEMu+Ifh3umr\nH6mwsNDkzmkYjTSf/lrK5VnNZWM5dQaO7VP82qMRfj/kZ89w/g+kLC6+tmsw1vL//mWd0xfXB9YL\nV1L+y2eXwaR0Mh9rwRiTT679vAAh87KlVlvwLe9+e4n7jxkuXGzyme/CuTnQRiCEpRxaRsbLeFKQ\nxBmNZjwozayk+ZH/zsbHKz/ya3it3LX/aLzVr/2t7GadLzZ6q//9e63X/s7bBPPL0N1QWE4p0Ei8\nbTbBpMwbh1lrMFZw5pLm//tCm996VDBc3jlAYWFV88Tz7S2PtzqWf/jGKtNjb80T7Jv1782P2lv9\n2t8Ir7oI+OM//uPBn//0T/+UPXv28Mwzz/DFL36Rj3/843zpS1/ikUceeUMu5MedsXBh2eOfnoGV\nVobph+akGbx4RvPZb8X8wnujN+z9nnsl4czF7ZqMwZMv9Hj/gz5GazZWCc0XAxajTb+tumD3ZIgU\nhi89A6eviA1fK2j2BFKCHypKlYBSNWTuchNrLW876PoEOI5z/dx8cfO671aPobLgWy9knLlkSY1A\neXKQm7adLNOkmR3kATTa8ORL8JH74fQlzYU5y2hVcMctEtWvpvfSmZTODhWs55ddHpvj3Kgb7rrx\nyU9+kt///d/n05/+NLt37+bnf/7n34zr+rEyuwpfOV6k3lWUhuG2aonVlZjTJ1dZO+E+MaOx1m4J\n/Vltar7xVI9mxzBclbz3vohS4do32HNL2Q79EqHeMuwdt9jtxjzLoC275ymWm5bvnIkwoeToIWi2\nNLPzyeC6jQFrNPW6xvMEI2NFSLrcfdiVB3Uc5/W5GeeLm9WhacX3T4Lw4VpFYPNTbEsQSMbGiySJ\nodmIqbct/+nzMScv2kG1Pt8T7Jn0efsRyeiQ7IetynzDa0OeXDFyScWOc6OuexHwyU9+cvDnv/iL\nv3hTLubHkbXw2HGod9dv3JWSjI4ViOOMmfMtAHqxxZj8CHTNy+cSPvWFNssbqvM8dTzhN3+uzL6p\nV4993D3hIQRsF0Y7XFU8c9KiTb7jb6xFkCcBS5WHBAkp8HyPF85klBckE2MBpZJHoaDwfcGFi+vb\nKbVagNfW9GKDkIJO5vPF78Mn3v3aPjPHcW5uN+t8cTNLtWXmOvK+rc1z0Ky2jI0X8X2F7ysCX3Ly\nUpteotAmQ0jA5k3Izl9OWGpFHNmtGBopkJn13jhxJ0Frw+23XDufwHGczdx27zVcXlXM17d/rlZb\nL7szOSI3dTK01vK5xzqbFgAAc8uGz36ze833vfNwwOG9Wwe1MIB33RXSaBt0ZvKdEMsgP0BnBgSE\nxQAvUCSpYHk55dSZNssrecOWasUjCvNr9TxBoeARhpK4m6H72y9nZyHTN1a1wXEcx7lJ2e03rdae\nNMb0u9xbglAwMVUkitb3If1AUaoWKFdCqsMFfF+BEIQFD2Msaao5cVGgbV7wQoh8o6tYDvmp+0o8\n+uAbF47rODcLtwi4hm6680ck+zf9hRAeunPzDfvckubsDh15j59J+d7xZNvn1ggh+O2fL/P22wIq\nJYHvwf4pj1/6YIl7joUsN7aPf7TWgsnDkpQnSdOMpJeSZbCwmNJoaoyxVKv54Fup5NWLfF+CsGht\n8HyVNwzbmpLgOI7jOFv4nmD36PbP9e/92T0Kdx8LmdpVIYq2bnJ5Xj7fKiUplEIgP+UOQo9eJ8n/\n66qoH6EU+/cWkdKFAznOjbrhnICbze6hjJdmobvNPbvJNHccVDz4Np/bDm7+KM2r7orAf/tqj04M\n77l3c/TkCy81+daTy3R7hgN7C/z6T09ggDixVMsS2c85WKzv/OLa5KcBaZwhhKC50qYyVMTzBL4v\n6CUQ9zSVimKoFqxfr7EkGZRKgjSG77xoeN/b19/TcRzHcXbyU3cJFuqW1db6Y3ZD7P7MAiw1E0Yn\nw21v2j1PDMJgpRSEkU+aaoJA0euYfE7dZuprtFxSsOO8Fm4RcA3FwHJkNzx/Lq+2sybyDO+5zzBZ\nLWz7fbvGFJ5i5yYqqeW7L2Y8fJc/qI7wt5+b5a8/e4U4yUe5x55Y4clnVvk//tUhhir5rom1lpfO\nZSyuaozO6/pv7UMg8gpBNt/ZV75Hp51QrkYkab5jkySWJEuxI3lIU6ed4YceNtZIKcg0PPYipMby\n6P1uEeA4juO8ur0Tkt/4sOHJlyw/OGdZbdktm2GdnqXY7lGqbJ07PU9SrQbU6/mumxAiD1ewebhQ\nXhHPIOXmE/qJEQm4hYDj3CgXDnQd3n0r3D0dM17OGCpopodTHrylx2RVs1CHLz0Nf/e44KvPQr1f\nwlgIwUht549XSMHCquXCbD5wLa8mfPbL84MFwJpXTnf49N/PAnmC1H/8+y5//vc92p18dyXPC9g8\n+AmRv7/nKaQQKCUxmSYIBCAwBipVj147JY01q/WEdkejpMBai9xQ1u34OUucutwAx3EcZ2fWWk7O\nZJyeyXjkTsFodefT8ImyJvA2PylEvvsfBJIoyuegLNMEkUeWZiglWFls0270aDW6ZFker7p7FB65\nx+UDOM5r4U4CroMQcGwq5dhUuunxVy7BF58SdOK1nXLBiUuWjz1gmR6HB++K+MzXt+vqK5FS4iko\n9zdDvvH4CvXG9kH4J87kK4vPfyfm+NmtRwtGW4RYL09qLYMKQaafHyCVZGwsHyiNhTgxGG0xVlOv\n5+8rFHi+wuj1RUWjA/Mrlr0T7jTAcRzH2WpmLuNvv9bj/BWNsTBUFlQrCmv9bU6q4fAey3JsuLic\nl9Nb27hao5QgSTK6rYTUE8Rxitkw9Vlj6XUS7n+b4tF3eoNGnY7j3Bh3EvAaWQuPv7RxAZCrdwTf\nfjl/7EPvjPjAAwUCXwyqGUgl8cM8GffALsnESH8QfJX3WhsbT13cuRui7cdc2n6zMN+XYAVGW6y1\nlCt+Xm2h/zXNZkaS5XkGAGEoMamhWPTotNcXO4UAht/ajUwdx3GcN1i7B197Dv76MfjLr1guLuYb\nTACrLcvMbIYntm5sTY/BfUclhcAipUBuE9IadzMay/kGWpZZsFtnSGtguKgZrbrbGMd5rdz/Pa/R\nfB1mV7Z/7soS9NJ8Z+MX3l/kD36zwvSUTxD6hFGAlJLpccHHH1lPCn7PQ8MMVbc/mDl2SwmA5FWq\n9Vjym3+d6rzmcuizMYOqWI5YXOyRZYZuN2N4yGdsrEBmBIWiwlpLWAiwFvSG0qCHdkO54P6aOI7j\nOLnlJvyXr+UbYScuCawKqI2WKZTW5zRroRIZjk4LShHUSnDnLYJf+aBCScHhyQxfbY0XiuOMuSut\nzaFEOxSn2KkCn+M418eFA71Gov/PdiGP4qoqZpOjPr//G3njriuLhpGK5O23eoNW6ADDtYCf+8gk\nf/3ZK3R76+E4tx0p8csf3wXAnjG5Y2t0IQRSSoLIw/e9/CKMQQhBuRZhrKTZzOj1NOWKolzyWFqM\nSdO8z4Dn5X8VklhjLURBvgD42IPumNVxHMdZ99gPYLGxeW6QUlAsh3kpz7WJ0Vh+4yMeaWaRkk1z\n3p4Rw/23JBy/5LHaUYCl006Zu9La1An41eQJwY7jvFZuEfAajdfyhKRLS1uf2z0K4VUlkIUQ3HXI\n565DO7/mJz46ye1HS3zjO8t0Y8Mt+4p85H1jBH4+0L3vvoDzs5rlxvoAKYBCyadSDel2EtrtLM83\n8CRCSWojxU3vkaYWKQSLCzFKClotQ6eTUSoFeQxmN6MQwG8+ClPDCsdxHMfZaHabeQ9AeYqwENDr\n5NV9OokkzeyOMfu37s44MpUxV5esNjSf+kKHON3mC7fJMJYSPv6ISwh2nNfDLQJeIyHg3bdbvvAU\nNDrrA9xo2fJTd1gyDT84nzfcun0fFK9zrDp2qMyxQ+VNj2ltefZURqcHv/yhkKdfzlhYMWgr6dqQ\n0fESQgisLdJqxFw4t0K5WiAIt//1riynJImmWvVYXs1o1XssXKqTpilSSuxohW5XwvBr/ngcx3Gc\nn1Di1TbgN9ywdzKPf3zS8LGHdt5QUhJ2Dxt2DwvefbfHN5/NNjWqnJ5QXJwHsyEeVsj8hPvF85K3\nH3k9P4nj3NzcIuAG9VJBkkE5shycgl9/v+WpU5Z2D2pFuO8wnJ2D//EdWG7li4PvvGR5+2F49x03\n/n4vnc/43HdS5pbzgbVUgPtv9fjEeyP+89cjSmp9cBVCUKlFTEyVWZxvEUbVbV8zzQy+L+j0bH/x\nYPthTYIs09RXm/zlV2v8wrstbzvowoEcx3GcddNjsFDf+niWanrdFCkFXuChlOD5M4Z33g4TQ9c+\nWf7ogyFH93k8dzIj03DLbsmlFY+WFhhjiHspQgqiKM89uLCAWwQ4zuvgFgHX6QcXJN8/49FLQCrJ\n1CgcmdRIYZkaExwcSwn9vCPiPz4lSLVAKTDG0uoJvnPcMl6DY9PX/55xYvnMN1OWNoT/tLvwzWcz\neloh1PaDarEcoi+3yJIMY/KkYT/wUEpirSVLNWkKQSCw1tDrpIPKRdZaslhjgM89mZdyiwK3EHAc\nx3Fy73kbzK9aLi2tzw3GGNIkIyoGg2o/Whs6PcmffSbj0QcsD9x27VuOQ3sUh/asz21zT+T/llJS\nKIabvnabpsOO49wAtwi4Dv/0TMZXnvNQKt9tz2LDqbZlblkwMhQSp3D8ss/0UMITL1kM+QIAQEpL\nlhkyI3hpxm67CEgzy+yyoVoU1Mrr56xP/GDzAmCNsXD2UkZtYvvrzQdGSxxn6CxvtR53U/zQw/O9\nwWltEmfE3WTwfWtlTK2xtBtdvOEyT520PPwaTjAcx3Gcn0ylAvza++HZ05aZRcvpi4aVVgZCDMJQ\nszQvMmGMoRsLvva05q5D6oY3lY5Ow3Nn1suPrhHA4T1v0A/kODcptwi4hl4CT54wjI/6BKFAAElq\naLYMq6sp2JTJCZ9mW3JqIcSQsLFmkOh37I17Kb1kayDll7+X8NQrmqW6JQzgyB7JJ94TUC1JOvHO\n16WEJcs0nrf1NKDbSfsNv/KGYVbnrduTXoa1FqXW+wUkccpaLSOxoaxRa7VLbbhML9ny8o7jOM5N\nzlNQ8A2vnNO0ewIp+3ORAOUJlK9I+xtRQkK9Dd9/OePhO70NjS37gag7lAAFOLwb7jsCT5+CtT6W\nSlqmRy0LyzATwfj4m/qjOs5PLLcIuIaXLwqKJS9vvtUXBgqvJkliTbOZsWtSEfqSOBUUC4p6urmg\nv5R5QdHlhmFja4ZvP5/yle9lgx2OOIEXzxriNOF3Ph5xYJdCimzLDgjA9ITg0moPWSn2Xz/X7aYs\nzrUZ3NhfVchUZ3ZwSiGEQEjIEo3y8l4B1lisNQgpsVimx17Pp+c4juP8JHrqpR6f/nJMN85LUfuB\nR6EUIsibVHq+xAsUOlsva/21pw2PH0+ZGMrnnqWGxJLnGLz/XsFYbetGmRDwkXfArfvg5RlodQwX\n5g2nrwjOXBF84znDU6da/Ow7LZ5y8UGOcyPcIuAamqm/aQGwRilBpayYm89oti1RCKQ772hIBZdm\nY1rdiHIh/5rnTultb/DPXDacuaQ5tk9ybL/kpXObewOMVOCRuzySVPMX/1inVA2RUhL3MhbnmnS7\nCaVyAcjzATa5qtRaVIxoxm2M1ijPQypJGqcUqwHVYn4U6ziO4zhrnjsR86kvdEj65TwtlribYIyh\nXC1iTd7FXimJ8gSmP4V1k7yR5kozn4ekMkgpWW3lOQa//VFDMdw632YGVuIAWfIoRXCoalluQJJA\nu53x4rmEgm/4yP2urLXj3AjXaeMaSoWddxbWInGkEoN76yzb2sxLSlBKEceGVy4LLi5L2rGg2dm+\nIYo2cHkp31359Y+E/NQ9HtPjgolhwT1HFL/+aMjUqGLflMf/9sseftxg5swSly+sksSaYqkw2NnP\n4zI3vI8QGGMHjylPIZUkLAYEYd5oTABRIRycYDiO4zjOmm8/Fw8WABulcUaWrXfxtcYONsbWQmM3\n2jg1za/Cd49vfU1t4J9eKXJh2aPeBm0FYSiZGJGM1Cy1Wt4n5+QVN1c5zo1yJwHXUA6379ALsHeo\nyeqywpOSNDOEytKLN3+9EBCGalDF4JWFIqdXJFjDxG7FUqOxpQ+K78H+Sdn/s+BnH/J5+bzgwqyh\nWhJMbeiSWCpI/tWvVOjGhn/732PmVvJBV2uNTjRZpgkLfv4eIl+wQD74WmuRUhBGPr1OQqlSQCmJ\n9BRRKaTRETxxAt517Pq6NzqO4zg/+RZWdp4XsyTD9/t5Z6zPM1Lm85YQ67kAV09+S82tc83XTxS4\nuGiJ+zlyUlpKBcFoDapFQatjKBQk3bYP6C3f7zjOztwi4BpuGU+ZWVKs9ja3APaVZvdoxrDf4lS7\nytFdPdptxWLdI03NoO15EEiklKSpJoxUP5HX4nmSQjli7z7LhfPNTa99ZFqydzIfROPU8p8/3+Pk\nhfXQoW89l/FLHwg5sGv96FNJQaeV0Gub/iC7/npGGyanh2is9gbHshtZ29+xkfngHBYCfF8hJTx9\nWnHHvoxK4fV/lo7jOM5bX7kgWFjZ/rmo4A+aifkePHDvMHFsWFhMmLm0udKEvKrGZ5oJ5lYFEzWL\nEGAMXJjP8+XWGAPNtkVJwUhVUAwtxkrCUOIWAY5zY1w40DV4Eu6cWqFWSFDSoIShEqUcGO3ghwHV\nqkfgGQqBZayqEQKCQBFFHlHkDXY/0lhz4MBa8648RtL3YGQkYO+EIvRhqAzvuFXxqx9ar4X82W/F\nvHJ+c+7A7LLhM9/sbQrzWa5rFlbzO/yrTxbSRKOUpDa89U7eWEuvEwMWa8APPKyxmH48ZzcRHJ9x\nf00cx3Gc3J1Hgm0fj4o+I+MlwijfXywVPYJAUql4HDxQ4MD+cFCYQik4erRMpZw/ICUsdEL++xMB\nn3nSZ7EhuLAkNy0ANur0LAhIsrwfgedJ6h0XEuQ4N8KdBFwHzxccnuygDVgr8NSGajvSp1bM6yNP\nDRnGK4aF5tXJSZZKRVEbWr+5T1JLIRIgJYcPRuzeZakW4YFjEG4YX09f3H5nY2bOcnJGc3Rf/ius\nliWV4vZ5BsqTSCVQngRhMdoOFidxJ84TuDyJlIKJXWUunUuJIoXvy/zkwEUDOY7jOH0femfEworh\nyePpoPpPoegzvmcIqSRB6GF0wq5Jj9n5jGbHsH+3x+6pCOlJrlzuMTFRoFIJKESKl0+0KBR8fF9h\ngcuriq+9KDi6e+edfW3AE4ZM56cDvhJ87QeK99+hqRXdpOU418MtAq5DEIWQGvKcps2DS2p8xqsZ\nSliKAYwPW1a6oHW+I68UhIHA9yOszeP1rbV4niVNQWeG52fWE3BPX7H8zAOWfeP9Ov7bJF+tXUVj\nww1/MZIcO+Dx/eNbv0Gq9XhMKQRpprESjNF0W3mgpRCCciXA9z1GxysUiwHGWAJPc9veneM/Hcdx\nnJuLEIK776gwG0s6rQQ/kBRK65tcYajYNVGiUlGUQs3zJyyvnEnZP+1RKvrcflvAWsxQGCr27yvR\nbG+eWxebgluMRQqLsVt3+H0FK3WD7yvCfg+f5Qa8eFHx8NFsy9c7jrOVi/O4DuNDRbbdDrcW6SmE\nyP/84jmYrwtKRUmlLKhWBOWS3FRi1Nq8cZfJ8pdcXU1YWWqxNN+ivtphpWn5zvH1agq7xrb/FQ2V\n4Y6Dm9dwB/eWCCN/UNBHCAgLPuVqkYUrTdJUE8f54GiModvuISR4vspv+PtHuFEpP4qQUnBwKj+h\ncBzHcZw1Y1VLGEqqw4VNCwDIQ3s8X7HWi3LPVF6cYuZyRrdrEGJth79fpW7b+v4CrGD/+NbTAIFl\n73Cbdk8grKUYSTwPKhWPiwtu08pxrpc7CbgO5aLHcEFS71o2Di/aSqQQXF60fO8HkswKpMwoRJax\nMX9Lz4AkNVhj8X1JmkEh0Jw/U19/vmtY6rVQskA3VhRCeM+9PpfmNY3O+usoCQ/c7lMI89e/vCw4\ndUXwwgXL0FiZNNVkicYL8kRkay1pnNFY7WxayyhPYXV/EPYl3W4K1qIGXYgtd+93A6rjOI6z2dSQ\nZc+o4cLC1tr8hUgigDSD5TpEAdQqkkZLo7Uh8iSd1KI1qMCSZls32aSwTNYMd+/P+CebMFcPSLSk\nFGoOT3bYPxZTKYKKO7RllaaR9PDp7FB623Gcrdwi4DrVCpJSYGjF+X10KQBPSf7mW3B2bn233hho\ndzRqRTAysl5RyNg88TZOLe1ORqmYnwqUygHt1obMJwuL812EKANwZK/Hv/iZAt9+PmWpbihGgrsO\nezxwe76z8rUXFMcvSDKTLwiigiFLNWmq8xwA+p2BlaDTSjctTMLQZ3zUp9WzCKHwPMnlmVVGJ6tI\nmfc+OD9r2TP6Jn6wjuM4zlvSuw5lLDUkni+JCgKtBUlqqfaTfdPM0orBIBiqSnqJpRtbpMjDdzo9\nS6Ascc9wdU+a6VHD9Kill6TceyDG2jwPQMn8lBtguJSwmATsLy1xolOlHitK0dbXchxne24RcAM8\nJRnaEBqzUIeZxe0Hm05XM2y9QQ6AAMJQEgSCbtdQb2iGd8HIeIlOO9lc0tNAo5MRBfmv58Autakc\nqLWWV85rvnvccG5BUygFg7rMUkrK1ZB2M6ax0qFcLRBEHjo1eL7E9xVxL+s3DIOVhua++4Y4N5PS\n7WRYC43VDkHgkaWGS0tv+MfoOI7jvIUZC199TvHKJUmqAQxRTzA57lEtK9J+SL7AkmYCrfOeNkNV\nw0rd0ktFXgJUw1LdkhmBFHn8fymCPSOGh49lm3oKCLHeoHONJy0XmkNMlHoIKYm7mkcf6DfFcRzn\nmlxOwOuw3IJMbz/YGGPR2pJlliSBJGWQGByGEqUEcQqep9izv8bddw+xf39psMPxZ/8geP7U1tpo\nxlr+61dS/tPnE46fzei0EpbmW7QavcHXSCkpVUKshW47Jkmyfgk1RaEUUhmKBouGJDbMXIzZMxWg\nPIUfSOJuRrcdY4xFuLHUcRzH2eCJE5IXLyjSDfNfL7bMLWQIYQc367K/a6+UQEpBsaAYHZEYBErm\nz6/tfxkr0EZw74GM996R4fe3KAPfQ+8QlWpQaCO40B7GmpSj+yUjFXdb4zjXy/3f8jrsHYNStH38\noZSCJBUkaX6EqTWsdVNXShAEguV6vl2SppZ6C4ZGIu65ZwTflyhP8amvZJyf25wU9d0XM549ublv\nABZazZg03ZpAlWWGTrOXhyOx9v6KQmm9DqnOQHoSa8AYQZqkdDt5laEDk6/ts3Ecx3F+Mm0Mgd2o\nF1va/YaVkM91oQ+hD80OgGSkosAKekk+J+oN05ZFcPaqHANPKXo63Nr/Rks6WUQpMqx2QxptH99z\nu1aOcyPcIuB150EThQAAIABJREFUKIZw67Rlu8pBhcLWZKmNg50U0OnknYXjWCOkotGC1abh8JEK\n1oLve/z7z6T8x88b2t38m0/O7LAlYqHbyU8OrLX0uhtKhYq8RKg1ZnC06vkKP1QolS8+jAGtNcOj\nEdZYslRTKxruvsUlWTmO4zjr4h1KVwObknylFJSKgsDP56U0M3jKYkW++YXNT8uv9dpdU2A1LtFN\nfeLMo5WGLMZVNB4SS7udsJyWWGm8UT+h49wc3CLgdfrA3ZZH7jBMDVuqRctQGYpFhbXQbmd0Ohlx\nrDd199XaEoagfJ84ziiEkolRhZQCIRVJBmEgEVKQZoaFhuDf/oPk1MV0EGu5rf5btFsx7ebm8CCj\nDZ1mQqveHTwuEIyMFigUPHS/SlAp0GAs1hiWG5bPfddVB3Icx3HWDZe23xwyxuB7FmPzMCAhLIWw\nHxZkYaSsCXyDIM8r6PY0jUayaX4c2qbRV+hBR0csJ1UW4xr1pIyxCmOg3jIsn7xIqXUJJdymlePc\nCLcIeJ2EgIdug3/xAcP/9NOGOw8CCIzJm4UZk+94xLFByjxXIArzAdEYQa+XMTYWUCkrhmsSIQRJ\nAsMjAaYfCFmqhLQ6GX/+uYyLS+yY82SMYXG2wdzFVYw2GJN/f5bqQVfHbiclSzXWGCanIqb3VQDL\n4lJeOejSTBNjDUHkY7GcvASd2A2sjuM4Tm5XLUVnW8NP282EizNtIL/xVxIKgUFYjfIs1aKmqOLB\nAqFeT9Da0u3mu1vlyHDn/q2vO140ZHrrPKSkYWrEknkFHn7ujyjb5hv8kzrOTza3CHiDXVnZ/g49\nyyyBbygVBIGfZ0t1uxlT4wGT43l8fqmQNzwxJj9GxVoqlRBjYNdkgajgE6d5B+DNCwFLmmQszTWp\nr6w3FDDarH+dGHwpcTdlaChgfKKEtbBaT2m1MoxOQYUMjVWoDBWxxtJoa5bqbhHgOI7j5BaXExbn\nWnTaMWmSEfdSVpfbLC+0mJ3t0utleNJQjAxSwFAxphgKCoGFNGbEq2+awnRmODSp+fBdKePVrfNN\nJbJk6Vrobf6PIH/t0aqlVpaMts6TfOOzfOkZwzbrE8dxtuFKhL4BVruSC8s+nVQiA0G5aGht07DE\nmrxCQpIYLs/GaG1ptPK8gDwUCAJfYqwh7mn8QFGqRkgJhVJebjRNU3zfp1wS7BnJd1pePtsjibcf\n9ayxiKu6Mfa6KWdPr7JYCQgKAVIKqlWfThvGdkX0OglCQJJofGkZG/K3fW3HcRzn5uOpfB7ZlHvW\npw1cudjm4fsL/QhVSzHohwkZ6IoCe+R5TvSODL7HWJDCMF7becMp9POSoFfzPdgfzgIwlV3kK3MB\n57+S8i8/LFx1O8e5BrcIeA20gefOwMVF8iZdQUCxlN8oFwp5PwB/VbNS3xxPb4xhZdUwv5AQ90Ns\nerFheTVjbMRHZ3ljr2LBZ3UlASRC9OMp+0GV3WaMP5I3Cvu1jwQIAf/Xn3WuvsQt1voVAGhtEEaw\nutJjzJfURksAlMqSdjsliDykFAShx3AhpRC4kdRxHMfJvesOn8eeSWh1tz7nKclyXdNsaSplhSdN\n3i9AQ72rCD2DSA3d7vrGlRBw/KKHJ+F9d27e0FpswDOn4HJdUyoIDk2LTVWAjLEcefpT+demVaaj\nZa5kIxy/mHDHXhfs4Divxi0CblCm4b89Bmdn1wchIWImxw17doVAHspTq0hWG2ZQ1ixNDWfPb5/V\nm2UWbewg9l4IQZxohMybrZw/ucitd04gpUD5CmstFkU7huGyYN+Ux4unty/XIOT6dXq+JEvNpq7B\nzUbCcH8RIKXo5y3kJxZBoPjET732z8pxHMf5yVOrKD74QMhnvhlvKt1ZKPmUqxHdTka9ZaiU8yIZ\npv9F9a7iQKXFxWSSMPKpImi1Mnw/v1k/tyB5+ULKlWUoFyAKBF95VtCJ1+NZryxY7r9dUC7l32Oe\neZrlv/wiwaNHODH1fubmygilefxlyR17f5ifiuO89bhFwA16/KXNCwDIE4DnF1NGhrxBaVDflxQL\neXfgLLP0ejuX9YlCSaNpB5V/tLEEYT54+oFHUAg4e6pOZbiQ36RrgxdGPHHS49F7Mx59KGJ2UbN4\n1cmDUmpww++HinIlZMRvcn5O5icY5CE/zWZMpRJibd7gzOaV2xiqCPZPub8ijuM4zmYP3R1wplFG\nSUGWWRotDWK9NPalOcueyTx6P1QZ5Ugzv+pTiWd5oX0rAGGYV8XTJs+Fa3QFf/1NBtXqPGWxUuY5\ncn2NNrx03vKOQzHm+edJ/82/xrZSTlwepzF6FN1KGK+AChTgkgMc59W4O7wbdHFx+8eNgeWVjD2D\n/gCWu/Zn7K5qPv89S8tuH1ITRYIkWz+yTBJDr2dQSvYrKECxElFfbCOVxGhLqRJSKgfMrlrmVgVn\nVorccXeZZicj68ZcuNjBeiFRwcf0a/57gQdCsHdS8Iv3L/HtE0W+/XIRKQXt/k6MlJJ+QSGMsdxz\ni2C7HgiO4zjOzaubCB4/W2RyYv2mf1xbFpczGk1DEHq0e/mpszGGUGkkljS1PL56jE6/grVY62Fj\nLQaL0XawAIB+g01jEL7YdIK9fH6F7v/ze/DM04PH4o5AKYHvS7SBoYLELQIc59W5gLkb9Gq3xEKu\nNw6rFTT3HcjYPWq5Y19eDu1qYSiYnIgAS5pqOt2MRjPFmHw3XkpBluUhRVExQKcGFShK5bx7Yqbh\nyy8EnJr1WekoMkIoVJjcO0alVsAPPMLIp1gOUf3k4E4qGa8ZHr27xa17enh+Pognccbtu1YR/esv\nR5YHjroFgOM4jrPZiYWAbra5IaZSgpEh1c9jE/05yrJYlyhhWFiVjJZT4mTjd62FwK6/xpZkXpsX\nuNik08VuWAAApOXh/qJCkFrFcGVrw07HcTZzi4AbtGd0+8elhIlRReBZioHm1ol4MJi96zZ4751Q\nK4NSEAaCiTGP248WGB2SDNckSknabY0xFjMY8PI+AsZAoRzSbPaoDRUol73+2CnoJFcPdAI/UJsG\nUiEESuW/6qEojzmKAnjHoWSwU2MtvONQh4dvbQF5EzTfnRM5juM4V1ntbH+D7fuSSllijMX3BGma\n0Yx90kzgy4yRSsbt+zLGh9Z26AVef56REjxPUiyHWxYCV29HVS+8uKnEqClVaL3/43lX4tQiBewf\ndacAjnMt7jbvBj10G1xcsJyb35AYDExNeJRL+cA4VU0ZLq7H5wsBuyd9Vm2A1gKl2HC0mXdUHK4J\n6g1Lr7fhKDQzJEnebbhZ75JlhuHhAkEgiWODNptCMAfWknrjeGP1BUEx0LzjQH3wWCnKr9EaS+Dn\n13Noqkc7LXKrq6rgOI7j3KBqWbC4ZDmwV9HtJ/TG+BwY75KKkG6s2D+l0ZllueXhKUEqLErmIT+e\nlycYW2PpdrJBWOyaIh32v/xVrBAIa8n23kLnF38L/4F3kl1O8TyBsIJbxpIdrtBxnDVuEXCDfA/+\n+XvgK88Lzi1IpITRYY+RofW78VasgM3VeuabCiHkYNdjI2Pz04FaNaDb7Q1OA9aqLujM0OuklKsR\n1kq0tnieQEmL2fpyAJvasPcf4f6DLYaL6wuDejcYfO2BCY2vLH7Bsn9ME6eSKLjBD8dxHMf5iTdU\n1LS2nEKDwFApSXZPSNJUoqrgdVOkNUShYKXuoa1ACpiesCy3oBBCnOS9BzINpZJPo5GiraE2HHBo\nLEV5gl5iGalAR46Q/ps/pf78kxB3Sd/+MPgBgbUUIoExgtGqRrl9LMe5JrcIeA2Uglv3S8JquO3z\nQvcgbkFYZqEheGXW5/KKxAhDMQLf2zg65cm3UliklHheXlFooyTOBmXWjDHEscZamB41XFje+v5p\nqun18pv9fBdFUClJwvFxZmKPveEcqx3F989XUErgKcs7dl0CIjqx4IvPeHSfgINTgo89YCkX3pjP\nzXEcx3nrOzqRcHlVUW8ZFhdTokgyOuJRjRI8TzE6HNCutygon7GxNrXFE1yo3kMzDiiFeUiq7+en\n0UJKShF0YqiU823/Tie/mS8WPBIki6sSJcAPDSIA6UnSe9615bqkkihlieOM589JRsqWPaPWNQ1z\nnB24RcBrNFXNuNjwSfXW7YYhu4S3NMMVuZ8vn9lNvKH6TxxDrWIIg7XHLAJIdX7KcOygx/mZlJVm\nXjEh6aXE/a6MOjOkqcFow1hZU/QtvY6hl4BSkihSFAJYWo3pdrN+gpakXJYcPhCCkMxloySZ4YVz\nBdomQIiEyWrGkRf+hnhiHy/XPkg3ya/t1CXDpx+T/PaH3CDqOI7j5FZbmu8900QphedJWu2My1fy\nU+z3vF2SZiVmVzwyBe890CBsNZlrFJDCEvkWYTVYxS27Us4vFPA8i+4YsAJPCSoVj+XllDQzLHU8\nMm3JgAuLikJomRi3m8qGAsSJJdN5/sDFRctSWxGFil3Dhve/LaVa/JF8VI7zY80dmL1GoQf7ainq\nqqo/w2KFQ94FhNUUerMk2eaBylhob2jwm+/UWwqyRxAIlFJ8+AHD3UcM77tf8qGHAw4fyLsRe57E\naM3PTP+ATrPNd16SdHp5edI0NXTaKednOnQ6Wb+iQr5waDY1s7NxP0RIciWZxKsNMTTsUygqxqZH\nOf22f4a/MIM5/tzg2pQSzC5bTl5xKwDHcRwn9+eftxRLAYWih/IlQehRqoQEvuKbz2iW6yY/Mhce\niVbI2hAHRtrUihm+Z+mmCiU1BV8TBNCNAQRpZjHWIoQgCASFwtZ9ym4MrdbmvjtpZmm08hN0o/N/\np6nFIri8ovj6cf/N/kgc5y3JnQS8DnuHM2pRyvxcA2MFVdlkj5pD9hcGQ16HsbDJQlzd9H1pZsEY\npLKUgpTxUpta0OOpi/lAFQVw7IBE27zKT/U+SZIKGj2PyWKL20ZXSVLB3509tql2sjZ5V+BEb66K\nYLTl5NmEVivjvnsqFAJDLxGMjfi02xrlKbrV3cwfeh+3nniWb/vvp9lM8QNFlmouLXkc3e3KhTqO\n4zjgBz5RKJmc8AZlPZeXNSvG0KtnxEke5iOF5dTKKCbqIAtQjWJ6WUCqJaUgJQMkeUPNKMzLe1pD\nP0E4r2rX7W6t8jMcZTTaAisEWkOrYzEGtDaDUNiNrqxIFhuCsaqbxxxnI7cIeJ2qoWEkPIcwWzsC\nGyAzWw9bPGW4d888ntwcZrOr2iHrJZxdGmJ3rU0nk0S+IQoltx4JefJ5zSP7rgCwf6iFFAbL5uSs\nq49I1wghOD/T4+D+iMlRSa2o6aU+1bIikBm+snSH9jAuHmdkNMTzJcuLHSyWyHf1lh3HcZxctSqZ\nGA+oNzTdnkFJGKopSkXJuTTve+N5Ag/NcjeAeJjpGvhSE5OH7GRastLxKYSaNBMUi/1FQL+oRam4\nucLdRuM1y7FyzOMnPOrdfI7NMku3kw4Kaqz1xgHQRlDvuEWA41zNhQO9XlJhg9K2T62kFVbSrc+N\nlBJ8tTXOvua3uad6FoFltlEAaymqLmApRoJySVKaHAMgVJqH9sxufVMLgQ+jQ/m/12ht0BouX+mh\nJISeRQhLqSxRyiAEWOlxoVXFxE2CQJCkGY3lDvNL6db3cRzHcW5KI8Me7eef5baVb/Cg9yQP6G8R\nvPQtOu2EKJIEvmS4JqiVNFJBQ5cQ/fw3AIEg0RIEDJcyAs8gBXhe3mAs8gwPHY1R27TnHC4Z7tqn\nOThh+OcPJYxEMfXVhGYjIcvyr1dKEEXrm1el0LBnZKdaeo5z83InAW8AXZ2GLEFm3cFjRgWY2i4q\nC4bE5GE9SQqVMOO23U2W2z7L7ZBCoNldy2/0pwp1VAqBTGllEWGWUvQSJJZl6fPxdyfM9SapiCYj\nYo537Znn6YVdg0RegF3jlnfdBsWCoNW1zMzCd1+ATiu/kfdVnodgAayl3bIUI58k6xHV53k8vZeX\nXu6ye9owMlriwkqX4xfh4z/cj9RxHMf5MSVOPsexQxGz6h00qBEQM1mbY+TKE1zZ9RBKpIS2w1TF\nEIaSWmQQCLSRCGEQWBo9n+FCShTCaM3SigXaauJYcGxfytEpgzApz5xTLNTzcty7hw3vOpoNGllK\nCb/0sOWlywE/OBvT7AiaPYkXqE29eA5P6S0lr63FFbxwbnpuEfBG8CP0+K3Y9gLoHkgfUxon6UQM\n1ySd/k26wLKrajkxV2OhGWCRgOXsYpnRQouSF3B0WNM1ERZBnCkkGYGCo2NNAmkIPcvp1kEqpZhS\nb5n33N7hhYUxstSSZZrdUwHFQhuAckFw20GICooTl0a4MlPnwP4Ia6GXCpSAY3sT6j3J6krG8VcC\nXrHTeL6m1cgo1zyCSDFUcuFAjuM4Tm7PlM9JdesgHDUhZIZ9TOzyya5cYGxvhUuXYor7JSdnQ3bd\nIsBmdHURTxqkUHgSwsCiDdSbGuFBt2NRSqBNfnd+ZLfh8C7Dalvgq63lqtux4OSsB57PbQcNx6ZS\nLixYXrpkqXcMhcBycNJw74H1sKLnz8JzZ2ClDcUQjuyGR+7IFxSOc7Nxi4A3ipSYyuTgPzMDL14O\n6WxoqGIRXG4EdHr5wLfWDL3RC1htD7G8HGPvkPhSkxlFagRxJgg8jS8MAQkl1SH0QkSvSz0ag0wx\nUUlY6UaAotmDbiIpBOtHn5PDhnMLisNHhykVLUlmafdgeiwhkJbVrsGsLPH9pWkAglCRpIYsSXnk\nnQWEjoHoh/AhOo7jOD/uGtHUlnw0gEXG2RtcoNkNmM+G+PQXljh8sEVZZGgUXRkSKIs1gmohz6Nb\nXDG8/HKbI7cGWJs3r6xE6/OXEDBc3hoWdHlV8eSZYMMcG3J+0ePhIz0+tmv7ENbnzsCXnoZU54uM\nVhfmV6EbWx59x+v8UBznLcitfd8kMyv+pgXAOoGSkCQGnRmstRhtEVJQrfg8fbaMMBoFWCuYa1VA\na3QrryvqkRKpjMrKeWJZxvMse4dbg1c3Gs7MF0g35FMVAkMxSEmNpBsLupmkWsxbtFsE1ULK/loT\nJfKBd2xEEoSSobLh+8926RpXXs1xHMfJ9XbYFDJ4pFGNi91REi25MCuYGoopez1qfpvxYBUh8hv/\n2UXN0qrluRMQFAIW57t5U0xjefmi4sz8zrcn1sILF4Mtc2yjp3h+ZudW98+fXV8AbPTyRWh3t/kG\nx/kJ5xYBb5I02znY0FrodA3NtqHTNRhjsMYSRpJ2T9BKJEpqLAJjoat9xttnuNSs4SdNRnqXUDqh\nWj9HQcQU/fXKRBZoJz4XltbPTZUwvH3fKtbkJwBnL8FqR+Y9DATEqaDoawqBwffgjsOKQwcilNU0\nW3D2fPJmflSO4zjOW4int79jVqRcknvR1mOoqpic8KlMTbCc5mWyI9mj27OcvSx5+ZzgsWcs7S74\ngSJN802oLIP5huRLzwacX9h+Hl3pCJZa29++LLUU2TZFhYyBldbWxwE6seD8wjV+aMf5CeTCgd4k\n4+WMkwsBxm4dxJJ0/WgzTfOKCWGQ11oOAoGne0wUM5a6ZQIf2knAebOLlo4gy9i//CzCGgqNWbLx\nu/DE1qoHrZ6HsXnWQdE2qZYs+8faZLJCuST4/9m78yDLrrvA899zzt3e/nKvrKqsVSWVpNJily0b\n4R1ovAHNAIPDTcMMA8xAxDDRQQdBhAk6Jmamo2GADv5o2vQMHdF09BgDxqZNA8abbNnGi3aVttqz\nqnJf3363c878cbOyKpVZJSHLqirpfCIkpe577777Xry4Z/ud329uSRN6kkaQk+ZlrBmw5+AwocoZ\nGzIY6fPQ85JaM9wxzanjOI7zxjQQZZTN0GLrKnFVt7mUVkkSy9QuxbGDdYRQtPIaNdVDCs3z04JM\nS6wtMuRZW1S2DwKJ2WjKrC3CZ//mMZ9DIwlvvR2aVXj0pObsjCHVgraWNJvRllo5UEyE7ZQIVAgo\nhdCNtz/mKctI7dX5bhznVuIGAd8jQxXDZD1jprV1aTLNDJ1OjqfYmK0Q5Lkl8C1ZVtwMx8o9fC/g\nWPg8s2YPs70yoT5EPWnhyZSlaB/j3bMok6CtR41V3pR8gyeCt3HEO8uCnSA2VSIxILQpQ3aZLPcZ\nq1aZHQjKYZGlYX45x2Y+99bO88LyPpRS5Eja/YQ0l6SZpTlcYveYAnbO1+w4juO8sciwjNSayLSJ\nKeHZDCUNvWCESlUzNqqZGjM0y4LUgEHR0RVI+swsX+l2XM7pb42lWo8QolgJuEwbwbdPwjPTUA0y\nzs5cPeHVo9vJ2TtV3TIQGK5qdiptIwTcNglLre2P7RuDiaHv8ktxnFuQGwR8D90/lVANLUtdRZwJ\nltYtk80+99yboZSl3VecmfOZWQkwxhYVEqVlf7PNsh6hK2tYIxAyYLrtMdWw5CXBYvMoNS9GdlaZ\nLK9ipcft9Xn2tz/BSAkW7BjfUO8m8Cy58VnJmozINUbrKVp2WOjVKQWSlb5gj7/I+eUyF/WejasW\nrHU9tLF4qigytnfMFVhxHMdxCgaLJqAvikmuVPhFus2NhBe7Gjm1UvF3IFNSU2z6PbtY3lwdv1wU\nDIoMdkpJggB6PcNVDyFEMXvf6srLb7BpbTWh3ghoNEIAqqHm2J5rh6+++55iE/ALMzBIBZ6y7BuD\nD771u/9OHOdW5AYB30NSwO0TKbdPQJJZnpvTjNavzKiPNTT1ssYY6MWSZj0kHeT4UlPOuvT9Cutr\nIRaQSoEXEdkErUqsVA5SFQGRyskszIy9lSPrn8LGZcZLS9xVPg/UURI6XpN+LAlMQkl5QB0pLdbC\noysHtl13P5OceGqFWqNIJ3rn3gEQvjZfmuM4jnNT0xshojvl2ZdSMFROECLEIinJAYHMODcreHS6\nvvk85UkiJZBCUKl4KAmVEvT7xXm11mgNWWZQSiKVLJJZ2K2TUjKLuW0yxLMJd0zmVMJrT1pJCR98\nAN7Rg+kly2gdJodfne/EcW5FLtj7NWKsYaS2PaQm9OGefX3ee3gOIST1MAEBkR2A9cBTJBkIDBk+\nMs+o6lWmsz3k1REGlAFLX1SId9+BXVlEAkNee/M9pLC05RhPn/W4sOgDFq3BkzvfLGcv9ZFKMTZe\n5vZ9llrkKqo4juM4BbvDXrfLpBRkNkBrgUCQaJ+SSnlhvgJsdOJtsRfO8xRRSSGlQCnwPYnZ2BiQ\nZ5osM5j8yux/ECl40VuP1TQfeovgzQey6w4ArlavwD0H3ADAcdwg4DWSanvN6oTNimZXtc9wdpHJ\n+gCBQClDXa8yEa3hK4MvYyazc2grqOoWfRMy2x9CJQNyEZLj0ylNENuAPDP0ZJPqYB6sJTcSg8dC\nOsT8uocUBkXGocntG4o7nZTZ2T5KScLQo1oPKQWuWJjjOI5TUOr6ne3UBKS6KIbZz3yUhLv25Rht\nsabI1FMUuDQYU5yr2CBcvP7yhuFeJ0V6xUEpBZVqwMhohVK5CEOSAu484Nonx3ml3CDgNeLJ68yc\nCANKcXdwikOjXayFiu2hpCYiYXejjxeEjMklAtMnI2RYrHE+30O9NY22korXY1WM8IU9v4jOc/q5\nT22wQrV1ifWehwGUlEgVUQo0hyZzmlHGykrMYJAzGOSsrSXMzRdrsf1+jjGWOPdJcxc15jiO4xTu\nm0qu+ZiUkOSCOFWkuShSUVNkwPO8rSFEJrebM/9SFh1/zxMEAfR7GVoXoUAASsmNFQhBuRIQhIq3\nHJUc3ffddWPSDE5ckDw/I9Au/4XzBuN6d6+RciDppoJMv3gGxRISI7KE0WyOmewI6/2A26MOuR8R\nZx5lP6FZ8jBeFWnWWFST1JM5Vvxh+sEw5e4iojrGAI9WXicfm2Ly5OfR+/dQCducPddn/wFBrEMC\nX1KPimqKZ2cta+sZa+tXqitKqfADTZZopLB4SmwUV3Gbgx3HcRw4Oql5fjann3qbLUPRubesrSZg\nfUabgl4SUgszslywGlcolQxpVhTIzDZSZV/OBhQGAm2KaJ9uJyeODVIJSmUPrS0CQakkMFqQZZZ9\nu0N+/F16W4rQf4zHzkienFZ042Ig8chpw9uO5BzZ7do7543BrQS8RoQQDJU9wrwHdmPmg5wyXSIS\ngrVZtAy40BlmKWkwECWkEOwqdRhPzlHxM1IT8LS4l6rfZzy7SLOckHgVAt3DCkVJxORGEPtNQtuj\nk4UgBHeNtjkx7VMpSbTJ8JQlkIbnzu18o/N8D2s1oQ/1yFAvbQ8bchzHcd64fuRNMWmaAQYhLNYa\nFub7LC6mzMzEtHsQZx6ZFsx1KvSTooiXEALlSTy/6Lz3uylpnOL7MIgt3a6m2ylGBsqTxb6B0NvY\nLAy1ahH+k2vxXQ0AppcE3zrtbQ4AANZ6kq8+59Ppv/LvxXFuJW4Q8BoKlGSsWWXX3HcY6p5jJLlE\nrTdHafYFopnn6YzfTq2UE+uAjhrFJ8NXhqpeZ6+apd9LmTeT1OhgylU8BaHMwCsWdHKjMNoyLyag\n2sAawylzO6iAldWc3Ei6rYRmZGgGhiSHoabH3j0heyZDwqC4oQohyJKcCzMxdT/mOpFMjuM4zhtQ\nri1LSzFZalFK4PuKXZNldu8p0+9rFhZTcm3opgFznSr9eGvqTykFxhjy3LKymtIfaLrdHGNASIHn\nSXxfIsTG//uSMFQMEksUCpSEU/OKb54O+OoJQ3fwj2uoTs4qcr39Nf1EcOKi22fgvDG4cKDXmheQ\njhygdu7bhN1FpDWk5RFW9z1AUptA9U2xEdjL8MlQWUygB+xRl8i8JhWvR47Pev0AJoVK3mLgN8it\nYikpE6mEga3gpT3WvXFWGOZCNyBJYiSG4ZJhKLQ8dR4mxgJ2TYR4GxuvRkd8ZuYSZi710LkhSSzn\nZiz377+xX5njOI5zczEaxsbKlCuKsWpM6GnW+wFKFckput2MVttQGpMYrVlcyjfbGigGAVoXq8zG\nwKWZmCyz+J5CSPAChefJzdl+peTGvgCDFwg8JfjGqY3U1XPwlB/xloMphydeXmB/kr2yxxzn9cQN\nAm6E5i5ceG+RAAAgAElEQVTm73w/3qCNMDlZeWhzt1ScewyX+vgSchNQG1xCAtJotCwxUopJVQWE\noMEqXp7SKo+SEWCkz6HGGpmtcKlyJ6fFnURY4kyChTxLWe36/McvBWgDeW544fSAiTGf0ZEAz5NM\nTgScObmCFxQ/jbm1G/g9OY7jODelVizZO26IAoMVPknuIYRmotohSSL6/ZQ4MfT6UApBa4tSdrNT\nr/WVzEAAaWIQUpDnuqghEEmCwOPyU4QoVqmlBKsNIgq2XE+cSZ6Y9tk3unPF4BdrVq4d9z9Sd3sC\nnDcGFw50A5QCH09J8lKdrDK8OQAYZArQ7G+sA2CFQgsfKyUWQa+xhyptlO4h0ewenGYl3E1KGRBo\nLVhPykwv+jxZfR8EHp3YY7WlGR6WrKxpZBCCkJspQEslj7mFlEFczJ4EgWJyssTwWA0Az/1CHMdx\nnBdZ6PmUSxLlKTwliEJJo+aTmhIT9Zh7DhQZhJbXYbCRTOhy59xaS5pqpNoejmOBKPIIQ7/o9G88\nRcmixoC1IOzO+9S6ieLs4ssL5bn/gGaosv08k03DXXvdPjjnjcF18W4AIQSNUkTkeygpwBqMzqmr\nNofqywjyy08k90pY6dHz6gQKov4K4VNfR0rBbO1OWuEkUKRWa/V9lnsel1Z8Aq+Iv5xdsrTXE24/\nENEYKm3bSCWExPMkq6tX1j9Hx6qMjVfwPMHU6Gv2tTiO4zi3gExDrOUO7QlUygIjQs6uNADNUF3Q\n7Rcz66VSURQsCASBL1FSEkaKSsWjUvU3z5Om+WZlYCGK+P8ihail18uYX8pot9Mdr02bl7c3oFqC\nDxzPuGO3plkxDFcNd0/lfOgtGcr1jJw3CBcOdIMoJamXI5J+i8z0EVdNXkirSa0EIRC9FvF6G//A\nLg7qM6jBGnqoQmotwcnHyA49AEFIe+Cx1lMsrIcYayj5hlbs024nvOOBCr1EEgaWXn/7DEet6qE3\ncjVrbdHWQ0oYGVK85143I+I4juNc0Ukkxu7cU7ZAGEjGhnKefW7ArvGQVgeUguEhj1IMcQJhoOh0\nMzxPEkWKNDX0+8UEWJ5bOp2UatXH8yXGCIyFNMlJk2LVetDPaDTCLe8d+YaDY/nL/hwjVfgn97/8\n5zvO640b795AxmjybPDiKugIAQEZOk7JPvHHJP/t0+Tf+RqeTlFJBz0ySaedM7R6kspf/AFrHcFM\nq0KuJcYKet2MCwsSY+DwwSrdxMMi6XYTOu3B5gzLZZ4SlKJiqbU/MGzs1WKiKSmFOI7jOM6myLdY\na4lTuLhgOH0hZ25Zk2VFGk8pYbhWbO71VVEdeKihCPytG4MrFY8wVAghCAKJ54nN2H9rodfLEGJj\nFcAUtQLCyCeM1Ea60CutpxKWo5MZpWCnK37l1jqGFy4Y2jtMoDnOrc6tBNxA/ThnMRkiMT4SS0UN\naPqd4iaIRQY+5X/6U+hnniQ98QTmrvuQ/Q7aK9FdTsj23EH4tT9GPfY1grs/ROhfLrcuWVnXTI5o\n1uMyQkCWGfoDi5QwP9Ni7/46Wkt8H8qR4fAew+lZTbd/5aZajdxNz3Ecx9mq5FkWVi3CxEwNZ3gC\nVnseM4uKaiUgzzS1qqRaLUJTm3UYGfLItSW5KopHIBAbQf+X56akvDJQ0NqSZxrPV3h+Eb4qpcXz\nPOJBTpoadg0bhiqKyVrC1MirV/I3zSyfflhzaqYY7JQjOLrP8KMPFnsgHOf14CUHAYPBgN/4jd9g\nZWWFJEn4lV/5FarVKr//+7+P53mUy2V+53d+h0aj8Vpc7+vGIIXpdpnMXIkDGpiI1HhMRGuAxcNg\nmuOU3nwc0+tiTz+LkJbADpgYLHKmcjvh3R+isnyKNDPUyoZGTZImUK1CvWR4/PkuB/ZHGCPQRlCu\nFKE+y0sD9u2NqFUVU6M55Uiwf0LzzLniJxH5hrv2uGVSx3FeHtdWvHEYA8OlHocnYgJV9N73DkNr\n4PPktCHWEa1uysSIAAyjQ4rAN7S7ckutgKuXwdNUY+0Oefv7OY2mQglQPsSGzRoC/RgOj1s++FbJ\n0tKrNwAA+MzXNE9fVVCzH8NjJy2+0vzIg27+1Hl9eMlf8pe//GWOHTvGL/7iLzIzM8PP//zPU6lU\n+N3f/V0OHTrExz/+cT75yU/yS7/0S6/F9b5uLPbUlgFAQdDRFZq6QyRihDX4uo/wfYJDt5GXa2Se\nj590CbIWC/MWdeiHOHj6s8SxQVUER3etUS3VqJcht7BrVPD8Cz3uvrNKFAmsgWYzZHGhx8JyzuSE\nYqXjUyllVCONrxSjNcM9UxkTTZcmzXGcl8e1FW8c3zltOTx+ZQAAoCQMlTOO7JI8dSFkdiHn6P6i\nk9HNPDxl6cdXzmEtmxuLjbEbm4oVvf6VzrwxFikEUSAYbcDFBUu5BIMYopJCCMvc6qvfTvViy+mZ\nnc978pIl19atBjivCy85CPjgBz+4+ffc3BwTExP4vs/6epHGstVqcejQoe/dFb5OxdnONxCLop9H\nRP4AlSV4duOGGEQkXYt3dIrg3FM0hcf+z/9bnvup3+HcxPeB0Fyal4weChkq51QqitwISoHgwF6P\nlfWc0SbMzBuqVcXQcESuBa2WYajp0R0YyqHhp98+oBy9hl+E4zivC66teOPwVEbgbe8kCwGV0JBn\nKdZaZpcs1VqxqVcbwUhVI4CRalEj4MySR24kSgq8yCMKFcrLabczrLUYbZDKZ6hmKJckeW6ZGres\n9z3WWzlaW3z50oOAXix4bs6jl0pKvuXIRMbQdeoErHct/eQa5xpAnBbZhRznVvey17Q+8pGPMD8/\nz8c//nF83+dnfuZnqNfrNBoNfu3Xfu17eY2vS+I6Ny5JDllOczC7eSxd63L+j77AsT/835j7ziWG\npypUJocZ+tJf8MyxH2Xu6Q7DY1WSXDJcicFKWrqKFxiOjGsePSnZNw4myxF+iO95CCmQ0uJ7lrWu\nolrSbgDgOM53xbUVr3/hdTbfWixSWEolj9Onltk1WcEAkS944LaEREOqBa2+YKRuWGoJLscFCSEo\nlxSt9Xhjtl1iNLTahnIoEBLWWhY/Ap1bTG5p9UCba7eni23B109FdJMrK+/Tyx4PHE7Yf409BKMN\nQaMCrd72x4ZquIQZzuuGsC9OFXMdzz33HL/+67/O8PAwv/qrv8rx48f57d/+bSYnJ/nZn/3Z7+V1\nvu6cmc85u7D9BhSQMOnNoP0KAkuQ9Wh0LxLMnOHE//1Zpv7Xn2T95ALizNN4734ns0+2mXnfT/PF\npwL2TA1x/1HJRLTOSPscj8u3Mjz/NONHR3hyboSJSo9qmDDdGaPdVyAsQ1VLGCoWVwXfd6TH/bfV\ntuV+dhzH+cdwbcXr24mzXUze2TGf/sXViMdPeaz3NKdOLHP49iYHDg9TCiH0LHuGk8v1MTEW5lcV\nM8v+lnMsLw/I8yubhKMQxkZgtS3wPUsY+iwupUgByvd45zHJD79l5znNv/wHy/nF7ccnmvDRd3HN\n9u4vvtjn89+JtxwTAn7snSU+8KBbBnBeH15yJeDEiROMjIwwOTnJnXfeidaab33rWxw/fhyABx98\nkM9+9rMv+UZLS53v/mpvkLGx2qt+/TUJzUjRiotqwAC+NNRUlzwoNs5ZIA6baCR7/LPsevA20laf\nfHGV3omLjL3PQw01OLanxxcf98nyjN3xGZpPfAt16QLD39dk+Et/Rvnun2O03mBXPM1SeJhd+hJt\nswc/VIzUDOsDhbWWSLS4OJNTCv3rXPlr53vxvb9W3LXfGLf6td/KXq22Am7d9uJW//293GsfrcCz\nMwET9a0FuzoDj7lWBCJndXEAQJwIPGkxVjLIBL1EbmaekwJG65r5VYU2RYffWotSCnPV7H6ew/SM\nIQxhfMSj3bH4viSJNcqHkzOGu3d3MFZQCq68Ls1hbrXMTtnQF9YtL5wfMFLdOQveO++xZJnkufOG\nTgyNCtx7SHL8toylpVc3acYb5Xdzs7nVr/3V8JKDgEceeYSZmRk+9rGPsby8TL/f58iRI5w+fZrb\nbruNp59+mv37978qF/NGIgQcGNZ0EkM3KSoihrJPmm1/bhbWaQ0dJhyeJt8/Rfbnn0PYFG/uHOZU\nn5Hqgxw7kLKQwKA2xOH1Z2mvD7jvkX/LcmWI0CRMMEPZjxlincx2GaorhIR0o+pjrWToDDx8T980\ngwDHcW4drq1441ASTi+UaA98hioZShjasc+ltRKVMszNp8SxQQgo13wCX+B5kGbQjeWW9NOBD82q\nYaVddNSzzJJlWwMU0txitMVaQRhI0tSgpMUPihCflTZ8+tESBslIVXP37oy9w3oj3fbOBCDEdcJy\nheAHjyve92ZJrsFX1141cJxb1UsOAj7ykY/wsY99jI9+9KPEccxv/dZv0Ww2+c3f/E1836fRaPCv\n//W/fi2u9XWpFlpqYXEjavWunZc/D8qkS20W/uATHP4nd7LwcIzVKek3H0dIQTmwHKutshqPQGOE\nIdWhdX6GkfsPsXhxiYNmjksvtDl8fJHHuRffF1RLmjT3KQVQ9/rMdyqUohSXwM9xnH8s11a8sRyd\nyJnpRaytlDGmqAg80rRcnMlYmO0wOdWg0VAoJQk2atj4Hiyue1gr2NXMNuraWLK8aAMDZVjpbJ0J\ns7YoTKZNMRDo9AABzaGAXs8SJwYEm9n2FtsenYHkfeGA4YpltKa5tLZ9JWCkqhkqX3sQkG3sXSj5\nELiMoM7r1Ev+tKMo4vd+7/e2Hf/TP/3T78kFvZHJ68wy2FaLS597gvq+YaJGRHl3FUYmyRaWGKSG\nlbzOveOznOjtYf6O97B/9QmC2VnM7AXOd25nVJ2h9f98kbWzR+Hv/g21j/wC2Y//AlIWFR5vb66w\nNKhxYb1CoiWTjczd+BzHedlcW/HGcsdewd99qsP+fSXKZUWWWZ47mfL0U6scu2+MJBd4KmT3hGKQ\nFJtppbD0+hqBIs1g31hGJGLeMdVi3UxweNzwB582gA9cnsa35JkhTzVRPUBKQbkkyTKL70OSQhBs\nTbc9yCSn5n3edjjlvn0pnVjSGlx5TjnQ3LcvY6cmN8vhqZmQC8semYHhquXAaMZtYzss0zvOLW6H\nbT3OjRIFwY5Ll6K9xsz/+YfEiy1MpunOrVIda5CvtCEzXPr9P+IjB05wPptgfinhfOVedFAhHK4i\n8wT19a+z8sXvUPnpf0r35EX0cpfws3+CuHAKay2jpXU8BYHKEVmf/voSf38i4NSiCwtyHMdxdmJ5\nx1sijk7lTNb6lOlS8lN+/Mf3sn9fhJSKsVFFp2cIPMMgKVYDQl/S6WmM8Hj0BckL5yw11eNgOE06\nGFAqB0Qlj6ikiCJFFHkEftFVGR728TyB70vaHcMgtlTKkijc3pXppUVrOlyxvP+eAfdOJexpJhwc\nHvDDxwbsbu6cGeihFyKevuCx1oVuHy4uCR49F3Buxc2KOa8/bhBwE/E9RbUUYZZWsLnGphnxY08x\n/y//d9onzhdPUlB7+3Hs0fvw0x7VeyfoP32JPKiwklRYnE/pdeDry4fI9xwgqISohUWqk03EHXfR\nONTEZBYx6FJ9+FPsqawwWWmDEPRiyR3ZM9xRvsTxsQt86QmYWXU/EcdxHOcKay0LbctoLacUWOpV\nyaF9Pm+/NyDybZEJKIChqmJ1LacWGbQ2KGEIAtCm2B8wPOTzyCmPmVYJqxNsZ5Z6tDVBvxCCMPJo\nNjzqdR8hBKutogOf55Ydp/OB8lUbhM/PpHzpK0t85rNzfPIzC3z8z9Z5+tT2QgALLcHMisJcFZlr\nbVEb4IU5NynmvP64Ht5NJgp8hkaarPz6v2L+n/0SS7/8a6RPPgOA36yw72c/hFetwj0PEP7gu5l4\n614a3/8mOj3Bvt0eCIuf9jgbHeNT4T8jN5JdBwLG3nY70eEpKgd3oaJiWbR+/js0RFHIJ8kEpxcr\n9AdQWZ9hLGjz1oNt/vZxN/vhOI7jXNHPLPEOCXJ8ZamFKUIIPAWeZ/A8ycWFIrtOoAyH8hdQQrC+\nnmKRNJsB35puoq0gUjn3TaxsO6+UguGRiCQ1zC7k5Fe99+rygOnzHRbm+2RZMTgIPcORiSJ8p9XT\n/Oe/7nLqQk6uiwHI2Us5/9/fdllY2fohnpnxuVbJgfXuy9sUbC0stWClU/ztODczNwi4CQVDTaZ+\n5ecpN0KE74ES1O7Yy22//CNU9o8DYKXCjE4xdOwQh9/cwLTWqZQku8Z8jpVeYK5f5tn5GtkP/Bj7\n3rEb2xxheDBNfvAudr33TqKJBrWaYPDw11nuBjx+cZjlbon5bBhlMsLeKlPNHp5vEfGtmULLcRzH\nefUl18mQ6UtDu5OhtcZTAp1DN4aF5YwwMIR5B5n38T2JNpahhsdy26OdFykPh0rpjudNUsPMfP6i\nWXpLPNDkmaHbyViY6zNUznnb4YSRatED/8ojMSut7Uk3Wl3LVx7dWgfAu04//+UkBjo5I/gvX1H8\nyZc9/uRLHp98WHFx+aVf5zg3ipvmvUlVj9/HXf/mf0HPzGDynNKu4c30ZEZIJBojYP6JOUY/+hbW\n7UFIEnxfElfG6XZyohC6OqD3lg8zkS5SXp1jfWgX3s/9c3bPneGJ+3+Ju77173jh8XkulHYBEIni\nBixNjpAwOaRRnQXy6NbOYe44juO8OtR1HhMYTp3s4vkSbXyEhHpVst6Gfj9noXo/t81/lYXag/T7\nIVEkqEaGgQ1JrYdGEQSCTjtDeQLPU/jKEidg7NaeeDzI6HYSolIRJpQkhiGvz/6RK/Obrc7Osf8A\n7Rdl5Jsc0pxe9NgpsWijfO3zACyswxeeUgySjXbawsyq4HOPCT767pyyqzLs3ITcSsBNTAdl/F1j\nlCZHtgwA8lIdz2SofID98E8Sj+0nr4wzHp+jESZ41hIPcqb2+szZ3bT9STydwOIlvLEJ8sYuqrft\nYayW8/xtP0mjUfwMGrLDveWzAOTSp0eV0XoO0v1MHMdxnEKtJPB2bBYsVa9Pzesx6Of0BjA6EuKp\n4vlDtWIg0I4mwKQoafCV4YGpVVayOsv9iIu9JkJYVpb7LC30WFvpUQoMoQdcTheqNb1uwupiD50b\n8vxKZ36ptTUGZ6h+7SFLs7r1Qxye0EzUt3f2A8/QrOU8fCZAXyNe6KnzcnMAcLVWX/D4WdeGOjcn\ntxJwM1OKuDaO1RqpU6yQpGGNwCbIPCPDp7aniUUyIpbwkhb37ioTrl7A8/ayb1zR8kbJEk08yAhy\nje33qI8M00+nkMLA6C7kzCr70uc5Pr5AIHNyFTBfvg1Q5FqQN/be6G/CcRzHuUlIIRgqGZZ7GosC\nBAJNIDNKKuPNR+GhExKlFI2qxVqoVzRCGNa7krXoDoY86HYzvJqkk3h8+xQcnhhjzQzjqaIjbi0M\nBprpmZSx8QpBAEmcszDTQV/V8de5wfeLzn45vNIRtxaO3lFjJaswiC3z830WF4pKxkM1yXveGr3o\nc8EPHUt49Lzh/HJxvkY559DEgHJoiDOfr56OeM+RZFvhsP72fcabevG1H3OcG8kNAm5imReS55ZS\nZwEv7WOkh6lZBrVxAtumH41SyS6RWMOd5jnWyzVKep5PnDzKgw+UqZUNvSRkPlbktVH04ipLn3uM\n0f/+B8jCKtJoqqEmnHmOd5/5HOU3v5ne4buZrR4lVk0kOQoJXnCjvwrHcRznJuLJjLo3IDMeFoEn\ncpQsZsmHGzC5u0KWWxpVi9aWfcMpxnhUqx7zizFhENAMBuwaq7HaGqfdzbgQ1ajVwLxoR208yMgy\nje8rwsij0YxYXe5vPn65P16J4PgRS5pbpIAnZ0osdj3GJ4snTO2rcfFCm6TV5gMPlhltbu8CBT4c\nnMjYNdzDV1uvoxxkjNcF5+c1WpXRwGQ9ox5ZatG2U21qlF/BF+w4rwE3CLiJaSNozD2Dl1+ZRgj7\nq/TSAd2hKQyWXEWUeksMnnmC6Mi9fPPsLg7ct5d6TWGwICH0Ybp0F6PqIVY/8XdMfeAYbV1CDE0A\nmvGn/it4ltapaVaO/xxCChQ57TTg6O4b9/kdx3Gcm5MnPYQo6su8mLES5Xk883yPo4d8RpqCcgSX\nVhS5hqXFAVobxiYz1tqC0aZCKc3ifI8wrFEtK8bGQpaWiul1a9kcBACE4ZWuixDQHC0T+YK9Qxl/\n9U3BaqeoYFwq50zt8TYjWqUSHDhY54H9HsOV7ZuFL+ulltDbOewn8jSPnQkIK0WQ/7nlgL3NjPsO\nJpyek3TirSsEIzXDmw5d+70c50ZygWo3sfLaxS0DACi2K5Xac7CyhLSWKO/TbsHC47P0Rcibu1+i\nWffwlcYaKKscJSyJLNP+gY8ydNsoph8T5W18ZSgvn9u82XnLM/iXTpIYj9iUODgsqEYux5njOI6z\nled5eGrnecSzCwHPvdCl3y8y+jxzKuPJU4L1vkevZ/B8RTywrPV9Ls3laAvDQ5I8t7TWYqJIMjFx\nZWpdyq1Vge3GSoGQUB8qEUU+KI/ptYBLy0VoTqcPi8sZ56YHW67NIphvX3/+s+Rfu9MusEhxpV3M\njeD8qk9fe3zgeM6+MU3oWyLfcmjC8KG3aAJXYsC5SbmVgJuYF3d3PK50RrRwmrg9oFTpczEbx56a\nIfjUX1I7spugM4NpTOCJDGNgLOpS9QZUKzmld+5n4T99gvr/+D/RHwimPvV/bZ5XAOM1gRqVxX4B\nx3Ecx7mGclTm7FyfRiXHV9BPBOcWAh49U8YPLEms6ceWQQKrLRgb16ytpSgJvifwymXKac5qS1Gr\neAwPW5KkaHsqFZ9SSTEYaEplH8+7PAiwVEJDrRlRqYWbqwMASimCwJCmVzb3rrfz4hyl6+U02mqk\nLFjoFnsEXizOJJVaQLpl/7BgoePxln05U2OGQWqQoliFd5ybmRsE3MSsCoDejo9FSRvVBeGXqT3y\nOebOzGG1of5jH8D++R8jVzLEv/x1FnoN3jQ6jTUZIs2xD76X3fFn6D3zOFPP/tGWc4qJKbxDd72s\nfMiO4zjOG1ucKT77nTr1ck6zopld8emnRWdbKUG9JjY65IJcQ7utyTKDlIbmUICHplyVICWVSk49\n9VldKUKA8txgLSgli5l+wJOWI5Oa3sAjFTvn3AxChVKCJNEYYzEG2t18cxAgsEzUr1PoAAh9iRIC\nY+2W9jDVgktLklRu3yenr5o3K7ltdM4twoUD3cRMY2LH41p6hNUS5fYC6WqLYOks8aU22coy0enH\nuX2fJv/AB1n6959gJOpQsV3WzQhKGGbrd9Nf7BCuXtx60koN9a4PI1w6UMdxHOdluNxBXm77nJ6L\nNgcAl+2bCqnWfIaGik68FFCqeExMNvF9hY67ZLkljXNmFyxCetTrxXN7vZwsK8JuTJrwpgMpP/bW\nhPfcnVEKrx2mao3F8yXlir8ZQtQfmI0QIsveoYyR6+wHuGx3w6cWSgappJ9I1roKtCKVO9fMqUdu\n9dy59biVgJuYKTWwSoHWm6VLrBBI3wMpUL0uXsey/O1prLWMHG4iOh2CQ3fQ6Qc0vv9u9tZbWHza\ng4iyP4ySIdNfucQdf/XbqG9/Abu2hChXUcffg5xwqUAdx3Gcl6ccwnBdsNqVmyMCnRu0tpTLknLZ\n48A+j4WllKEhRRgGVKze2NhruHPS8PSsz8JCQqUcYLQhjlO6XY/p6SvZf9o9y1g1Y3yjps2xfYan\nL1heXNTLGovRkKU5UdnH8yVCWDItmZ3p88EHYLLx8jrrQgiaZZ/mVZl9rIWVQc5CZ2ucTz3SHB7d\nudKx49zM3CDgJiZMBmEJdI7VurjJen7xX2OIF9aJ9gyz/PQalaky1oPlp6YJO5a9b76P8ac/z7OL\n/4LbxixpZln3R2nma9R/819gVpfwf+AnbvRHdBzHcW5Rz89IepmH8q50xqUUKGMYHwsQQqAUjAz5\nXJo1NBsCnUkyYzHaMNcfZr1niGNQEqxNmbnQZ/ZSjFISdXW8/1X9/fGGJZCaOFOIjcVra9isHaC1\nxZgiTaj0FULAwqplbsmyu/nKP68QcHwq5vSSYaWnMBYaJc3hsYxgozdlLSy2Jet9xXgjZ6jskms4\nNy83CLiJmVITay1CefCiLAw6SemtxFQ6CeWJCN3XLD22SLI8YPxtLcJ2laWvfQ17ocKlX/gVyrbF\n6Ys1HhxdQdxxlLn/998z9Zv/CpSPNQaLQQi1rQCK4ziO47yYtfDUtCTXW9sMIQRBoKiUroSWBoFA\nSpib6XPoQIlWD5Tv0Yo9ur0eQSBYXRmgc1104K1GCIHQBqkku0dg366t77Nv1PDsBbEZknS5tIA2\nBm2K2gRSFmsFRhcPPvS0ZLENb7vdMLxzVM9LkhJun0g333N61efJmYhMC3xpaPcFy21Z1E6QAbub\nOQ8eSVAu0ta5Cbmf5U1M1yfR5dFtx22es/TQCcbecRfl4TIqioiXE0xsyfuapJ3Q/vTn8ZRkX+tx\nDmbPMtSU7B2O+fyFQ/RkDfmjP4V96iHi3ir9ziKDzhKD7jJpsnNGIsdxHMe5LNOw1t25C6E1xMnW\nsJs8M6ysxHiexVhBHGssYDJNueSRppow8otOvS0688ZY6mV475sU8kUTVO+82yBFsXn48gDAWEu+\nsRogRLE6IKQgywxKCowVnJzz+KtvK7pbM4e+IqeWAp5fDFkfePRSxXrsk6Pw/eJacyO4sOrz14+H\nGAOnZgWPnZGsdr7793acV4NbCbiZCUF84G34Z7+NXJpGKEW23mHlO6epHNpFbTQEY9BpURRsMF9k\nEhosG+JLa4zcPQw6JWBAY+F52uXb2TMesJ5U2TtUo3fhDO3/9kUaP/wOAKzJyeIOQij8oHQjP7nj\nOI5zE1MSAs8SZzuvHntXxe/EsWFxvoc2lulLKZVamSS1BIEhKoWkqUFIgTEGz1N4viKJM8JIsnsy\n5NEz8PBzgqGK5d6Dhsg3/MOzliwxxGlRBEwpiTEWC1fCkwTo3IItViKUEvi+ZLVjeehZwYePXz9L\n0GWvdgUAACAASURBVPXkGmZbHi/elyClIAosaXbl2Hpf8h8+5zHIBCD4hxcsRyYNP3i/3jENqeO8\nVtwg4GbnBWRHvp9Sfw3VXkCVYOrdRzYfjldblIYF/ZkrL+lPL4MSrL+wCFgqJ89StW3C2/czVJL0\n8waJhLN7f4LRf/dz1N77NuRV1UzyrO8GAY7jOM41KQlTo4ZnLm5fDShFgiAojqep4eyZDpaiIz47\nl7DX8xkMNDrXIKDbLZLuD/oZQhSz934oiOOc+TXwPEG3k7PWUUXV4czS61+JtTfaYozG9yVKCYSU\nWL31moQQIARSFtcxvy45vexx2+grGwisDSRxvnPtAfmiw54n6KcWsbGBIc0Fz1xU1EqW7zvqsgo5\nN44LB7oVCEG69x50prcsieZxSuu5c5R3RfgVhRqq0Xz/28FAUIuIl/v0znc5/3/8R+h12LPwdQ40\n1vFNzMCWSXKPo//ze1j+T5/Z8nbWuJuS4ziOc33vuktzcFyj5EYFXyySnEE/Zn4+5sKFHo8+ssJ6\nK6cxVEJtFPzqdovY/3Y7xxpI0yIDntwInPcDhacUvi9ZW+4jAM+XdFoDWq2Y3Gzvukjgv3vQcN8h\ngRQ7d20ERehQo6GQEha6HskrXAwo+VsrB1/NvuiwFAJpss1Kx5edW3RdMOfGcisBtwg9eoDFR09R\nHSnjVSL0IKF16gLJ4irKV5T31qn96IcpHT1A66FHSDsxbGyGSpfWyeZmKR8+SlfH1GnTt3UyI+js\nuZ38r/9my3u5WgGO4zjOSwk8+NG35syuCubWinCdb59WnJuB1ZU+xoAf+lTrIZ6viEoe62sxvW6K\n8hTWQpzkGGPwfUWa5Phh0fnXmUB5ivZ6n+Vlychoibjv0evEyB3aKGNhcV0wNQYnpne+XqmKDQfW\nCHRuybRguSvY07T0E8OTp4tm875DUCtfvx2shpbhsma5t70blb9oYBEEUK4oevHW2J+rQ4Yc50Zw\ng4BbSJZLlv7hyW3HLTD1P7yf8g+9m+5si9JIjf7M2ubjo8d2YdcSBpUxSt1lamnKeng7Ruf0R6YY\n/rF3bTmf55dxHMdxnJdj97Bl93Ax6fSt01CuhpSr2yv6SikplTz6vXQzNCaJc3xPYYzBWEs5CvE8\nRRgq6s2QXjsm7qUk1SJk1Q8Ua0sdGiPb0/ucmQNtLY2KodXb2om/vB8ABFmmsSh6PYMdsXzrOcNX\nn7Z0NkoTfP0EvP1Ow7vvu/5A4K5dMU/PRqwNFCCwttgL0Ltq07GUlnKQM5NuDx0arrn0oc6N5aZ8\nbyHhvcd3PB5NjjP69mOUukuoQXvLAADAClg7tULymf+KCSLGuqcYlUsMB11Ea5l88gAAUnr4Uc3t\nB3Acx3FekfHG9Tu2QgrsRlir0aZI4WkMeW6JSj5RySNLc2p1j3JJkWWaNNX0ujFQhNqkaU6abJ9G\nn1kRfONZ6PQMYmMjsFTgB5KotJECWwAIPE8QJ9DqGr74+JUBAEAvhq8+ZTk1c/3Q2HJgeWD/gLdM\nDTg6ETMaxSSpoZias4S+ZbRhabf1tlSq5dBy30EXeuvcWG4l4BZS/uBPoVtrKJkiGw1skqAvTDP0\n5ruLnMpY/Hh7is/WySVsvszuikcwc5rurjuIpMVLUtLyEMlSj0f6+7ljMue2mrspOY7jOK/MA7dp\nzsxLBunWOUZBkTknSw2eJxESdGrIsqLN8XxJrR4hBOyaDPF9xdLCAGshz3KMiZASsqTYRzDoJfiB\nt1nbRiqx+fcgAaU0pbKH1jAYZGRZsQrheZIwkqQphDLj778jSbIX7SKmSIF64pzlyJ7rf14hYLSq\nGUVzYBj2jxqeuijJjARrqXqGqSnNRA3m1z2SDIaqlvsPGvaNuZUA58Zyg4BbiLCa+vt/CJFftdb4\n9rch4zZkxTH/wD7Ul79B9rm/Q/3h71F98x10HnoMgLlvniMY+wLhL9+N/IeHKY/fjjq6i13Tf89X\nho+z1otQMuHg2PYbouM4juO8lFoJ3nlnzheeDjaPeRsd9MEgK3rNFP8flhRKSbxAUi77KCUJQ4nA\nsrQw4OJ0GyhSYAupiPsp/Y0VgSzV3HvQcGrOI9NiW6FLrS3WQhAopBT0+zlaZzQaPkIopM2Ynrf0\nE4HyJHm2fQLslcTsj9cNP3i3Icvhr78FXz8JcQqlAI7syfnJB8HbOamQ47zmXDjQraS7uHUAACAV\nWVjDbuQqzkWAV68S/MRPEn7yU0z+8k9vPtXzPVaemEV11+j/6V8y0j6JEIJGPMdPhH9Frg0n59y4\n0HEcx3llrIVTCwFhoDb/UUoipcDzFGC50l+XjO+qMDZWplLxiaIiZKc/0FycbqO1Js801XoJow3t\ntaIWzuUsQkrJYlXhJSrdK1WE/2htSVNDr5ezsp6zvlG0S10jWf/EsGBpHf7uEfj01+HLTxShQi/H\n33wHnpkuBgAAgxSeOgd//+jLe73jvBZcj+9Wkvd3Pq48cj/CywasehPFIQnp8G6ysQRZCtn/z99D\ndOwQs3/+MJx8Ft3pE+4eZz6vsCfXNEWbnxh6iL8fvGvn93Acx3Gcl9DuCxZbO88vFhtz2czus5G6\nfxspBTrXqEBRa5SISgHWWurDFUAw6MdYa5lesIzUDDMrO6QM3dgTULyPQClBnlu63Qxj7OZxIQRS\nCfZOVcAaVldT+n3N7hFoVgX/+YvQT66c9/mL8OPvgF1D1/4OBimcndv5sTNzkOXgu96XcxNwP8Nb\nyXXCB3M85r2DzAaHN4/5StOiSe1T/4UR7zxGZ+x92wT4EcpPScb3M/TQJ8AaVL9NdbfPPcmzwG3f\n+8/iOI7jvC69OE/+1a6etY9CQa0qN8KFINeWOLYkCCb3j2x7XRD6pElOqRzR7wyYXzEoaRmuh6y2\nryoeZixJnIOAWi3cOFY8pvWV51lrEcKiVJHdR0iPvXs8KrLP++4x/NlXxZYBAMBKBx5+Gn7qOvNl\n3f61Vwx6cTFIcIMA52bgwoFuJf7OWXtSLflG/n08Ze9jkF0JNqyJFikhuj7MTO0u/HqVxpCiXLaE\nx47S/uJDZF/8W6y1yDzFzxMOpSeu3C0dx3EcZydGE1x6itKzn6d84m8Jz34T0W9RL1vGGzu3IcZa\nPE9ircUYw/paUkxCqaKSb+BLKmVJnu1cwevqYlthudhzMLNUrAbkWU6eabJMk8QZWlu67ZQ0ydH6\nygbky6sRVxPAoF+8pzbQHI5Y7UoW1nf+6DMroK+zda5ZhWbl2o9Vomu/1nFeS24QcCupTmC9rXeP\nzCrOZ1PEslgmjXOJNlBNlznQeYrAtxihSP0yamUWUSoRxm3Kgcb72z8n68ak3Rg8D5UNKJsW8szj\nN+bzOY7jODc/a4nOfINw7hm83jJqsE6wco7S6YeRSYe79mTblgOMseS5xfOLGH4pJWHkcfp0Z0vn\nXilBtbp9mlxrQ5Ze6XlfvaLw3HmNkMXgwhqz5a17vYw41ggBnifxfbUlBMna4lxZZjGm2K/QGly/\nayQ2/7Uz34M79+382N37i3Bdx7kZuJ/irUT5MHQQWxlnRQ9zKR3nycFRZvJdVz1J4MfrHG19jXK6\nRsn0CUoevspRl84iazVEGhO8/8Pw9vchlETVqzAygchikmgIdfKbN+wjOo7jODc31Z7HW5/Zfjzp\nEMw/j68scVLMvue5KWbnU7PZ4YZiL4AfeESRx9ra1tgZX1oEGp0brLVobUgGW1P1XD1w0Nps7C8o\nBheeJ656zBarDIHa3ERcKvnb9iJYC0ZbohA8Ydk3BhPNnT//npGX7si/73545zEYb0IpLM713vvh\nHXdf/3WO81pyUWm3GqmgOs659YjODhUIAcbj80RmQCpLVPJVemqKRrIE43v4/9m78yBNj/rA89/M\n53zvt+7q6vs+dEvoaCQECCOEAYFtbDAz9jjs2ViHY8yGZ8fgWMfaDkf4j1mHY9fY4XBsrD3r8Y7H\nXmAxgwcEBoQsARJIQrfU91HdXff5Xs+VmfvHU0dXV1WrJbWk7lZ+IhR0v8dTWW8X9eQv85e/n5ga\nQScZ4aYNODtvxz/3EtJ1EY5D6gSQzmNa8zinn0NtufEt/uYsy7KsK53TGEesc0hNdubo7jf4riFe\nI6tncfK+OGkXjqTVyjh19ByVesC2HV20OoqZ8SZpZhYO9Upcf+V0JY0z3IXEemMEWpuFHP88EBBC\nLZUIdS6YsTuuICx4dNop8rynggB6alBxFY4D99wA3/wxNM+LUXqrcO8l3BqFgPfeCPfeAJnKy4K+\nShEjy3rL2SDgKlULFY14dRAQ6Bab0+MApG6BQtogcwTFqZO0S104qcHLOhR0A8d3Kf0P/47sO/8A\nx48S3/I+HCU5sfWD7Bo5YoMAy7IsaxUjvfWfc1yqRdjcqzk6uvoepbI8N18ulOWUUhL6mvGxFuNj\nLSbH2hSrBTIt8vQeY9BagcjLXBtjSJOUrv4KhYKg09akiUZlGqUUvp+PLSy4GJ03IVsMDgAQLOwW\n5Kk/vu8iEBQKgk2DEteBGzbldT33b4aBGjx5FDoR1Ctwx558Zf9SdWJDlORnAV6tlKllvdVsEHCV\n2tqV0oglc9HyP6GrY3YlLxIQg1YErUkmi9uJE0EjlQx3HWSf8wyer3AbU8hiQtI1SP2ue4gOvYJP\nSlqugd+LGD30Nn53lmVZ1pUq7duJN3EEJ1lZttoAqp632H3fdRlTDUkzyvcMjGEhNWjxIHA+OQ9C\nwexssnSNudmIKNZ4gYfrOWQL3XyzTGEw+L5HvbcC5JPqjZsKjIzERO2U2ckm/UN57U4hBUHg0mrl\n1/Z8h3rdo1R0aLTyFKFC0cPzHAJPs2e7h+MI+kspheU+Z3RX4f5bX/tnNNfSfO0HihMjhjiBgW64\nfZ/krgN22mVdOexP41XKc+CmoZiR+YzO2WFc1WFzcpS6ngGjIUtxEEybGgXZpt23m24aZEEPMp5A\nYNCdNvNhlb6eAWT/LFJlOEJSV+Nk4TqlDSzLsqx3Ni8g3nwrwfAzyKyDlhKRZWgnRBkJxvDDwx7N\nWILIz9AKkVfmKRTcpTKdnicohvDi8ZVleJTS+AtpPZAHAUIIat3lFa+LIkO56OC5ktSV+L5DuxVR\nKod5/r+E3p6A6emYJFZo5VCrhriuYnpW4Xn5TkW1LBHCUA8Vu3oS3ihjDP/wXcXJ0eWUqZEp+MYT\nmlKouGGHbRlsXRlsEHAVkwI2lmNKzUeR2eqixAJDt5kiDrtxdABjp6CrAFkGSlOKJjnObrSj2dQ1\nh4dGGqh1JmgPbkfGLdzABgOWZVnWSqrcS7RxHzKay2f4nSbO2DDhj79KtOEAp2cfXPUeIQSeJ5DC\n4PsSoVN+/IMxsmx1SdGlRmICWDhQnKUKKcVSx+DFwKJadYlihRf4NOc61BeaiiFAGcHNN5R5/uUW\n8/MpeoMhDCSOVCidBxyvHOowPgIH90u8oTdeL+Xl05pTo6vPTKQZ/OSwtkGAdcWw1YGudtJBFypr\nPpW6IX6tgOdokFA48jTlaAapEzCQJXlDsePRAPPV7YwWd6CQzHTvRVV6GJuJWeN3s2VZlvVOZgxi\n9iRTkcchuY9DYi8z4UbU5t3oco3w3AvsiF9Y863FQPAbH874xK0tnnlyZM0AwHGdPF1IG4xe3DVw\nMBpUZpbOFZSKDlIK4kSB0RiTnws4/5pCCKIUbryujBSglcJxBJ6fnzmIOinGwNQcfPNJzYsn3/hN\nb2zarNvbc659kU5qlvUWs0HA1U4I0oE9GLlyU8cAUX0DWroExEycmIGN2/GSJkIplB9wVm6kQEKc\nuZzztpB6RcbD7Zz29hFpD2MMX3/OdjWxLMuyztOa5LDazmSwBeMXMX6R0WAHh7wb0b1DCGAnx9Z8\naynQuA6AoK/bw3EdXM9FOnn5Ttd3CcL8cK8R4Dj52QEvWD6MrJXB9wWDgyFzczGjZxp0WilRJ8V1\nHLI4BvIKRaiEmZkUITSbN4UEQR5ctOZTGrMxndZyCaM0g2eOvvEgYKBbrNtGoFa0h4OtK4cNAq4B\nWf8uOjvuIi7WSb0CcaHO/MBeGv27cXWKpyNuOvn/UR/qQpe7UGmKOnWK9PQwVTNNlgpwPTyd0nZq\nxNpjTlVpJAFBKDg5bn9MLMuyrNxwu4jwfKRcTtuRErQXcLa0B4BKsFZLXcPOQY3Whv/7aw2m5par\n9kgpcVyHIPTyXQAW6v4vpP50WitTXvt7fZIo5dCL08SdlDRJUZnC9T2kgO6aw/6hJvfvHObs2Qbt\npkIpSFOFlIY4zkiS1WMcmcqDgRWjNoZzU5pTYwqlX30lf/8WydbB1ZN9z4Vb9tj7qXXlsGcCrhFp\n9yZiZ6E6g3AxCALVQWJAZdDdj3Ac0IYsE6QvvUBp8xBpuoUgEJS9CIxG6hQQNLICp2Z8XE/y0pjP\ntv7VZw4sy7Ksd54OBZw1FrSlgJniFrYCXVs3sD/KOD0paceCWtGwc1Bx2w7FD5+LGB5dK0iALM0n\n8ouEFAvnAc5foTecPjHH9NTCfUmAyQzFso/ne3iepKvucuS04URxI0MbfUYnEzqRJpgx7NhZZ+PG\nAkeONFd9/UYH/ubb8Av3QrUIJ0cVDz2hGB43aJNX+bnnBod37V1/+iSE4NP3OUvVgaKF6kB37pf2\nPIB1RbFBwDVCOD5ID6FTXLNyGUPELczemxDGIAA1PgpA4pbxpU8lTNDCIxM+JT0H9BCnLu3Mpys0\nBJ7NYbQsy7JyFy13LyTDOx6gZ8d+3i8zkgw6iaAcmqUuu1Oz66fcGHPh3xebgBmSJMX3PbJUMTe/\nvDDleg6VWpFWI8L3XaQrKZUcZpuS9nTC3r0FTp2Yo6unxPCZBtu3VymVHCpVj8b8yk7E0pGMTsP3\nnoX7bzN86ZGMqbnl58em4es/VHRXBDuG1p/QV0uSf/VBSTta7hOw2BvBsq4Udl/qGiGEQIbV1U9o\njeN7CCnAKIgispPHaQc9jG55N4dmuhkqN2hnDpkWOFoBhpmmS+CB52qq4dorNpZlWdY7jxTrLwy5\nZEz3XocWDi+cgidegXOThvPnv4O960+eL2yopbVBa00QejSmW6hMEbVXlvEMCj4Iges7xFFKoRQy\nPWfItERlhlMn50jilPmZNlKAFIJ2x9DXF573dcFx5VJ34bOT8PhLakUAsKiTwFOHL+3sQDEUdFeF\nDQCsK5LdCbiGyEIdhETHTdApMu0gswhXpQg0ZBnZ0Vdo+T2cPPBzTJs6zXlJlrVwBDTSkIAmUQKt\n1KVaMDhCcePQGr3fLcuyrHekvjBlPPJX7QhoA5vTY8zGJf72u9sYmYbFGp9PHoWP32WoFuG2/QHf\nezLixNmV9xbpCMLSwqFgY1BKk0YpjuegjcBxHbIsAQxSSqQj8AIPx3GAxUpCBiEFrbYmCH2arTZT\nk4p2s4M2ee+BTEEn0oShQ7nikST5hP78ACRKDXON9YOdZsfukFtXPxsEXGNkWF3eEVAZZT3N2ZEG\naQat0Rkm/fcwc+8tBIFk0MCRkzDb9sBxQDpkuo/MCLrKCldkbO1K8exPiWVZlrWgWg5J54aZ8YeW\nc4OMoVePUOuM04q7GZk+P0IQnJ2C7zxr+JmDeVrMgQN1Jpst2q14YaXfp6uvRBh6jJ6dI8syQFIo\n+wgESpmFvgEOpWqIWcgbWpy4K2XIUkVYzHe+00STpBqBIO7EaGUQKqFSr9GJ8wpDZ0/PUCqX0Dpb\namC2qNE2HDonkQ5otXrVv16yK/vW1c9O765ljkurtJ2nTho6iYSagBqgIekYSoGiqw7d5ZRXxitk\nBrqKAb3FjK6Cor9i8OwZJsuyLOsCG5JzDKbDtP06Qgp81caNW3iNSUbjoTXfMzwhiFND4MF022No\nWzdaabQxOAslQjGGsOARdfJmX1oB5OcCVJbvDKhMs1iIX8o8DchxHKQj8sm+EERRRhzlOw1ZmmGU\nJs3ynP84UiSJYmy0zdBmn3LFo9lYDgTycwj5WQbPkyRmuV8BQLUEB697YzfHM+OK8RnDzo2Svr43\ndCnLet1sEHCtMoZs5DBadbin6GIqLrOxz8Nn91AsBziOJEolhVBRdFO0yWsnHz+XcfMuYQMAy7Is\na12qewuFE4/jyUm0H+DEbaRWxG6Zbx7bseZ70iz/L/AgXThqJh258nCiEPkZNlia6BtjUKlamJwL\nVKoRMi8rqrUhiTKCUCwEB4IsM7QaCSrTGG2QSJTOuw0XCy7tdkqWKTzPRWWKWjWk2YhJkgzHWTkt\nEkLgunKpOpHnwifukQz2XPqRSmPgySOC42OCVgSNpmJqOiOONcUQbj8wzwN32IPD1lvPBgHXqOiH\nX6Pz0NeR7QbagNy4merPfZr7tx/m60d2UKoUwHfoCSMCVzNYadFMQ0YnJKzb69CyLMuyQNU2EA3d\ngD95BK/TwABZoYt0ww3URjwm1jhQ21eD0sJZ3N6KodFZ/ZpyqNl3wPCNH2R5zVGTp+MopZHu8sRb\nZRotDW7eeYwsUyRxRrHk0mpE5KcD8gDC8SVID+k4OI5AZYbJsSZpqqhVXYwGP/A5eXiCeneBcr2C\nXqcfQJrB5Cyw9dI/q+88J3nmuIClFmKSUtVBzUa0I8UjT0dIXB64y7vYZSzrsrPVga5B6cs/Jv7K\n/0vJV1QGa1QHqwTz48R/+X8QSsO7+04yNZORKUNVT+FnLYq+puDl7dSLnt0FsCzLsi4u699Fe9/9\ntLe/m/au99LZex+m1s+tOw2eu3ISHbiG23abpSMEt+5QlIKVr5HCcGCT5kMHQ/7XX6twYIsgSzO0\nNnlnYXd53VIIQZZkKJVvKWSpwvVcjDF02nkakOs6aKUpFAuUSgX8wOPM8CxzM23mZ2PSOCONM2Zn\nIowxBKHP7HSHcvmC9dEL6pZOzV/6ZzTXgleGzw8Aco4rKZSWJ/2vnLJV+Ky3nt0JuAZ1vvifKPWW\nlzotCiHwyyHCSUi/+WUKD/wb9iRNzrbqFAoRgWoTa4lwMrYMOgxW7S8jy7Is6xJIB1VfeQbglp1Q\nDA0vnoJmJ2+6deM2w44Ny6/Z1GP46LtSnj3hMNuG0IOKF/HSc7M8/kNFT93l/jsrnBqD1kV6VWql\ncRwnDxQciTGGNM4wWiOlxPM9lFYUiiFaazrtlPHxFhgw2jA83GJoSxdz023kQge0iXNzVLrLRJHK\ne+tccGi4XLz0j+fYqCBK107zcc/b2WhFi/0QbEqQ9daxQcA1yMkiZDFc9bhX8Ilffgn/gZTthTEm\nsiqDpRYOmi45z7H5LrbVOmSpA/7bMHDLsizrmrB3I+zdePHU0sG6YfCWfNX+saca/N1Xp2mdV3rz\nmZfbDA6VaUWrt6YXqwMtHthdrO9vzMIZgsygpQIBjuPgeJK4mSy8B8o1n+ZcQhznO+BzM52la8Vx\nRt2VGJORKb1iI6C7AgcPXHoSRSmAPDFp9eT+/Ov21KQNAKy3nE0HugZd7HCRSRWl00/idOYoZPME\n8Uz+uFYUXSCZ5W8f8fnLb3qMzrxFA7Ysy7LesZQyPPQv8ysCAICpWUXSjrnwnJrWeikNiDXud4v5\n/EobsjRDCIExeTAggKDoEXc0jitxPWfh6gYh8z+F5WBh9V/gOJIgdCiVPTYPCH72XpdS4dKnTrs3\nGvprawdDSZwHQIEHd+y3ObjWW88GAdegLFn7F47RhvnxhM5D36U2eYT7n/tD+P63oTVPe6LBnuo5\nBisRB6/LqFfgiz8MiNM1L2VZlmVZlyRKNVMtxVRL0U700ir+omPDMcOja99sRicStg3kZT5VpsjS\njCzJMCpPn/GDixymNSAWUoQWdwp6Bqt5VaBUUSoHdPWVSWKF60kwAs93kELSbi2OR+C6Dl3dBSr1\nEql5bQkUUsAHbtT0VZd7DQgMQqV4JmHHkOSXP1rh1r02McN669mfumtQu+96/ObLeIWVvxzbEy2i\niSZ01yg5RapkdLSgMHqS67ITRN5BYu1RKsDeLYqZBnzzGZcHb7cdgy3LsqzXbrqtmY+WJ/2N2FD2\nDT2l5fQXzwUpQa/uyYUj85KaazXsko5ECoG+IKjIV/4XmomRlxyVUlAo+fiBS7sV4/oSA/i+S6sZ\nE4QBadKmd6DG3FSTsFrEW+iUuXj9+Y7kB4c9Brtiaq/hXMDGXvjX79e8MmxoxrC1zzDYJTAmRAhB\nX1+BiYnGpV/Qsi4TGwRcg/p+87Mcvf9D9F3XT6G3hFGa+dPzTDw3QXmrz8zDh9l+2xOkUQK1Almp\nC9VuY4DIBACEPmzuN4zPOIANAizLsqyVWhF85XGXqYZAG3AduHVHxt3780lzJ10ZACxqJhB4hkqQ\nBwHbNgbs2ORz9HSy6rXddY+RqTWiA6Do5yk/qxnMQg6+MQYpBUKIpUDCZJru/irN+QilFGHBZWaq\nhco0nVbM/GwbN/SXggDXWU6a6CSCF4Yd7t772gpoOBKu27o6WLGst5MNAq5B0nUpHryFke8+QTKf\ngga34lLeWkBXu4lnTpDMNMnaHeSufZhKnaTYx7yuM6crS9dxXSgGFz/YZVmWZb3zaA1/+z0370a/\nIFPwoyMeUZLygZsM7XVSUwGi1FDJ15wQQvDJB7r4qy9NMTG9vOi0Zcjjur0lxn68OjgAqJUF77vd\n55++n5BkLJ29FTJPAcrSDNdz8TwHrTXzsx2kKylWQwQCKWBseJqN23pJ45R6T5Gx0zM4C6VFjTFo\nrRk/16BWH1z6ukfHPGolyfWbrsx82WMn23zju+OMTSRUKi733tnNXbfV3+5hWVcgGwRco9zP/THO\n/i9T+6e/QWYJxnVJ7vskycGPUTj5q6RZinPbu5FbdkJ7nnZ5E7EooFmusDDfgpu223KhlmVZ1kpP\nHxMrAoDzvTjsct+N6UX7Tl6QwcP+nQX+4N8N8u0fNphraAZ7Xd5/V4WxKc1jzyQka8y3B3oc7r7J\n58vfaZFpges5IMTC2QGFyhRSShxXkiX5n+MopVItkKUaR2iidsr8bAfPd6j3VJg8N0etr0inTKtO\nDAAAIABJREFUmdDTXyJJBI7rMDPVoqunBIDSkueGHQLXsHvwytopf/7lef70/zrF1MzyB/bUc3N8\nZmKIjz8w8DaOzLoS2SDgGtVREnX/p+jc/6kVj0sg/M3/EeeuTciwkC/nRC3E1AnktptxhEEZmGkI\nqqFg34Yr6xecZVmW9fY7ObF+XRGlBaMzUC0LGuvsBgTu6lSYcsnlEz/VteKxzYOSm/b4/PjFlbsB\n5YKgf7DIobOCLDMorZFS4LgSpTQqyxewlMo7CaexyvsIKI3K8nr8szN5A4K4E7FvfxdRJrnxjo0U\nix5Pfv8sWZri+yGlaoGxkVm6eko4Dvi+wCA4NeVecUHAPz40viIAAEgSw0MPT/DAfX0Evq0HYy2z\nQcA1yqxRk3hR+d47kXIajEF0GjhPPkZNO7Q2XU+c+iQpbO0ybNp1ZW51WpZlWW+vSuHiz3/vZY9f\nuCuh6BvaF2TzhC5Uw0vPh//lj1Xorbd5+XhKO9YY6RPWSpyYDjkxbdi8q5ckVnkX4cyQxBlZqmjO\ntZGOzCfuVZ/WfLKQ4qNwXYdKNWBqIqMxF7G9x2Eq7ebo8Zi4FDO0qczkaJNte0pobfBcF98XBL5Y\nyuXvrNME7O2itOHEcHvN50YnEp55YZ47b7VpQdYyGwRcowquIcpW/4Jyspja7CsIRyPGziBffArI\ne4PVRl5ky423A2sfwrIsy7IsgPfsV7w0LFmrCZYjNeWS4OvP+nz0loSGa4hSgyHfAaiF4jUdim1H\nhnp3gTtqBaY7LsPT51e+E/iBh+u5aGXwjcHz8hKfpUpIqeozPxuB0YSFAM8TeL4hDFxEV5H5uRjH\nc3jk8Q4f/0jE+Jjk2LFZbrixl9HRFlrnpUir9ZBqxUVrw2KLgqJ/ZZ2ZkwJ8b+2VfimhXLK9CKyV\nbBBwjeotGTrZhYGAoev0ExR+8pU131O+SLlly7Isy1pUDOG2nRlPHXNZGQgY9myG/m7DS6ccjIFq\nKKmubmJ/SR5/SfPo84ZWnrmDECl+YCiWV7a1X4wphBD4gUuWaYzx8DyPsKCYnmjg+S5+4DIx2mT/\n9VWSJCUoBhhtmJmLKYWa+bbA8xzOjURobRg9M0dQ8BjYkNcElTIvPyox7Oy7snbLhRAc2F1mbGJ6\n1XO7thU5sKf8NozKupLZIOAa5TmwpaaJZcDM2DRuZ5auxglqzZfICgV0p7Pi9abag7P75rdptJZl\nWdbV5t7rDDiGw2cMSQKVIuzYCLWFInNdFcPjhyDNoL8G+zYvT9YvxfiM5uFnDfF56UTGQBxlOK4k\nCJenMPnOwvLKvOtKEiFQShMWPMqVkKidIIC4lTI70yaO8hQijQYER4YljiPo7S+hlEClCsdzSBKF\n4yyvorsO3LwpZlvflVc441c+tZHxyZiXDreWPo2NGwL+zS9ssiVJrVVsEHANcx3om/gJAy8+ijDL\nv6ycoUGSsTFUM88dNH6Iufl94NqtAMuyLOvS1Spw1/XrP/+95wRSSsDw9DH4xLsNpeDSrv3MMVYE\nAOeL2glpklEsB/nq/Br9AoRkRZUgMKhMk6aK2ekOSWIQUqBihV/wmJh1yLIMISRCLlxPCDrNDtXq\nchAQuIYd/VfWgeBF1YrHH35uD4/9aIaTZzp0VV3uf28fQWAPBFur2SDgWqYysuPPrggAAKSUuBu3\nk6UOJizB3tthcOvbNEjLsizratVbUsxEq3PNkxRGpzTbtxU4eaqDEJLhSfjuM4aP3fnq100VtOOL\n9BnoZLSbMVI2qfUUqdRW5htlmcZoQ6YNaZJhBPiBn9f/1wY/9MhUilb5PbG7v8rYZEoWa0pVj+Z8\nvNR1WC80HFvUU1Y4rzKnPjKsOHw6w/cEdx5wqZbfukm4lIJ77+rm3rfsK1pXKxsEXMPk3Bg0ZtZ+\nThrMB34R3JV5lUrDmRkXY2BTd4ZrFw8sy7KsdQxUDPORRrGyadi5SRjsD5FSsGdniZcPN3Fdh+EJ\ngdJm3Ul0ksF3npWcmhC0I4egoImjDKUMznkT8SzLUJnCOJKZiRZSCooLWwy+LxBIzp2axfNcStWQ\nNFXUuko05zoUywGu5wJ5Tn+hFCKEYGayjXQEg5tKeI5hakyitUY6ghefneCm2wbRStNbXGd7AtDa\n8Hf/HPP8UcVCg2K+/3zKh+/yufM6u9tuXVlsEHANM34BHBfU6m1L43hEjz+CGj2LqNYo3PsAp5oV\nXh7xacT5qs5LI4p9gwk7+q7MbU/Lsizr7eU6sHdAM9U0vDzikGTQ6DgIx8NfmLMXQkF33WO+qclU\n3p5mvSDg609Kjo4sP+l5Do4jmZ/tkGQG33dJkozWXAdjyFN9XMncVIuu7gK+LwhDB61ciiWX5nxC\nz0CZYjmf6Lu+pL+7myzTaLW809CcbRO1E7TWTI636O0tUKqXyOKMeleRsTOzNFu9tNuap2KHXYNq\nRVCy6JGfpDxzeOXue7MNDz2ecGC7Q6V4aStrxhhmGxrfE5QKdjXOenPYIOAaZsrdiL6NmNFTq57r\nnBuj9chXl//+2Hc4fte/p9F/09Jjzdjh2TMBtYKmp2zLhlqWZVmrSQHTTYeTEwHnVwryPaiW8sZc\n5bLDfFPTVwdvnZnH6AycHF89sZZSUCh6zE63yZKM1ny0ouOwzjRJnFGrLa+0O66g3l2kOZ8wM9mi\nUAooVQqUqwW0NkSLzQuModOMSKJ04a+GViOlUvHRmWJgsAxCIh1Jq5XfBycbgj/9Yszd10nuvH7l\nbvqRM2sfFm604YkXM37qdn/N58/34xc6fPuHTYZHUzxXsHurz6ceqNHXbads1uVlw8trnHfrB8mq\n/UtVAgyCqKOZ/sGTK184OcLWJ/56VS/3VElOTNktTMuyLGtt7Vjw7LDPhT0DkhQ6cf7nNNUUA8Pt\ne9bP8z87JcjU2hVspJP3JIijZClX/3xaaZRauVil0vzvaZzRaSUImQckSZQSRylgSOJ0KQBgocCQ\nygzDx6eYHZ/l1NFxtDE4riRL811xIQRT8/CVhyOeP7qyTGh2kY3zNHv1vgKvnIj4L1+b5dhwSpJC\nq2N45pWY//OLMyh1ZfUlsK5+Ngi4xslaD53bf5boup8i3nEH7Rs/xPgTL6DT1b+pqhOHqI2/tOrx\ndI2mY5ZlWZYFcHTcJUrXnk4kaR4A1PyYn323YdcGmG9pHn9J88wxTXbexLa/ZpBi7Ynu4oFeL1h/\nUWp8PKLTye9tnVbC2EgDACFFHiQsBAVZpug0OgwMVdmwuY6zcPhNa43jSYzRNGbzxgRRO2FqdJae\n/jIjZ/MzdmmiaDcTohSeeHFlELChd+3PwXPhwPZXX8l/9Mk2zc7qz+DE2ZQfPttZ4x2W9frZvaV3\nAiHJNuwBwKgMs85ShUTjJK1Vj1dCmwpkWZZlrU1d5BZhtGFHd5vrb5IYY/jWk4ZnjhraCzsE33/B\n8MHbBHs2STb3wVC34czUyoUnYwxRJ59sCyEQC7n4xpil1gBh0UdrmJ9P0SrjxJFptDY4jszLgwqQ\njkBliuZMG6Nh5Ow0W7b3M7CpzpkTkxhtCCsF4s7K3YY4VgyUC8xMt5gcmwOcpU3zuebKb/6+2zxO\nnNOcm1z5+M27HbYOvnrH3pnG+h/m2NSV1ZzMuvrZnYB3GOG4uJu3rflcp76JmaGVDcOqoWJ3//qV\nECzLsqx3ts1dGa5cewV/Z3/K9Zvz554+YvjhS8sBAMDELHzjR4Y4zV/zsTs0USdFL0QWaapozkd0\nWinSyVN5HEcu/SelwHEl9d68G26WGUaHZygHKdfvDSmV8rXOIPRQmWZ6bJ5OOwEMrbl8IOVagUIx\nz9VXSUq9u7AwujzY8H0HIyAIQ8bOzZOmy3n/4QX192tlya89GHDvTS67Nkn2b5N84l6Pn//ApTVH\nqF2klGhv3a7bWpfXq/5EdTodfud3foepqSniOOY3fuM3uOeee/id3/kdTp06RalU4gtf+AK1Wu2t\nGK91GRQ++DNk505jZqaWHwxCyvd9iK39MNVUGKC7pLluKOYiu6+WZVmAvVe8k/VWDdv7Uo6MeZx/\nLqC7pLhx8/Ii0qFhc+GxMwBmGvDUIcO7rxeUQrhxa8L3n9cIIFtI4ZFOvgPQaXby3H+TnxPwA4+w\n6ON6y6vs77mjxM6NRQBGxjO+9I0W7WbE3FTzvK8q0NnCtaWgf6jOiVdG6bQSugYqy2cAhCBNNGmc\nNx0TxiDIdxi01mRuie++6PPe/fFSxaNaSfLgvZfYEe0Cd99S5IWjMZ1o5Qe1ZYPL3bcUX9c1Xy9j\nDN94ZJbHn20yN6/oqbvc/a4KHzho/z98rXD+4A/+4A8u9oJ//ud/plAo8Ed/9Efcfffd/PZv/zau\n6xJFEX/+539OkiTMzs6yY8eOi36hdvvqXU0ulYKrdvxrjd3p6cfbewNogyiVcbftpvixX6R0171s\n7FLsHkjZPZCyqSsjeBsXHq61z/1qYcf+9ihdahvVK9TlulfA1Xu/uNp//t7I2DfWFQU/z+kv+Yat\nvSl37ogpnFcM58eHDHOrM04BGOoV7NiQBxB7t7hMzSpGZwXSkTieA8YwMzFPlqqlFCCjDSrLMNpQ\n68l3ArTKuG5rSnGhrGalJNFKcezkedsPAoQjqHYVqHaVAHA9h/mZNirTlKsFjNHEnRQ/8EEIEIIs\nzVCZxgtdiuWA7t4CXb0lZtuSTMOm7teXOnv+Z9/f41IrS6ZnFfMtTeDD/u0Bv/RgjVrlrb0hf/mh\nab700DTTs4p2pJmazXj+UJswkOzeFq4a+9Xmah/75fCqP1E//dM/vfTnkZERBgYGePjhh/nsZz8L\nwKc+9anLMhDrreVu2kb5X/362z0My7KuEfZe8c5jDIw0JPORJNOCwNPctDWhu7jYaRdePiM4N+0g\npaFU0CzN4M8jBWzpX/nYp38q5JNKc+yMplgQ/ON3WkycXaPnjYE0yei0Y4LAY3q8xWNPpHziw8ur\n1QN93lLlHykFSEEap2zctmnpNfk8Pw9CiiWfqXGNXwiQQiIExFFKlmm80KNUKVCrh1QqyxHOuWkH\ndl6enP27bylx8KYiY1MZhUBSr776WYLLLUk0P3i6gb7gnytT8OiP5/nQe2oruihbV6dLDis//elP\nMzo6yl/+5V/yW7/1W/zLv/wLf/zHf0xvby+///u/T71efzPHaVmWZV0F7L3inWN41mG6szxBzRKH\nTiKBjFpo+MbTLifG89KeAFI41Osps7MrJ/O7NsKujasnlK4j2bs1X9F31jlzAGA0TI/PIxB0mjEk\n+SHkxUl9b13gCIORDjrTSAy7b9i49DzkVYAWewfMTreX0pAAhJALKUgGAfiBRxCsnJgn65Q2fb2k\nFGzoe/tycc9NJIxNrV1EZHQyZb6pqFftGYWrnTBrFdxdx8svv8znPvc5kiThs5/9LB/5yEf4i7/4\nCxqNBp///OffzHFalmVZVwl7r7j2tRPNE4dTsjV6Y3WVBVFH8q2nVqfHuA70lVMmZxS+C7s3u3zs\nngKee/FJ9D98fYL/56uTaz4npSRYSI9I2gmVsuQ3f7VnaZK/uS+kvx4wMZ0yPid46OmVaUlpmjFy\naob56fzBvExo/pwjJa6/MNkVgoFNNbQylMsutXph6RrbB+AX3nPt1FqZmUv5t59/mUZr9T9wf4/H\nX/1vBwj8a+f7fad61TDuhRdeoKenhw0bNrB//36UUkgpuf322wG45557+LM/+7NX/UITE403Ptq3\nSV9f5aodvx3728OO/e1xtY/9ana57hVw9d4vrvafv9cy9qmWIFNrr1Q324rjwwpYncaSKdjQLfjk\n3YuTfsXsTHPV6y501w0+f//fBdkaDbekI3EcJ5/0h7BhYDm1p+SDzBKmplIkMFiFB2+D//rdjOmG\nIEsV0xMNola+C2CMIUsVjpuPffF/EQIpoNVISKKMxmy+Q1CtBYSuZmdfzMTE6zsTcKX+3OzfGfKj\n51Yf4jiwK2R+IYq6Usd+Ka72sV8OrxrGPfnkk/z1X/81AJOTk7TbbT7+8Y/z6KOPAvDiiy+yffv2\nyzIYy7Is6+pk7xXvLHnRiLUTCVy53jO519P3thhKbj5QXmrsBXnKTFAMCIrB0qTf9SUfPFikGsBA\nWdBbkivSfgC6K/DT79IMHxvn3MmppQAAwPEcjDZ5nwABSIFcCDCUMiRRniKjNTTn2mztSXnv/pgt\nPddeP51f/fk+bjlQxF+I9cJAcOdNJX75Z/re3oFZl82r7gR8+tOf5nd/93f5zGc+QxRF/N7v/R4H\nDx7k85//PF/60pcoFov8x//4H9+KsVqWZVlXKHuveGcpB4ayb2gmq9N4qqFmQx1OTazeCXClYefA\na58wR4lhuuVS7a6SxinaaIIgQEiBUnqpr0DoS/ZsDimGF08vevGEIiiGZEmG1hqBwPVcpCvzqkNK\nUfALOE7+PRhtlncFFrSaKUePt2jPS2o3C8rFays9plJy+Q//dojjwxEnh2P27AjZNHh1VzGzVnrV\nICAMQ/7kT/5k1eNf+MIX3pQBWZZlWVcfe69459lczxiedRcCAYEjDfVQM1jR9JXg7LRieGp54iww\nHNis2ND92vcCDp3KmG3maT5+6K94TkqBXkhdH+pzKFzCPPXcRJ6uduG1IE8BUrFCK72U1iakWNGL\nACDTguEJGJ7QnJmAX/2wIPCvvYo5OzaH7Ngcvt3DsN4E9mi3ZVmWZVmvWeDCrt6MViyIs3x3YPEM\nrevAx96V8fxpzeiMxHFgW79i1+ByADAypXnpdJ51c9NO6Kmuv5JeKealOtcqZbL4WCGA99zir0r/\nWXPsF5msG2MICy5Ga7Ioo6fHJ1KrIwvHWb7G2Un4/oua+25xSFJIFRSDvPSoZV2pbBBgWZZlWdbr\nVgoMa/Uuchy4ebuG7SvTf4wxfPNJw9NHIV2oQvmjQ3DwgOaGnS5nZj0SJSh6mu29KQXPsH1IsnVQ\ncHJkdRRQKQi2DrocvMFn//ZLK6t5w06Pp19JOb/DMYDWmjRNkY7P3e/byEvPTWKyiB1DIafHzFI1\nJMeVq9KDzkwYvvx9GJ7Iv69KEQqeoRBAXw3u2geFYO2oYGrecHIMBup5B+afHEoIfcGNe3wcW4/f\nepPYIMCyLMuyrLfMS6cMPzq0clU/TuGxF2Ay9giLyyk6402XWzZ1qBXgwfcEfPE7MSNT+RulgF2b\nJP/+l3poNtqvaQy37vP5m681MDiIhUm2VpokThBCYIyh2UzYd30vjz18mk9+UPDgewK+/ZTi8BmD\nlKt3LY6eNSBSlMp7FLQ6Dq4r0drwyjAcOQu/+D5Dpbg8qVfK8J8favPcsfwzEBhUmjI52kJrw1Cf\nw4PvLXLjHpuLb11+NgiwLMuyLOtNFacJaZqiMaSpxHd94nTlRDpTMDal2FpcfqyVOBydDLhtc8SW\nAYf/6VMFnnw5Y66l2dzvsH+bQyF0aL6OSo9JojBG5XUSDWRZhkDgOA5aG2bnMgY3VBjcUMaRgk39\nDh+6UzA8qYiT1dcTQiAdiecJEBC1U4zJAwFjYHQGHn0B7r/N8OyRjCgxTLUcnjm+3JTLIJCeT7Wn\nzOxEg3MTin/4VottG12qpbe+c7B1bbNBgGVZlmVZb5p21CFK4qW/bxmAT7w745+eKNCKVk5s1+pf\nOtfOJ9FCgOsI7rr+4ik/xhhmGuB7UC6sn0ojBGhtYDHFR543FgN9ffnqe1+vx3W7fBodKAaSD9xi\n+JfnNM3O8sulzAMAyK/pBw6Fkk/UTnGc5WDn6DnN84ciRqfyFCnHAS/wKFyQT+WHHq4ryTLNzLzm\n0acjPvKe0kW/b8t6rWwQYFmWZVnWm0IptSIAWNRf19xzfcqhUZ+5ec3MfD75LxfdpQn/kteQEv/M\nUcUPXlCMTIHrwrYBwYfvcuivr07fuW6Hy3NH0rUvJKBeD4mjjN0bDH//iODsZJ7CtKHb4SMH4b/9\nwBAnICSr0oOMNriuREpQmUIrgxe4zM4bZqaXz0goBaqdIh1JEC4HN1IKitUQlRlajQ7PHlV85D2X\n/jlY1qWwQYBlWZZlWZeV0pAo0Nk6k2xgy4DGq/pkyjA1ozh8UtPbpdEXBAFdRXVJVXaOndX8tx8o\nooWYQyVwaNhw+HTELdszPnFfBfe8ij437fV57ki6dtUhA4dfmeH9txd56pWQmeby+4YnYablUClq\nsvVaHgiBWPhPpZo4zlDKYMzab0jjbEUQoDJFay6iq7+CENBMJV99LObj99izAdblc211trAsy7Is\n621jDIzMKmaHh2mePc3ZGYeZuLRmac9FriMY6HW55fp8gutItfRcNczYN7BGAv4anjyklwKAFWMS\nLo88nfGfvjK79NijP2nzxe8qwlJIUAwJigFSnNeNWEjajYSona4IABY1O4JwnUo/QuQr+cYY0jTD\nL7gIAVmaEbXXDorOT4MyxtBpxmSpImonFMsh0nF4+qhcM13Ksl4vuxNgWZZlWdZl0RodZmPjBJ7U\nSJUyFB9j1N/GXHkT9aC14rWRWtmoq+AZZo1L3YuolxwqAWyspRyf8GhEksDT7B7IKPprT4Qb7fUn\nyNKVPHuow+mRBCnhSw+nSEcQBA5xrNAa/KJP0l4OOFLtMDG//vfaVRF0lfLKP3rhSwsBnucghCCJ\nM+JOhuM6BKFL1MkQEtbcDDCQpXmDsqid0JrLDxzoTOP6Dp12TKFe4PvPp9xz4+oGZ2+X6bmM06MZ\nG3odPNeWMr3a2CDAsizLsqw3THcaFNUkcfcgkeODyvCSFkPzR+m4FbTvIYXBGOgon9lk5UFXIfMG\nXL5IKQfQVzY8/EqB2c7ygd1Tkx7v2h4zVFcXfvmF0ptrBwI608QJvHI84eFnFEMbq5RrIb7vkCaK\nxnzEyJl5nCDvFiykoFBwqRXW/36rRfjQbZKXTin+63c0XuDgeQ4YiDoJzbkIyCf33kK34b66w/j0\nyrEXAujECp0ppCPwfAfXk2SpxnElAtDKEEcZT76iuOfGfLfg0Sdb/OTlNp1YM9Tv8aF7qgz0XFqf\nhDdqrqn50ncjjp1t0Imhry54136P+++06UpXExsEWJZlWZb1xjVHyYr15YR+xyUt1ADo6oySOLvw\npGaiJQiclE3FSZSRtLKQubSE0gK04fh4iBGa0VlvRQAA0E4lz5/x2VDrrDon8K59ksNnNJ0LUoLS\nJCNq5w/Wqw613hLdfcsBiOc7dPfmKUsjZ+bRjqarr8rWQcHd1wuOjhimL0gJKhcM79qd//nAVoe0\n06HdBMeRaGMwejkY0UqTCUG9DL/+MwHf+XHK8bOKTBk29Tt0dRd47iRLnY7DIoRFn5nxBpXuIkpp\njMkbmXkLLZn//hsz/PNjjaUdiFeOx7x0NOKzv9THUP+bu1NgjOG/fDPiyPByMDMxa/jmEwmlguDu\nK2inwro4eybAsizLsqw3xBiDkpK1TvCmfglfxFSLIa4jqfkRBVfhOZrQzegOmtT9JkmWp/xMzgoC\nzzDZXHuKMtOWTDRWP7dzSPLgux0qocYYg1aauJMwP5U3EdiyweWmfSGV6tqr1ZVamJf6lIJi6HDw\ngKQYwMfugm0DBs8xuNKwudfw0Tugp5q/TxtDbx7r5BN2vcZuhFZs6IYkMXzyvpDP/VKJ/+VXyvzc\nfQVOToilAGCR57v0DFaRUtJs5DsKvb0BN++STMykPPZUiwu/zOhkxtcfuUj+0mVyZFhx7OzqnRit\n4SeH1j8Ibl157E6AZVmWZVlvkMGIddYVHReCACkMcRqvihOEgKITkWZ1hCOplzO29WjOTK2XYy5W\nTYAX3bTL4YYdgv/8tSbPvtKh2dYIAds3enzmI1XiTOL5azfd8jyJEAaVaYrVkB8dM8RZQrkiOHij\nwACehA0V8FxoRZqvP644MaJpJh6Op9CZXnUIWmcZrU7G07Pw8rGIO28I+Ln7iggheHkY1mt2LKRk\nbrqNzgz1uk+1EnDXgYhvPdak1V67ytCpkUs7RP1GnJvMz1Cs5WLnMqwrjw0CLMuyLMt6gwRIF8zq\nFWKUQoVl3NmT+FoSOyWMXDkRD11F4Gak2mGw26GnnNJVVnRmVwcWtVDRX12vNmdes/9XPl5l+r1F\nnj0c0VV1uHFPvsqfKUPR13TS1YFAEiuSWOG4DjOTDYSo8LUfZHQiQ73m8MF3u3iuYLRpGKoa/u7b\nGcfPLU96HcdBCkmaZktHEwSaTnu5I3AnhkeeitnU73LXDQHhRTJntNIIYejtC9myrcqu/gRHCsJg\n/SQO/zUezu1Eim8/Ok27o7npujL7dr56Q7Jtgw6eA+ka/9T1ik0wuZrYfy3LsizLst4QIQR4hTVL\ngWqRpwmJLKKg25SzmVUlclIlSZXEdwzCyS9y3VBCOVg50/Rdzb4NKfIS5rrddZf331Hm5n0F5MIb\nXAd2DyrSVJEkaqnkpjGG2ZnlJflWIwYEN+4PuG6H4eTJOb74UEwrljRiyStn9YoAYOlzkIJKyWFT\nn2RjL0SdbNVrjIHnj+Yr9vs3w2D32lOxes3nhpv62LajRikw7N2QX+vuW0sM9Ky9hrtvR/jqH8yC\nx5+e43/+wyP8zZdG+eJ/H+cP//cT/OlfnUatt82yYNuQy+4tq4Mo34M79r81B5Oty8MGAZZlWZZl\nvWFuoRvteGjyhXADKCRaekizPBl2TUbWTomy5SlIIwmQUuI7GrnQvKunbHj/3oi9Awkb6yk7+hLe\nu6fD9r7VE+tLdXIMDp/RtNsZnY6i0UhpNhLGRxqMnlnOpx/aWML34PhZuH6Px9YtJaYnmvzjt5rM\ndQTz66TwAAz2OvzWLxbZOrB+pBKn+UTbkfDg3T710vLEWwD1imBog0foGwaqGXfuiKkV8tf4nuTn\nH6jTU1+eiEsJN+8r8Imfql/S59CJFH/75RHGp5Zz+JPU8OiP5vjqQxOv+v5feiDk9v0u3VVJ4MHm\nAckn7g24zQYBVxWbDmRZlmVZ1hsmpcQv95O0Z9FqMTfd4OoUT6/MVa87czzd3MRgOIMWuV9FAAAg\nAElEQVQRLmOdKkU/QwpDwY0QC+cLSqHhlq2XJ8+9HcO3fuIw31menBsDSaaZme4sPSYkbNtRJ4o1\nc/Pw1GHNrQd8Tp9u02om/OTFhH3bXGDtYKQU5tffudnje0/Fa+6ODPUuT7/2b3X5tQ/B00cNUQJD\n3bBnkyHTHbSBYI2Z2ruuL7FvR8j3nmjQjjW7t4bcvK+w6oDxeh7+wQxjk2sf4n32lQY/+9P9F31/\nGEg+86ECtXqZM+fmKRUE8hK/tnXlsEGAZVmWZVmXheN4hOVetErQrSmcpIFco3a/IzQbghlGoy6+\n9pWTVLsibru1RrEcsK06B3Rd9rE9c1ysCACWxywpV0PiToqUgltv788bXxmJrPtMTCh2bEiod5eY\nnmpx+HB+XqCnBlNzK6/lOXDTzjyAuWm3x3U7XV44ujJYGOqTfOCOlRWKAg8O7l99rYspFx0++v5L\nW/m/UDtaI6F/QRxf+uFe3xNUijap5GplgwDLsizLsi4bIQSOG+CUujHJ3Krn8ymmoOjESCH4vXue\nZ24q5m8e3scDH6jS3ffmNJzqXGRDoVjy6NlXZ8vWKo6TBwqOI3AMVKo+rU6KH7ioJCNqZhw/Kbjj\nQJlSqDgzbtAGuitwx36HG3fms3chBL/28Qrf/GGHo6czUmXYPOBy/8GQWvlVZvhvsluvr/KPD00Q\nrTHh37rp0s8VWFc3GwRYlmVZlnX5uSFaBkgds7j+np8VEBghSfFxpeZMz63s9Z/iP5Sf568f282e\nzUNvynC6K+s/19MdMHBBk63F7BbXEcTKQZsEJBhtyGJDOxH8+oMep8Y07Qh2bZIrqvNkytCK4EMH\nC3z0PVdWqsyOLQXe/a463/3+zIrHhwZ8Hvxg79s0KuutZoMAy7Isy7LeFKLUT9Y4h8N51YCEJKJA\nhyKuyJhx+ki9IslAPwfHR1CNEK9y8Zz01+OGrYYXT2tGZ1amr3gu1Ourp0OLufxKGZTwiTsdPM8j\nIUVIQZRKhMhLZp5PG8O3fpTxwgnNXBMqJTiwVfLhu1ycSylr9Bb59X+9kc0bAp55sUknVmwZCvnY\nB3vZOGh3At4pbBBgWZZlWdabQoZlmnE/TjqPT4JGElNgTnaTKUmvGmUmK1FKZhBSsHV7geMvNbmu\nL8J4l3cy6jrw4B2aR1+Cs1MCrWGgbghLHtkFjc6MMSgt0NrgOxqjIUsVnu9RrBYIQp/ZpqbRgkpp\n5Xu/9eOMR55dDnpmGvD9FzTaZDx496tXzzFa5SVUpXvJB31fDykFH/tgHx/7YN+b9jWsK5sNAizL\nsizLetNUqjWmRv9/9u47yu7jOvD8t+oXX36dAxqNRESCQWDOSSSVLVmWzKFlW9JKa61sr9bjMJqR\nZ3zOeNZx7fXujHd0pGN7pdVYtmVLNk1JVCIpUswZRCJy6Ebn8PIvVu0fD0Cj2Q3mBLE+50gk33v9\ne/V+jYNXt+rWvU1m3F4QEqUFKoVyNM6qYCeptRGATDBHMz9MPCIQc8fQvetf0vWjWPPIzpg40bxj\ng00uc+aDqsUsvPfidldfDUgBzTDi2THJRNVCCEEcaxotRaulyGagq2xz9HgAop0i5Pku9fkmGsF3\nHnO54nyHwVKMbbVTgHYeWr6R2a7Dilsv0Xju8hP7iXnFvuOaIBJkbMU5ndP0dHpIv/iS7oNhvFwm\nCDAMwzAM43UjhKDQHGegsZeWVQQ0ubSKpRMiXPrTEQQg0EgpUKvWkOgprLCOdrPtmp1n8ORzMT98\nbIqJ2Xa1mx89HnH1+Q43XbL84eL5eso//bDJ5LxASMFQr8VNlzis6wx49oALQhDHpxr+0gqg2kiZ\nmk7xMx6V2QZCClKlaTVCdh+SFHtLHJx2WNMdUfZi5hvLj7XSgLmapr9raRBweEry6EGfMF2Ylo02\nilwajTA80EB6L97J1zBeLhMEGIZhGIbxurI7B2G6QjGZWfR4XRTorh8CKVEIml4ntvbJTO1FVo+g\n3BxpYYCkc/WSa85WU+64P6R2WuOuagN+8FjMQLdky5rFqTf//MM57rynShS1V+pt12Z0MsfIpMcF\nW/JEydLJeZzA6FiM1oo4StFodApoiIKI6oniR63EYte4j1AWQ0MOk5Mhzebi0qCFbLsJWHucigd2\naqbmoZCrUw0Ewrc4PfsnTB12z/awonPcBAHG68IEAYZhGIZhvK7sfBl7tEXLLSCFJsXCSVt0tQ6e\nqhzU8juoWWXWTt2LwxxKlLGEQM4eQFs2aWlo0TUfejZZFACcFCfw9N50URDwwwcqfPN784tel0QJ\nteka48LCPxzhFs5UmlQQRynNeoglLZI4JYmT9uHgZrTodc3YIZt3WJnxmJhoMDcTnMrr3zQs8V3B\nXE3x9bs1YzMKrTWWpdu7JQXo7c8ueue5IEMrguVOEjRaKfc8FjBfU3QUJNdf4pPLvLmlR42ziwkC\nDMMwDMN4XVnVcey4ST5eOmvXQN3vYaxjC5nKOKoVMdmzhu5oHLwMApCVUeJsF9LJnPq5MDpzU6vg\nec89+FR92depVFGv1olCD/cMJUSDIKI628JxLXr6C8zPNGg1WwgEWi1+n5MBjWUJenqyNGoxrpVy\n7mqL91/VnnL96MmUw6MxadLekRASHKf9nJ8JKJYWDkRLqbGspelQB0civnJHjcnZhfMHj+4I+MQH\ni6wefPHDx4YBYNq8GYZhGIbxulJeHs3yB2JDO89o9ztw7RRR7qTRsZIn3GupOgtdg2USkNQniOsT\naN2e+K7oPfOqd2/H4unNyHhyhldC1AzoLGqy7tIDvWEQMzXWwHEtyl05LLv9Tz/TnmjnC4t7CySn\nNeK1bUG5M0PR13zoWgf7RBOyJ/dEpwIAaBcCisKEKEyYnY3QeiGw6M40yWSXpgLdcW9zUQAAMDmr\nuOOeMxxIMIxlmCDAMAzDMIzXlcp3k+aXb0KVZgt0yVlyIsC3Qxq969l7RLHDvWjh52V7pVzHLZJm\nu8HVxZtt1g4uncb0dQquecfi1fD0zJsG2K7L5Vsk150b019OsYTGsTSWSKnXIjq6s/StKJ2a+Fu2\nRaGcQwAXvKPr1HXiRBM8ryuxZQmm6+0xHptI+fr3WwTB8tWD4jglSRStVjuSKHkB21YGS84DzNdS\nDo3Gy17jwGhMpb789Q3j+Uw6kGEYhmEYr7tg5Tb8o09gNaYRgBIWYbZMs3Mh199Ck7VCtCoSygL3\n1S/k6tzTRF7+1Gt00mq/Vgo++X6fe5+CXQcDlNKsPFHtp/S82v25rMN8FC4Zk+M6CFswMZ1w0zrN\n6p6YegCWhO8+Iak3l+9V4NiCbZf2USxniBNNkkIzWPyaJNEkiQYEf/2vTXYdStEIhBAopUCDPC3V\nR6t2P4LeQkI502TkWJ07x2D9cMBlWz3kiUZjSrX/txydglIvEPEYxmlMEGAYhmEYxutO+wVa669D\n1iaJ5kZRmTzKzSx5XSuAtYMpiRZscI5wf3wZF/jTp12ofaBWCEHGk/zS+wtMTb1wYsOFW3I88qxF\n2ApQiUJIge06ZPIZKtPz3PdEk+svzmJZgsKJIfV3wOHJpdeypebGKwocnfOo1DmVvnN6Y6+TK/pR\nmCJ0cqJ3wMLzUkqUUiilkLI9diEllgWt2Qo/errFybn8w9sjtu+L+NSHClhS0FGUDA/YHBxZmuK0\natCmXDBJHsZLY/6kGIZhGIbxxhACVexD9axZNgDQGg5Pe6zMzbHKHqVXzjAedi6+hOW+7E66m1cJ\nhIBiV4lST5lSd5lcKUfUCoiDkLHplMPHF6fYXLpeMdS9eMldoNm6SlHKgVLtiX6zqWg0FM1mO50n\nTRXVakK9HtNsRIsPCpymHQgsrNrbtqSnBA9vbyEsC8d1cFwHy7F4Zm/EfU8GJ26h4F1XZSnlF9+D\nUr79+PPvTa2Zcsd9Df76X2r8/ffrjE6e+XyE8fZidgIMwzAMw3hD2V6BZquJbS2eZI9WfIa7YtAp\nnWoGKaHLqzNSzTNUrAMS6b38Drqb1mVIGxPUWh6O5wKaqBUSNAIsx8FxBLns4smzY8PPXpHy1AHN\n+Hw7RWhtn2bTkGa+qXl0n0162vxeKQgCTZomzM60cC3Nb3xY8udff4GBaZBSYLk2jmvj0QLLxpIL\na7QW7U7GO/bH3HBxO3A6b73H537B4sdPtJivKcoFyTXv8Nl3XPDlO2OiWNPXKVnXr7jjvgaTMwv3\n+YndIR95Z55Lzj1TSVTj7cIEAYZhGIZhvKGkZXPX9k4Gy00GOiJSJTg65XL/cwWuWFelpfM8WlvD\nxzY10U3NXbv7uf3yaYo5F+lkX/wNnqej5HDx+XnufbhKUG+delwIges5rBty6O9aWlrTseDSDUsT\n8KerkjRt5/s/37o+za/cbOHYAq01mYxFjCCJ04VWxLTTiCxb4uc8hBB0l6BSX0gPWnS/pGSutvix\ngR6b2961UNf0H+6JeXr/wlhHpxVP7gUhchQ7FM1aQJKkNFrwvYeabNvkYlkvb0fF+OliggDDMAzD\nMN5Qx6cVu0dcth9xlzz3zLE80s8RtBKebKxl/2yRJE3YPVHk8g2vfNL6P/18P1IKfvJ4jShSSFvi\neC5r1+T46M1naBJwBrN1wXIBAECYSBxbcHRS84MnIbV9cgVBmiqiICYKErTWpGlKrtBO33EsuHSj\nYPtzElj+1K/rnPmzHxhJeWZf2j4wLNrBzcm0oDRN8X0Xy5bMz9TRSjM2rXh2f8SFG1/f3YD9RwJ+\n8GCNsamYrC+5YFOGW68unjrk/EolqaYZaHIZgfUqr/V2ZoIAwzAMwzDeUJPzEC+fKk+1aeGpmCRO\n2TPVgXQlaSo4VslyiWphv8KmuLYl+PRt/Xz853p56JkWs5WUzrLNlRdkTtXwf6ly3pkr8PiuJk40\ndz4C01U4GSxYlsTPumiVQpIyMODgZSy6yzabhlK2rpHMztvsOhQte93hgeU/+HNHUr7+g4jkRKq/\n1rq9y+BYSCnRCipzDUodOTI5j2atfbbgXx5IOTiZ8L7LrRcMMF6pvYcD/vvXp5mrLvyi9xwKmZpN\n+KUPdr3AT55ZqjT/+pOQXYdSag1NR0Fw4Qabmy99+edEDBMEGIZhGIbxBlvdD77Lkrr6AEorWo0U\ny5ZMtfJsKNZoZLKgFTKsQLb8qt7bsSXXXrS0AdfLsXkoZeeIYra+OHXHlpoNAyn37LDQjk1fnyRJ\nNK1WTLOZIoSgUM5R8OFT71ZkPUlPT56pqXauzzUXujy+K2a2ujjIyPhw0WaP5yZcEgWdmZT+UorS\nmu88FNM67T6e3AWIghgv47YPHwtNtdLAddopT9KSRMrmyb2aZpCyflBxfEpRyMLVF3j43qufUH/v\nJ7VFAcBJD29v8K5ri/R2vvzOxt+6N+ShHQsHmyfmNN9/JAYBt1xqzji8XCYIMAzDMAzjDVXOS7oK\nMaMziyfRWmtUolGpIpN3sW2L0XmfgWLAqq4muUOP0NpyC7wOq77zTcFIxSVMIONoVpZjRqcFO0cs\nqk1BR15z6wUxGa99SPiGcyMe3OswMS9RWlDKKs4dSohSyUjFJZNpj9FxwPMspAyp1xO0hkQ4/NV3\nA379g4vHUMhKPvpOn+88GHJsXKGBgS7JhVt8jjaKBJX2/TqIpnc+wYvrjM8uvyshpSAMIlDt+4kQ\nKFshBPhZ79TK+Z6jiid2hugTlYoefjbmozf7bBh++ZP00x2fXH5Ho9nSPLWrxa1Xv7zrNwPNzkNL\nKxtp4AePBNy4zcZ+pdtEb1MmCDAMwzAM4w13+WbJ39+TIKVsN9DS6lQAICR0drYr4bRii55CwJzq\nwArryNoUqtj7mo5lrGqxe8IjTheCkmNzNscnU8Ko/djYvOav75a8+8KItQOa3pLmZy6OmK4KghgG\nOzVSwB1P+jz/vICUglzOoV5PkFIgpaDRsqk2Unp6Fo9l02qHjatsjoylxAkMD1rcvz9HKz49YBJM\n1hwy+MDy3YNBEIcxWmmk1X7PJFbky7nnTZYFliVJVHvVfrqi+df7Q37jdhv5KoKtjHfmKvTl4suf\nrI/PplQbyz+XKsl/+K8z/MlvvLZ/Ln7amT4BhmEYhmG84c5bK+kva+IwIQpikjBtr1gDhaJ/aqVa\na6i2HLSSkMbIoPqi1w4jzQNPB9z/VEArfOEOulrD4Rl3UQAAgJB0lOxTk3bLkli25K7tCyvYQkBP\nSbOyW2NJaEWC+ebyUyvHsXBdgedZJy4vODK5/CFgIQSrB23WD9uMVVxa8fKTZsdzWFRy6Hkf7OT9\nlJbEsix0mi5ZLdenve6kYxOKfUfPcGjjJTp3/dI+EADDAw6XbH35FZ56ypLM0nPkQPt+BanL/iNL\nu0IbZ2aCAMMwDMMw3nBSCj5wtcNA1+LV5mzeoaOrPYFUSqM1RMrBV/X2hHXfM4j9T4FefgL94DMB\nf/Q3Vf7+By2+8cMWf/g3Fe59IjjjOGqhoBouPx1yHTi9+Ey7qo1k18gZJvq2xrXOFHRo+np9Bgc8\nclmJSjRDPS8+DUuW/5gAKC1OdVBe9LjSpEqdCqTyBR/LFkTh0l2DNFGLmpad1Apf4I1fgg/eVOLy\nC7J4p2X9DPU5fOwDna+oOlAhK8kvH1cAYNmS7z9YfwUjffsy6UCGYRiGYbwpVg9Y/PrPSZ7el7J3\nVBMol0zORoiUnKfIeorD4xJle1zX/BbJfAV59BB696NEj97D5HwnXLUNff55CCE4PpVwx49bNE9b\nEJ6vab59f4uVfRbrhpbmoVuinbxz5qn7YkLAVHX5Up6OBQMdKQcnl07uPVdQzDu0QigUBJW5gO2H\nLc5Z/cI7FYOlhP1TaulOBVDyE1xb0wjS0/oL6IUeBkJTKOdAwtxkleFBh3wRZqrguSC1Yqq6NHe/\nuyzYsubVnQmwLMFnbuvh0EjIrv0BpYLk8gvzL7sS0+muucDim/edeYfi9ahy9NPMBAGGYRiGYbxp\nwkhx9EiF6cmIAI+BNX24noOyNB2liGI2oae2g2phNbMD11DMPErHcz/Ga06gH3iIx/74ixQu38a6\nv/wvPPSstygAOPUeMTy2M1o2CMi6mnImZa61dEoURu10oec7b2jpAdWTNgwkzDZswkQQp5Ak7R2F\nYh5sqx0oxAg6ulyePqzY87cR/Z0ea3oSpuY1x6YFcSLoLikuPkexslsz3BFzcNpFn3bWIO+mbOiP\nKeQs6q2U9HkpPbYj6eguE4UJk8fmUEnKLZcX2bbFZmJOU8zCzLzgq98VzJ1Wjchx4OoL3NdsQr1m\nyGPN0GtTueeKC3z+8d4qUi6kNJ3qhxCnvO/6l9fv4e3OBAGGYRiGYbwpDo8E/LevjDE6sbAandlV\n5YLLV9HZU6AR+JzTH9DZl2HWHkZJh/lN1+JWxsiN76W4qszIPYeo3v8oR77wJwQf/I9nfK8znQ0Q\nAtb3ROwYEzRPy72PIr1ocgzt/Pmcq+gsLv8eO8dc9k+4YAk8C1ytEULjOeJUCozjaBIlcBybIGhh\nWZLJimSi4hIEijhuT+arLYvJecnPXJawZSCi6CuOVyxSJSj4CloN/uGuFtMziiRSiBN5/9A+A1Ao\n+lRm6sydKD/q+xaHjqdcspVTaUjFnORXPpjlvqcipiuKXEZw0SaHrete3S7A60UKwYev9/ine6NT\nnxUgiRO2rtb0d781x/1WZYIAwzAMwzDeFP9w5/SiAACg1QjZ9+wo179rI3EqODrtcc6QQ09ynAln\nGGyXxtBWcuN7ke7CRLD60BMM3tbkTFOb/q4zV6TpyCouX93i6JxDmAgyjkIoxb2zLiePT2qtyXkp\nt1+1fOnL+ZbgwKRLqhdW0Nur1ALfjtFCEqcWllBkPUEQpFiWpJTTaCkIQnAcQXxa2n49EDxxQGKp\nmH2jCa0QekrQkU34yRN1mqcfdUgVTt4mk3XJ5FwsS7YPBNsW0pJoIbnvyQjXafCzN+UX7ku3xUdv\nfoFk+7eYqy/02bzG5kvfrDNX1XiO5t+8J8OWdWfPZ3irMEGAYRiGYRhvuHozZe/h1rLPzU43mJoO\n6ej0iWJopC5dchpfNQisPFGpDw3k169g8ycdnvv/HiWp1Lh4VcSTox5Hxhbnja/olVx/0QunpDgW\nrOuO0VqTJCE6jfk3V1gcnc0TJoI1vSm5F7jEsVmHRC2fQqO1ZqijwXTdoxa0p17lvMXMbEKtpSmd\n2FmQUmDbnOr+C3BwDCanFnYkak0Ai1g7LCoPqiGNEnJ9hXbJ1VQRNGMcd/Hq+PZ9ET9zvcZ6Fbn5\nb7auks2//8SraxpnmCDAMAzDMIw3gVIadYYCNFq1n4/i9gHWJ0c6WDV8HEeHBORpFQYZ3/Quund/\nn/q6DWz81QJHfnCY/Kp+/uc++PYDAYePtxtzrRqwefeVPhn/xSvxKJUSNudR6cJq/4p8Ey9bxrKW\nTpm0hmogidPlzw6c5IiIklND5wVhYtFoaTQC29KkqSRN2gd5tQYpJbatSZL2BVvLFjYSuL5N2AyR\n1sLnisKENFE4jqRWaZEuU1qoWle0Qk0+e/YGAcZrwwQBhmEYhmG84Yp5m7XDPjv3Npc8V+rMki/6\nACglGJnzUMMQiQypglg7zK+5krlv3cO68+bZt/E6evvOQ1gWhRzcdkvuFY0pCmqLAgAArWKiVoVM\nvmvR4/Mtwd5Jj7mWBQg8K8WxNHG6dHLd4QcUZJ3Icsm5DvWmRZxqPFeSKpirxEhp4Zw4jLuwI6AI\nwuWr4Tiug+1ZtGotvKwPGmxb0FMSDPbAQ+PL77J0FC0yvgkADNMnwDAMwzCMN8kHb+6kq2PxeqTr\n2azd1L/QLAywLUlL5KmTJ1IOINGuR7xmM7Wh85k/MELvB659VWPRWpMmyzebUmlEmi6k3qQKdo75\nJyoKtccZphauq7Hk4i2BDr/BmtIsUkBWtohTcG2NJRQD5YhqNaGr0yaKEhxHcvK8q5SCnrIgjs5U\nElPj+R6ZQoYkjnE8m94ej54+j4NTLgNre8gWFnfXEsBFm12sV1Cn/41UbaR860dVvvyP8/zdd6uM\nTp6pK7LxapidAMMwDMMw3hRbN+b4wq8O8f/+S5Xx2RTPdxk+p5ti+bSOshqKecEIq1H69GmLgGIH\n9b2HyHZ1oqT9Klc29Qvm9OgTzcmU1ozNp/TlqvTloRnbTNazpLq9I9BbaCGTEB1HdMoKq0pzpKLE\nyXSf4zM2q3tDomaMbSksyyFNNKWijSXBdSStE+U+g0RiW4IkXVql6GSqj+u5NGst/JxHoWxTKgiO\nHE/BslmztoOxY7MEsSDraS7bZPPuq19+t9430rGJiC//Y4Xx6YXg57EdLW57d5FLtprDv68lEwQY\nhmEYhvGmGej1+Pynevjbh3OkzztYa0lIUs35g02UfN6URWvCH92L97HraMxlkdHyVXteOoGVJNjT\nR0gzBZJSz8Iz0sKyXLTWzNVDLKlPHRLOuQk5J+bgXBmlJQrBls4JOid2kQ0q6FAQZspUejdSaVqs\n6IxxbFjXPcUz0wM4riRKBMW8JAgFJ3t+CQGpkrgZm6S+eCVcKUUSpydeJ7BtSTbnMjkVIhGoNEVI\nQSJsnFyetJUQA0fnBNWmppR76+4E3HlvY1EAAFBrar7zkwYXbfFfUbdhY3kmCDAMwzAM400lBFy5\ntslPDmSRp9KANFprunIRA50hQaII1UJ5nvTgQdz6NPnz16N/eBR3/gjsfZBkZo7qgUmOfGcXsquT\nzvfeRM/PfwDpOiTzVRACu7S4qZSqVRB7H6VwfBeOCkFaxKUequsvRWVL2E4WIQSNMCFKl+4WZN2U\nrmyLqUYOV6YoN8t8z0b85+5G5nL4rXnS6QNsbtZxypfhegIZR4zN2+RyEinFksntyf/WSlMoOEhL\nEsUprUaCYPFrhZBYtkRpQaUl8XybZivF8+xF1z06ofn2Qwm3v3NxmtBbRao0h0aXT/0ZnUjYczhi\ny9rXpvGYYYIAwzAMwzDeAtb0aboK8zxwIEMtcPBsxdbBefJ+O+3F0jHgoeOYdPt29J3fYsV//iwT\nh+a4Nv8M8Wg38eBavI4O+ntz9Fy6mh1/+E9U//5rjP7R/4W2fVQzQLo2+W3nMfibv0I9aDL/6G7k\nmlV4526ldOgwKj9M58gT5FRKcd+jNC/7MI7XPmgcp2coZwRk7AStNUIoGolLxivQKA9RqI1CNo/f\nmkdbNuvrjzPlbObRuTXYjiROBCu6QqbrWSxLo5Smp0tSb0AQpvT0+Agp0bqdBhTkU2amm2Qtl2Y9\nQqWKcncWrcF2LaIYMr5gfi7BtiVBsLi78eFxTZRoXLsdHMxWEn70aIvpuYR8VnLFBRnOWfkmBgkv\nsNBv9gBeWyYIMAzDMAzjLUFKzbbhyrLP+S64c4cJd+wl050l+3ufwDm6j57Dj6NrVayJMcThfcQr\n1jF/4fV0ju9g+Pd/jYn/+6us/sx72f37/0Aw1SSNbSo/eoDqI0+jLQsr46EaLewN64h/99/RNf8s\n0xtvQB99lJycxp2fJEglYn6CfBIQ9W4i9QtLxpcoQRgLZhs+5WxIqn0svw+3No2DAAFxqQfmZ7h7\nZA2eIxCpIutCVy5hfB5sS9DTKbFtST6rqdYBYbXPAKSaMBJkMjbdPVkyGYvJ8QZBK6RvRZm52QDP\ncwFNPts+uxCFMep5Oxdx0u5D4NpwdDzmy/80z+TsQnDzxO6AD99U4Jptb/zZAUsK1q5weLK69ID2\nyj6bjavfmjsYZytTHcgwDMMwjLcE3zlzV1/HtuldtYqhd99E10WXkS2vwDu2D6oVxIkDvTIKcA7t\nJPvco0z1bkV6Hv2/+lEq+ycZ/v1P4F+2GeKEzd/6U2TWoXDrtaz87ldY+a9/Renn3kX9T/+C+fNv\nxQ+mCOeqJP2rqB3YR2N+jj3eeTycv5nqyAyFsR2LxpYqmG54aC1oRjZxPSRMHUYz65lPchztvgRl\nu2DZkM1zbuE478rczzWFZ7h8fYVG2J6O+R5YJ+r+27agkGvfj5N5/+6JObDrWrruYk0AACAASURB\nVEgp6BvIMzjciWVJHMcCAZ4DSoFKNWm8dOeiv1OQOZFR8+37G4sCAGj3JfjBw03i5AUaH7yOPnBD\ngYGexWvUxbzkPdfmlj0PMF9XfPuhhK9+L+Yb9yQ8d+xM1ZSM5zNBgGEYhmEYbwm2ZZFxliYpSCnI\nee3HhZQI24bxw4jJY0teKwB77AjSdlC5AqOFc0lHx+jIQend14KGw3/4Fc696/8hCVNaTpmoeyXe\nB95Lxy+8H/XIIzgqJu4cQNk++8qX0xAlLqr+kF51nMnudzBZdZFRuw5/nAjGqlkqLZ8MDRAS79B2\nXBnh+5KRnouZ3j9DtTDcHp9lsc3aTlHWWe1PolLNyGyWjAeus3iSa9uQzcDJpr8nS3sK0W6m1j5L\nwIk0JFCpotFMmJhOiOMU210cVHkOXHWe1e4orDVHjy+ffz8xk7Jj//LlUl9vgz02v/OJDt5/fY7L\nz/d55+VZfucTnVy0ZWlloIlZxV9/J+GBHYo9RzVP7Vf87Q9T7t+eLHNl4/lMEGAYhmEYxltGIeNS\n8F1cW+JYEt+16ch62NbiCa2oziA4w2p1FCCEZu8xSVOUyA6UseOAjnN6AdDHj6OqdVb/8jVoLBQW\nsXaxrrqatN5EF8o4OQ+KRRK/wBF7PbFwWd3ciZRwpHQJ6uA+js7leHa8g9FKHoAuZuiUM6x0pyFN\nkVox468kuOMugnoEcbRozFLFHJ30yWUlGb993FecFgcI0U6DymehkGv/txDtSqbpiQVvKQRKabRu\nBwHjkwmNRkoap9i2hbQWLljOw7mr21M/AbxQoR3bevMy8HMZi/dfV+CTHyrz0VuL9HYun71+z1OK\n6edlj8UJPLRTEUZvzk7G2cQEAYZhGIZhvGUIIch6Dh25DJ35DKXM0gAAQA2eg3KWrxSjCmWUFqRa\nkLWaWF1diDikEbe7EPdfu5nZr92BM7SC5NChk+9Mavs4l19CqSDxLE2cKdGZi4hwOO6sppjM4KgQ\n15esivdwdf1fuV7fzXqxD4AsTc619jDZfxHZ6nEiZSOFJp2cxKnPYh/ciTytI3FVFaiL4qkGYWKZ\neffJib/rQMZv7wCoExk8Wp/4Gd0OAvIFr33YOEood7YPC2u1MBmersB8feE+rz3DAeChPptz1731\n8+9Hp5c/qD1fh+0Hz3yI22gzQYBhGIZhGGefcjfJirVLHlaOR7z2PILUQRe6yTzzEN65GxHZLEmt\nBq5F8Zx+wkPHkSpm7rO/SfN/fOPET0t0oQxKU4sKNJ0yjqOxZcqE7EMj0bSX4ueGLsIRirJV4QL5\nFFvFdlZ6E5RFlbrXQ4E6Ngk2EVYuT2FyL7I2h1WbA0AjSIqDbFsr2dwfUPDSUz0COPGK57OthR2A\nk4QQKAVSQiZrgRb4GQfbsZASVq/K4fvtC0sBu45Kdo8IUgUfvDHHcP/iVfZyQfL+M+Tfv9XIF5jF\nuqb0zYsyQYBhGIZhGGcdISTx5e8h3HwxabmHNFcg6RsmvPQmosFzONQaQMYN5r7yLQpZhcqV6avv\nZ+V7L6J5fA4rn0VMTaJHjtP48t+QTs+gtcZq1Yn27qLy6E4mJlISJcl7ilDmmN8/QSJc0iimVRpk\ne/ctpArqbgfr2U2hZGPLFCUsFJqBcC+zTZ/oIx9j3/C7EWjSKCb1y0S9m8gODnP+KljXnXDFmibn\ndIeU/ISTAcDzdwaU0sSxao/ztM0RrTWOY5HxJWGzTv1EdZ1SwWJgIMPmzSWyWQtpWzx20OF7Tzl8\n/X6bVuzw2x/v5OduznPNtgzvvirL5z/ZwYWb/Dfot/jqrOpbPlDpKcO5a8wU98WYO2QYhmEYxlnJ\nyXUQbL2a1i230Xrvxwmu+xla/Rs42FqBH9WR/+532PJvP4BIFU7SonV4ioF3nsfoXc/Qdeul7P3b\nR9upNNMzBP98JyCoN1OqA5vouOF86l/7Jo3II1WabH2M/X/+Tbj3+3RmAnjyEXzR4rHiu/DjGmMd\n55G0WoSVFr0TT6KffQpxZA8ZGTE3cC71jmEmOrYQrr2SYPhSkvLKU5/j+HjAnd8fZ9+zo2zubeLa\natnUIKVOHAJG4XkncvsF5PMWA302riu56MISa/pj0iSho9yOFDK+xarhLFs2+NhWilKK6arknh3t\n3YKbL8/xsfcW+eCNBTqKZ88S+i2XWEsCgUIWbr7IelPPNJwtzp7ftGEYhmEYxmmk5TK0dh3P7T5G\nJGxi7TIVlvCjCgOPfZPVf/4JpC0gDknmash8gee+fDe9H7qGsQf30/j6HQsXixPCVDKpB+nL96H8\nhM73rWbnjKCYUZS/8TfMBwL/kXtYc/Mgx6ciCqRkGuMcLG5DpRJ/NiQzNEz8f/53nK4s6Xm3YYuU\nuarNOfIgB9a8hxWHfsze2WHKBYeNfSF/8aX9fPdH4zSa7TyfO743wTtvHaYwONBOPTohTdu7H73d\nDtOzMaWcRmlBnIAlLY6Ph5SKFrFy2LQxz48fi5meTentFliWpKPDpasQs6pf8fReqLdSZmoWf/L1\nmMGOlFsucxnsXnr2Yr6asOtAyIGRiDQV9Hfb3HBpFs998XVkpTSP72iyc3+IlPCOzRnO2+Ajlotw\nXoF8RvKp9wme2JMyPgcZFy7bIinmzBr3S2GCAMMwDMMwzlq2bdPb303QDKhUQzbFz9LX2IXcmgeV\nQKxI/RIjoxGNoy3SBI78139efJFCHnnLzVRDD8uBluhA5z1kPs/UzoQ16yp4nTl6vvxnVD//e2QI\nCC++gb7nHsDJF5ksbiO1XKZ7L2RlvJ/srTeS3v9jUr+MrS1SJcjOHYJSB0c6LmPdX36axm/+Ad9/\nsotv3jmC0guT4vGpiLu+e4TPfKbIWDOPkO0dgDgBEHieIJ+zCEIo5CXFTEwjshFCMD2bUCpaoH3W\nDQQcr8TMWNDb7aK1oNK06S1GDK+QPLk9IlewCGLBM/sSxqZTPnyDx5O7EybnU1xbUKmEHDraPFVp\nRwiBtCTfvr/BxefluP19Pj+6v8LOfQFhrFg54PKea0t0lW2U0nzpH2Z4ZHvr1Ge7//EG11+W42Pv\n73zNfv+WFFy6ZWE6q5TmyT0xU/OKwV7J1jX2axZ0/LQxQYBhGIZhGGc9P+vjZ33QRaL5PFZ9EjTE\nvevQfonezRC/713s/cX/bfEP2jbyZz5EpW8zaIGb1vDSkJrbw1wzS2F+lENzK7js5qtIBjqoXHM9\nOgzI+RH+4e3ITe8gSj1SoXHCmPnSEM01fRSrszSf2U20aQ2em7IncxXbgj2M660UPvMJRv/3P2fL\nf/hf+T+u3oHb6fPv71pLLWyvxE/PxDz86BQDG/N0pJP4qsmkM0Qq2g0DXEfguO2JbSu2KbgBcQxR\npJicSenvtujqcqgph0olpbtTA4IoFkSppKcYU8hbxKkiaLarFU3Oaf76zoBUWySRalcW0hbYLkTt\nMwZaa5RSBIHg4Wda7Dg0Rb0a0qoHAOw9HPHcwZDf+HgvO/cFiwIAaDdVu/fRBu/YlOHc9Uvr/r9a\nE3MpX/9ewJHxdmUgIWDdCotffq9PPmN2B57P3BHDMAzDMH56CEHaMUS0chvR8Da0Xzr1lNPdyYa/\n+0v8X7wdfc0NqFveg/q9PyL5tc8DAq1h3ZFvI3M+ldgl46R0HnqcPQcjqqVVeDLG/eWPcfjBI6wQ\nx4k3XYBVmwEUemyagWP34EZVam4PzQ1XUrfyBAoGijVimUPWKzhxFZEtwH0/5LnqEPX8APrZ3Xz1\nk8f5o9/u4V1Xt8ueds48y4ftf+KS8n6umP0mPxt8lfPDh9ofUYLvLlT8aYQCIWX74HCkiSJNUxXw\nXEkUp6SqfdRY0/4/SyjiRNFsRMSndRXWwqajO0dnX55cyUdIgeu7i1bStdJorUmTlCSFQkcez3dO\nPT8yEfPd+yrsPBAs++tJU3hyd2vZ516tf743PBUAQLuE6v6RlG/d++Y0PnurMzsBhmEYhmG8bTil\nAlv/4HNs310lKA6eelxrTefkdjITe/BLNjOWoNazBmv1mvYLPI9QuuTcBPvydxBHNZr5QfQDT7Gy\n90kOfW8HpaHDNFavIpXD1N0unK19DMt5VLNGS+eY7d3I5t3fIchsBQTNyKJvQ4meXB/7fnSYlR/o\nof/8q/kvlx2m+pO9hM4KOmd2UVl/Ce7hpxjomUPwDM84mzk6qujtlniOIIwkYdBCCoEgbf9TtA/9\nCqDZSsllLRxb4zspQmmmpwJq8y0cb2ECr1JFvRpgWZJM1gGtadZDMsUMcSsmjtodhnOlDK1aiFaa\nJEkpdReYHJk9dZ2jYxHlwsJ1n0+9DiX8p+dTDoymyz53YDQlSjSubdKCTmeCAMMwDMMw3laEFGy5\n94+Z2ngTzfIqhE4pzh+guO8BrMoMTimL3YgIuzYws2Iz59gWOTchET4FOYdtB4j9e0jcPuSxowzW\n97P/q19h+t1rKZ03yVBmJ7HI0DjSonvAwhrZiRgQpIHL1NobSWaOk1k/SF91D+GKPqZya8jIp5jQ\nfazqajJWG+L8m85nzO8m5/TQMX+Q2uYbiXc/Q7HDZWX3RgQWxycSujsklgXzcxHZvItLk8nZPCsH\n26vuSaKoVBIsS9KZTyh4EcdnBLOTVZIowXYXcuaV1sRRSkxKHCV4GQchJQKwXRut2/sJuWK7ERmA\nSjW2v7ixmOdINq7xeGzH0hV/KeCCja99KlAj0CfOTSwVRpo4Nr0Dns/cDsMwDMMw3naE1vQ+9o0T\nLXmB0zrrWq0KXTiM4VIsBJzTWUG26qhCDo0kDGP8VFAa24W7ukw4NUe2O0drLqB7bB/eoUcobNnC\n2B/fQc8vXUMydogVF8QcHnNx3/kuwtIq4l/4X/D+2x8z+vFfZXDmOIUVWfbIPrqtiB31MkOdw7hH\nn6Pa2ctU8RL6amMU84q9mfWEKXSVFMcnYa6Sks1Y2LakXmkxkjr4bsBgv4dOUuJYnUoTmptXdGXg\nqSfniFrtswBRK8LLtlOQVKrQqUZYApDEUYrtWMSBADSu72LZAiklftYjChLQoJLFK/Dnrve57tI8\nO/YFPL1ncVrQFRdmueB16EOwoseit0MyObd0m6G/S5I9O1ofvKFMEGAYhmEYxtuKEBIxuAb93Fw7\ncfy05rzS97EFCKfdiXdNT8SAPU+y5wnSjVfgBzOEs7OIqTEaTz+H7PJp7jqOU84Rph7R+AzVx3cT\n7Zkh3bmX+Se6SY8coyMISZsdJDmfxjtuobr5Rla9d4rkwON412xk/C++ytz5HdQqCteRTAVZhjM+\ntVwvk3OCfXMul2SrjEUdZHwLKRRCauYrMUJCNudQnW+RJprAkzRbMWLfblZMTrBh00b2N4aYrjlU\n6rB9Z/O0u6FP9STQul1dB6XBhjQReL5D1ApBtCf/+VJ7FV+c1q63Vmlfz3Hg0vNy3HJVESkFv/YL\n3fz48Tp7D4UIKTh/vc/lF2Zfl2o9tiW48jyHbz8YLtoRyHhwzYWuqRC0DBMEGIZhGIbxtiOv/QDp\noR0QRQsPWhKvo0DcDFFXXsWANUKfVQdATI+Rad1NdUZT9muo53Ywv7/GxH1TrPuFq6juOkp+RZZ4\nzCWaajL948cBmHt4J/FohczGFdjFDqxV50CS0ohtjvRdxqqLJLNuluCXf50wtmikDsVMhKMj0nIP\nc4HPSNWhFWV4xLmESGeQETQamjBUSClpNFLSVCOEIE01MoW9+xr0DG8k/9RDbFyxjtzIPp5qrCfG\noW+ozNjRORAsOvgrhECh0ArSVCFtgW0L8kWfRj1qB08nU4fS9oq7lPDOSxyUKnLBxgznrFpYcrcs\nwY2XFbjxssIb8Svlum0uhZzgiT0xtYamoyi5bKvNltVnPp/wdvaiQUCr1eLzn/88MzMzhGHIZz/7\nWW644QYA7r//fj71qU/x3HPPve4DNQzDMN66zHeFcbaRPYNw+68jvvVlUCnStnBLBdJEkXauoGJ3\nMFSo4QiFnhyH555FyQKjX3mMSReEbeH1FEnrMcHoDDpOSaOE1tgcjbE6xCkUBIXBMrNHK8SNCJEc\nJ/TzKJUQhD5xdiU9ZU2S2hwrDWNJsC1FKRsxUD9IPbQ5FAwxX0lwPIvZsIALTM4oYiXadfslRGGK\n7YDtSHI5Bz9jESeanh6byQ2X8I9P9vHJtQ+yK1xDlDrkCu30Hy/j4Z6o7KO1Rp0IJDTt3RGtBdJq\nV02ybAt5YvU/TRRx2F5uX79S8qF3Ft+U3+Fytm102LbRTPpfihcNAu655x62bt3Kpz/9aUZHR/nk\nJz/JDTfcQBiGfOlLX6Knp+eNGKdhGIbxFma+K4yzkRzYgP3zv4Le/gBqaoJQ2jjbLoRsmZWZOlpp\n1PHj6Pu/D0pReXIfaT3gZAZ8a7JBfmWZw//yNAA6VUw+PU0w0z4Q233+Orx1g5TjEPmLv8z+uTXI\nVoGu+DjNtJ8kLpJlnLGwAy0sBJo4FWglKR17iqPlG8m5MVMxdHQIWi2FrxSF2jGmvWFsWxBFiihU\ndHRKiiWHckeWoX5NZaxG1o1pDKxDKBvh2GzunmfPbBd+1qN3ZRcgFqXJSKFJ0xTHEYSBwnEltmWR\nKvC9dlWfKIyJTgQAfZ0W777SeyN/ZcZr6EWDgPe85z2n/n1sbIy+vj4AvvjFL3L77bfzp3/6p6/f\n6AzDMIyzgvmuMM5KQpB0nYO4rBPZmIQ0JkESP/0YenwUGnWYmQQgqkdM7ZxecomoFhBX23XoWxMh\nlp+g0/Yhg+zG1WTWFMgNnsdEYQhHFjlaKfP0pEtHZ4rjWkyGHVQaNvl4im6nzoF0JfWWhT+8iqzM\nMKTnGbHKCK0Y6IbCwWeY8AYZUoc54q+h1WqX6azXBX0DBRwbekshxekxeksWz7pZHAmPJ9vwHUlH\nLmXv3oBMzqNVD2nWA7TW2K6N6zkIKejvzzIyUiebc0mihKu22rz/Kp+5Wsr9T8XUGjalguRn39lB\nFLw+Nf+N199LPhNw2223MT4+zhe/+EUOHTrEnj17+NznPmf+YjcMwzBOMd8VxllHCHSuizTXdeoh\nqxKRjhxDTU+2a+VPNZl4YpKkvrQGZTS/uPpNGi5UymkcGsXvXoVYsw5fR2SkZmY2QeHiOBLHFtRD\niRYWxZLNxh3/QveaS9jNZYhykVVOg/rIDKv7i9RCh6yn8MePEmzYSDYXIwJBT7dDqxETRQm2nSXr\npQhhY3d2kcYpji2JI810M0OjBf1dUK+2KHcVsByLNFXEYUzYighdm1wxw+RUwNDKPBsGUi7bLOku\ntTsZdxQsPnCtderzlQo2U8v3BHtdRLHiO/fMsvdQCyFgyzlZbr2uE9syh35fiZccBPzd3/0du3fv\n5rd/+7cZGBjgd3/3d1/WG/X0vDGHQl4vZ/P4zdjfHGbsb46zeew/DV7tdwWc3b9DM/Y3x2s+9p7r\n0Fddw77/+Acc+6tvEMy8jJnuyUpDEqjVqB6YZ3B1ixnLYk51kKYxadI+tOp6cFH5IA/NbGEqLDL/\nxH4KgaD3yvPRQpKrjzPx/cfY8PGNPL7fw1Yh8py1DA1YzCSrSJuKJIFSh0d1roVrCxqBxXQFtqzO\nMTJpIy2BpTTzdc3kVEw5b5Em6kTNf3A8hyRO2o2/ooSgEZIt+FQqIRsuL7D5nBeurflC9z4IFVNz\nKV1li6wvz/i6lyJOFF/447088Wz11GNPPFvn4LGI3/u367Hkyw8EzuY/86+FFw0CduzYQVdXFwMD\nA2zevJlGo8H+/fv5rd/6LQAmJyf52Mc+xte+9rUXvM7UVO21GfGboKencNaO34z9zWHG/uY428d+\nNnutvivg7P2+ONv//JmxL1X6zGeYfGw/wT0PLTwoJdJ3Uc0XDgwGrhggqMPEXY+z+tYN+FEThYXn\nJURRSivS5DKKOJUIAUpZVH/204w7eTqdGXS1gtOcJfKKuEHEqn4FoaC1+lw8mVKPBXEMSaJJEs2a\nQYFlgUxgti6RwkJYDo1GQi5nE4SCaiXg0DGX9ERlnyRu71rYrk0ctLsBp6lCpZpGLeLr329Sqba4\neGN7ujg5p9hzRJHLwoXrLPr7i8vee6U0d/wkYseBhPk6FHOwebXFh67zXvGq/XfvnVkUAJz0wOPz\n3HHXKFdfUnpZ1zvb/8y/Fl40CHj88ccZHR3lC1/4AtPT0yiluPvuu0+dEL/xxhtf0l/qhmEYxk8v\n811h/DSSvseGr/4F09+8i/rjz2BlfLo+8j6yW9ZTufchxr70P6jd98iiPgMAnVs6sTMuSUWho4SZ\nJw7inHMtHU5MX7ek2ZTMzMbkfIeRzBBxpEFC3NFPrZWjKx1H5rOkzXn6btrMwcinrxwyUc9QbUBH\nJj11gDdNIQxTtm6z2DMGqWp39U1SQcaO6ezw6O8WTM1BEivGJ1M83yGJUsJWjJCC1Sscsp5g++4I\nx5FksjbVakKiBN9+MKWQEew4lPLsAUVwoqLq/U+nfOw9Ed35pfftzgcjfvLMQupUtQGP7EyBkI/c\n+Mq6du09dOazBzv3NV52EGC8hCDgtttu4wtf+AK33347QRDwn/7Tfzr1l7phGIZhgPmuMH56Ccui\n5yPvpecj7130ePmGKynfcCXVh59g5HO/jU5S3LxLYbiABqrjMbUdxwE4+q1n2LLtflZdMkTG6eDY\ncQuVgps0UMImSSL6cy0CXAbrT1NWTXZmz2dDr0f2uV2ct0owIc6n4Csm5y2O1SxK+YhsJkOvnKLh\neziORRBqNBrv6D6qa1dRzLa4YGOOINTM1QApEGiU0tQq7Um1EHDxeQ7nDPts3RDz8F1HcLu2EgYJ\nWiuCBP7xxzHVul5USWh8VvO336vz2Q/ai1b3k1Sz8+DiDsIn7T6c0go1Ge/MuwFjMyk/eSZhel6R\n8QTnr7fYtsF5wR0EcybglXnRIMD3ff7sz/7sjM/ffffdr+mADMMwjLOP+a4w3q6Kl1+EGlyDU53A\nyjhUJ1KaY/NEMw0ApG8RVWrtXQRRJchkKeTz2LbEyjiAZkPnLOfIw+wQ5zObHWYOcFpNGn4eZ+UG\nMoRE2sG1U/pKcCS0mKulKKlZn5uia1WRWdWJSjQgyB7fj2cNEqeSJG037dJa43oOpbxNJiuZGm+S\nLzis7Id1K9sB+9phh+FLa/xEaxoFh1ZLo5SiUtfEYYrtWFjWQnB/fCr9/9u78zi7qjLR+7+1pzOf\nU3MlqSSVkHkgJMHIjKIypfHVi4Bc9crVt+37SkOrfdUXh9t6u/3c7n7x029Ptz+ILbytEulGaecJ\nQVAQImEKIQlJyFzzXGc+e++13j9OUkmlqpIUGSrVeb5/wd777POcTW3WevZe61m8tEOxbtmR7mS+\naBjOH/Nq5JDhPPQPa1oa7XH37+sM+dbPSwweNUpn296QvkHD2pUpntk0jD7m1K4Dl6yZ3sMpp4o8\nphFCCCGEOAVOLMbQtk56XzzA4Ja2kQQAQFdCki0paD9AYqidILCYOwPmOG3UJkJqnDz95TiBXV25\nN0zX06NmUJsMKf7maeK5DoJEHQC2ZYh4mqSVJ51QzPXfoKbBA9elEDqEBixbUQotLNumPx9hMKfI\n5qtzAFJxhzmtKVLpGHVNKebNz7BsSXLUE/74gtk0bPoZgYYF81zQYKofJ/DDkQnFh+WPGaWTiClq\nkuM/mc8koD4zcdfz1y/4oxIAgFDDxtd8Llqe5J1X1OAetQ5YxFPc+PY6Vi4eZ0ySOCFJAoQQQggh\nTkFh63FWw9YQrYvxxr/+HqdtFxhNY6JMPtVK0i4QdYoke3ZRdlOUQxtba1AWKuaRa72Q6N7N+G4c\nR1XH6GPg0vZ/p6nG8K6GV0nHQ4phhI4+Dz8AS0Hf8qvJlmz6CjGKZYuDXZpcXhNqTRAYsrkA17Ox\nHAc/HP1UXrkOjckSkajLUNFm1QVl4l5AqVBdTyAM9MixERcWzRnd4XdsxYULxn/Sv/ICh6g38dCd\n9l497vbBHLz6Rsj/+f6ZfOHOVv7gmlpuekcdX/pEKx94T/PE114c10mXCBVCCCGEEGMF/WOr1hyt\nZ1M30RkRcjvbKM2EzqEYZV/T3qO5IL+bFr+LwL2M0qAiEgd0yEA+TmpWA0O/bafyrjSqbChUXMJy\nAdo7qF07CJlaPFVhR0+aviGFparj/XM+tPda7G8vMHNmklBb9PUV8Ms+Pd1FolEb3zeUKyGeM7rj\n7Qz1Erv2GtRLFqVCyNrFedLNF9C2v59XthRQiSiOW+3kr10aoaVxbKf+hss8ADa/ETKYNWQSiuXz\nbW660jvudXLHzx0ASESr37N0YZylC+PHPY84OZIECCGEEEKckhNPTC11ldn/b8+RurIPx0kTlBTx\nTIroz37Ag4k7+PiyASJeIzG7QtSNYnKDkHHpeGoryt2A/8E/IjQWq3Y9wt5l17DM2wfKIbQ9LB0Q\nagdlaUplQ6WsGRoO6eos0dQYw3MVjm0RYOFGXGK2ZmAwxLEVhcCjUKkQ90IilSHUnl1Yq5eSTFiU\n8gHNlTbqi7t5aeG17NxZwA9DmmsdLlzg8P7rU/T15cb8Vksp1l8e4bpLDLmiIRFVuM6Jr9EFLRZd\nA2MnFc9qsFg+f3SGMJzXPLcNhvKGZAzWLVE0HGeokRhLrpYQQgghxCnwZjSe+CAD+T1dNHjDpP12\nHNsiWeqh/PJWwmKOUMVIhQNEjY+vLdqDegbKKXSygQNPbUMXi9hoBrxmktk38FRAxYpSctMQBnie\nQmtFLhtgKShVFH4lpL2zBJbC9SyMMtgOlMsa11U0NTgE2qFzwCPdtZ3Y3q0ULrmWIAxJpl2aTBfZ\njiy77UXUlw6w9uI6IhGbXFHz9jUu1gkW6HJsRU3SOqkEAGD95R6L5lijUqqGjOKmK0Z/14FuzTd+\nbnh6i+HVPfDsVnjg54bt+8cfTiTGJ0mAEEIIIcQpqFt/zUkdV7O8kVi+j/5CnKgdYP+P/87AKx28\nb+hR8l0DNDsDzAj24BAQjzqE2pD6zKcoJBqp3fRT0naWaDhI8L2HyakkI58MdAAAIABJREFUQ9Fm\nir7NwYEoyoRobbAdhRd1GcxqlIJSSWMphW1bGAOeA7GozdyWCI5T7QbmgigFFSe3cC3l0KY+VsKq\nFHhX/8N0N1yISsTJFPYTT0VxXIfBrGbj66e/LGcsYvGx90T50A0eb1/rcNMVLp+6PcbiuaMHrjz5\niqFnIKBcrFAuVKiUKgznNU9tNmMmLouJSRIghBBCCHEKZt/zx9jp45epTC6fQ9M1F2MP95PzGpnR\n+wKVrbtw0i4N5Q4yA7tIOGV8X1EbyzEn2seK/LPUNMdYduNcnLoUdf5e1PKLKO9vY2BHO6GyaRuI\nExqbQtHguTB/drUkaCqh8GIOtm0deopuiEYsZtTCzCaLec0+rn1o6I2yiGa7aTnwGxIqS9TVXJd+\nHtPdTy45k1luH/3DinS+jXgyQiRi89hzecLw9He4LaW4aJHLTVdEePtaj8gxE4lLFcPre32MNli2\nhXIUxkCl5HOgS9PZL0nAyZIkQAghhBDiFFjRCBc++32wx+9WKc9hxVfuILOwmf5YC62fuJbof/so\nQc5n/j3/lfLBDmpy+4lkuzDDgySCQcJf/hitLaywxFP1NxNtqce1DL4dQ7seA0WPVw9m2NaZxlLg\nB4a6Gpv6FHieTSbjYFkW9dEChAHDQ2WWL01QKGtm1ARkYgGzMiVcOyQeDjMz3E+iPEBjf7XSkVcY\nZOPl/zeeFdBc3svG/GLmD7+ArSs4nsNAX4nfvlI+m5cZreHhJw1ezCOejBKJOigMgR+iFPiVACXr\nhp00SQKEEEIIIU6RV5th1if/cMwcYTvmsfwrH8ZNeJhCluCRHxDrqK4kvPDP78BVPm7aJUjVo4OQ\ndPYg5uXnGf7KfRQHyxw0LQSWhR1xsJVF0D9EYe6FPB97B/v7k4BCWQrfN3ieRa7iYNuKmfUWmZjm\nD2c+xlXWb1mzKkUs5jKUVXh2UI3ZMdTFylwQbMOlui1SHKASwGP2DZSSM2iM5tjcVUdWJ1nkb6Up\nlsOyLILQsPNAcDYvMT9/Adr6bWzHxrIUrueQSMXwPBu/EqC1prlWsoCTJdWBhBBCCCFOg9n//Y+I\nzKhj8JHvERQqxFrqmfWfriC9ZBZ6904CL03/k0/RcOMlNN/6NuzCIKVnniBwklDXTClbQed8ig9u\nAMB//Amia6/mkhU2beUZzI4OoIOQzovfB9bhajmGeFRRKiiUMviBTU1Gk45V+Pzyx4gEJZY67bRH\nS3SVkmgU4VEFeBIqz0r/xZF/Nyh2tMfpzXksmlVE93byy/7V2PiUtEvWZAj9EDfinNJT91AbntpU\nZNd+HwMsmO1yzboYtj3+SYsV2NE2drtSikjcI58rgWLUwmfi+CQJEEIIIYQ4TRo/eAuNV67A3b0R\n5ToQBgSvvIDONDK8bSsLbpyN21hD+OwvKA3nyHYME712PWE0Qax9J8W9+9Fv7AcFevfruL37UPHF\n9OVcFlu9hAf3M7TwtpHvi0QU8ZjCaXSIZzsIapoBi0poY8XjmMEcEatCtNSDHyQxBtoHXBbFqk/x\nC4FHaBS2qo6l35mfwSuDKRwrJN/Vw6P9i8Bo/rP6LjvNAorao1jIEom5mFATBOCcZPWfw7Q2fP27\nw7yyozKy7eXtFV7f6/N/3ZbGHqfqUM8Q5Evjf49tWyil0GdgjsJ/ZJIECCGEEEKcTq3L8FsWol5/\nHsoFzLJ3QtMckut66bn3f6Je2IopVygTIX7F28n8HzcQf/Vpcq9vQfUUUJEIJu/T/1ov9ff+FcN3\n/r+kIhXCwMDvnqL2muspRNN4LjiOhSmXSe55jcW9P6N31bvpSSxgsBAll24g7Q4R+IZuXUelolGW\nReHQUH5joHM4wsO5G3hbdBMBNr8YXgdAuQK7Cmlm0ca11hMMqxp+rG+gXPSplH2a6jM89UKOrbsU\nMxssghBqUxZXXuQyo/44q34BG18tjUoADtuyq8KzL5e4cm1szL66JDj4dHbkqJQDLFuRSMdJZeKE\nWmO0wXPlLcBkSBIghBBCCHG6OS5mxeWjNnl1DTT/2Vco79xMkCtQP28ulgXevq2Eb+xi+I02Op7u\nQOeLAPjZCio7wMUHv8vghdcy9KvnKWzdz/w5P2TP1XePnDf63BM0fPsvabyhmWLfcnL1Cykf7KS3\nfhZpdjMYxOnXKQwKpUAbRbGs6M8qDvTa+P4svlVej8ECZaG1IQgAyyUolNlgv5c8CfyyT3aggNaG\nMDTE4i49gz49g+GhCkSarXsDPnBdlIWzJ+5i7tznT7xvvz9uEpDL+7Tv7SObOzIPIT9UpFKqEI1G\nQEFjzfGTDzGaJAFCCCGEEKfKaCgOgQkgkgInOu5hTiyDtWgNhZ98BzvXA3tfJ9/WxdC+Prqf66KS\nL8PhYS0aBnYNUvzmkyT+/FLyfoLU7Bhm6AAArgqo84aYmdpH+8FuCoN1lAoGO9dP4vWXKK9Yhyrl\n2daeplyr8SIOrguua9GZi9E7qNHaoDWAgzEGE2jC0GAMYDvsKs4gN1wCBkf9jsAPsawj9WWMMSil\nGMrBr1/wj5sEHG+RMXuCfvxPnsqSK4Q4roPWGh1WFwYb6stRSQRYlsWy+dKtnQy5WkIIIYQQp6Kc\nhVwnKqiOszG5HohlIDWL8WbPWrEkzrr1vHrThzHFHLocYsIxh4ENyihy29uYXeygp1JAJ9PUpDWX\nNuwkZlfw7BCz/lLKm95B3vMpffMh7BVbwUpjB6uwdEhv1mJ3dw9LV82svnlwFY6tqM9YDAyNrvAT\nhAaOGlo/3kRby1J4UYfhgeJI5/9oB7tD/MBMuFLwRUs8nttcIjxmgV+l4MJF3pjjCyXNlt2aaDyK\nUgpjDDrUlEtljDYoo3nLihjrL4+M+31ifFIiVAghhBDizTIash0jCQCAQkNxAPK9E34s2trChc98\nn/TFa8ZPAABCaFo3h2idS9rxSTQlcW7+EN6KC8l4RbxDi30pyyJ9zTr6563D3rODaPtO1CWXk8ge\npL/o8st9sxjsL4xEF+pq59x1FenkMR31oxIA19YE5bFrAcSSEQJfUykFGF3tkB+dCDj2uLnPiJUL\nPa66OIpz1FN/x4ar1kZZvWRsR/67vyrga2vkO5RS2I6NF60mDO9YF+HD6+MTVhYS45M3AUIIIYQQ\nb1ZxEBWOneSqgHC4m7B3EGfuApQ19rmrm05x+RMP8ern/5r99z4wZn+0MU7dikYGd3XhJzJ4qxsx\nWjPQspJ6XcS2jvTY/XlLeG3WJVzwZy3MbN9I34VrKLb/hO9uXUR/KUYsUT1WKYge9bC92rGu7jP6\nyPkUMH+WxVXLEjy5qUhnrybQCi/i4rgWg335kWN1qKlojRdxAaiUA17dUWL10ui4bxKUUrz/+hRr\nlkZ4ZXsZA1y0JMKSeWPfAlR8w44J5hDYtk1djc36q5Lj7hfHJ0mAEEIIIcSbpSdeMEsN9aAe/wmF\nbAn7XbcRXXfVuMfN+OTHSRa2s/uRV6gMFrE8h/oVzVzwnhV0Prcd5SbY+6+/Y8b6t1DpHebx7HWk\nvDILa/pYWt8HQG9mITUFzYEF13HJ5XG6/QF+0L+O37cPAJDKVCfbxiLgHOr9+YFhKFsdk1MT19Qm\nAnqGbYyyqMtYZGptyrbLf3lPlNxQmb97OI/va4LQYNsWYVD9rFKKwA8ILAsdhnTu6eOrO+GiZSk+\n84fNE9buX9zqsbh1bMf/aBXfUKyMX/pTKcXb1yWIeDKw5c2QJEAIIYQQ4s3ykph8D4pxOqpD/dhB\niXgMgk0/xW9pxZ01d+xxSlG64HIW3xYQb65BORZ+tkjPS7vo29yLu3Qp1sH9BC86RGY3AJCtRNjc\n00zc8Yl7IXuzDWgs6pIFOsImoo7PmtW1qEqezTsNi5bWkopDMl79SmPAVZoVLQGZuOHCVs3OPo90\nzh0VWiW02DvgYuWKROMe1qGa/MYYwlBTLlbfghiqw4L62ntH3ii8tGWYXz0b59rL02/68iZiihn1\nNvs6xo6Zqs9YXH9F6k2f+3wnqZMQQgghxJvlxSE6tpNrcll4/dWRf3fCEv7PN0x4mvrb38/+rbBj\nw0be+N7v2fuTzRx84gDD+wuU93Sw6LpWen6xicritSOfCY3Nq30zeL53PvpQl05ZkFdJSnYCjcVV\nl9Xy7msSYHk4libU4AdQqUDBt5nTbLF2gcaxYbg0frewULF5o8fFduxR4/IdxyYScdFagwYUpOsz\nR10Ew+PPDE3mao6hlOLK1REio3MTHBuuWB2ZcPKxODF5EyCEEEIIcSrSszGWB5Uc9LZjBnth+2YY\nOGpicBDgHWfki1KKhV//G/Z/+n+Q3/QylaEibjLKBe9bztxrlzHUlkVrxbO5VaM+N1R0cANNLKpQ\nSpH0B2iM9WKF0FOeQ0MyYG1DOwdej9HrpKmvHd1p7s7ZLGyqjrk3TNyh7h4cf0iOZVuEYTjyG2zX\nqU4oOHR4pTLRrOeTd9mqKFFP8dyrZfqHNZmExdrlHpevGr8Mqzg5kgQIIYQQQpwKpSDVDDQTPvoA\nVn54/OMixy9hacWizPvf99L3//wvMokCidm1hJWQ7t/vZc+vdtHzma/SlRs9Cda2IZMwtHdXmFUP\nSwovkIm5dLszSUeKNEcH0U6c1X2/5KXozcDoQvyBPvxkH6J2SDkY+zagUAzpG9Bjth/+oEJhMARh\nABpiyQSlXAFjDI5j8Vdf76JSMcye4bL+6hQzGo8/D2A8a5ZGWLNUSoCeTjIcSAghhBDidJnROv72\nRBK17C0ndYq6T99Dt1nAq/+2i1ce3ExnX5rOT/89B5veOuo4YwzGGEolTVO9IlvQ6GgSFYS4nsWM\nxDCupXFMQH1xP66t4Zi5C+nIkc790KBPoTj6yb3vG9q7AuLx8VfxCoNwZA6ACQ1BEFTLd8Yi2I5F\nf97m9T0V9rT5/PaFAn/37T66eydeMVicPfImQAghhBDiNLFu/Aj821ehp7O6hgBALAEXLMNeccVJ\nnUNZFrPu+gjc9ZGRbS0V+O7GCqpYBGXhde4lteslKqsvZahxHpWsplzWbI6t5GJ3B7NoY0A1otBY\nYQXf8mjK+DSlArqyCQDiXsgFjUfKm+YLhi37KjgOWBZEPIUfGIolqEtAPgv+UTmC1ppSoXRkwTBd\nTQQAHNchVZcimogS+AHFXIlyoUxnT8CGnwzyyTsaT/FKi1MlSYAQQgghxGmiHBfe/xnM5ifgwA5w\nXNTydTBnVXXW7ps0lIcL/vZu7Od/h3E97GK1Tr9unY/9Z/8bNWsmvYMW7eUkpdwqrolsJWoXUSaE\njv10N1zE0jklLKUYKESwlCIZCfHD6gD+Yhn291RrHPmHqp6WjyrNuXi2oiVjeOJFH8tS1RV7ixX8\nyqGDTfWtwGG24xBNVMfsO65DIhMn9EMCP+DVnSVe3l5g9dL4m74e4tRJEiCEEEIIcTrZDmrNdbDm\nutN2ylf/9WlqX3qOeR+7jszqC7A8h9zOdvY/9AT+D79F7s7P45crRCIOFe1xMGimRhUpapvh1GJa\n39aE4xgCbQi0RaAtCr5Nb85hxcwy+9oDhgrjTwxOxQyXLIH7v1cgNzh2YTSoDk0adQnc0cOHLMsi\nmoiQGwwIQ8Pjv8tKEjDFJAkQQgghhDjHRbe/xPI//yD1ly0b2ZZcOIv08rm89P89jz/UQdzJkEk6\ngKIrbCQfZumvxHEdmyWmkyIZ/NAi1Ec6+4G2eKPHY3B44io+rU2GqGfo6J54YTStj8wtsOxqh/9Y\njufguA5+pUJbt39kGJGYEpIECCGEEEKc42Ysb6B23dhx9PHWJuZeu4LCrtdoveStBLbBtTWDeZe8\nXUMhcAgKhnWpfko6wVAhNaYUaK5i43gWx04aPiziVjvr0YiC7Nj9SsFFiz36hhXFwMaORsFAxQ/H\nnDIS9wh8n2hESQIwxSQJEEIIIYQ4x82/diWW7h13X2pxC4lwHp6do2RClJtguFLBTdhkIj5d5Tgu\nIZSKHByaMc4ZDHVpC9sKRr0lANDa8NKOkHJJsWSeR0dvcWxsLQ5OMoPxFUdX7rdsi1KxWglIh5rQ\nD7Esi0g8yvIF41cbEmePlAgVQgghhDjHRTOpCfeFpTJNLQ7JJ39AS/tGkk6OefEeXF0gZvlE7CIG\nUGiccXp+jmUILJflCyMkIkce3etQ45dDsnnYuM2QSMdZvcTDPeoRcutMhyULknQOjH2qb9sWCkO5\nUMYv+0dtV9x2Y+2bug7i9JE3AUIIIYQQ57ggPQt7YB8Woxft0qGmXDOD1gNP0F4eoKFuNiYcJJHf\nQVfsStCKhelujAHleUTcECtQVMLqk3jLMtQkQiKuQRub1Us9nnulSKEEgT/6u3YchLtvruFgh8/O\ngxXmzU6yeLbh35+ZYCExqonAsdIJhefKc+ipJv8FhBBCCCHOdY6HCRX45ZFNQVh9CzB34CU8XaKh\nJcJOfw7lBx8g8drTePEEWR2ndfAFdDaLcj1q40UsG+bW5klFfWZkfOIRg21BIqIZKtkUCnpMAgDV\nMqXDecOCuR43XJ7kqotTWJYi6k4ctj6qapAXd9BaU98YZ9MuCzP+FARxlkgSIIQQQggxDVh+Bbvr\nAFZ/F9ZAD17nbmL9Bw/tdIg01aJrm+n97TbUUB9esZ+mRJ6+xALCYjV5MEYBiqgb0piuoBQcLuxj\nW9XyoTXJ8SfsZhLVp/jHWr1w/ERAa025dNQwIGURT0WpWHF+s8Xi2e3SDZ1KcvWFEEIIIaaBIN0I\nBqz8MFZuEEtXy3oawHgRep7fycyuFwn378PP5okP7KIhViBv12HyeQqBxUAhhq00obFwLQ1UFwgz\nBkINqUjI4jnjf//SuYqIeyQJ6Bv0+def9PHjX/UQ0wPE3SMlRMMwpJAro4MjbxQiMZdUKkLnwUEM\nim0H1agViMXZJXMChBBCCCGmgaBpEWH7VuxSDnVoLI0BTDxFWAno39JB7aoSicYEvTSQmjOD9j6D\nbSfI6TjFShTP0TiWRqnD3X8AhTaGQsliXr3P/CUWGM22fYahPKQSsHSO4sZLjjw7fm1ngQce2Udn\n75En/c0NOaKZDBXtUin7o8qDWpYiU5ugVKhweBzQYN6iPxvSXHOmr5wYjyQBQgghhBDTQSROdl83\nyeULsMoFMAYTTWKMZviZpxnc1Yu7q5PYombyd3yRgUqMSuCDl6THbcG2IeH59AzZ1CVCQnOkU+8H\nCpShohTaKG68xOZdFxuyBUjGwXOOvAEwxvDoL/pHJQAAXb0BK+sKdJcToxIApRSNszIA2I41si/i\nGlKxM3e5xPHJcCAhhBBCiGlizw9eI7/xJcKyT6gVYV8PuSd/y55HNqEHsnT+3QasS68gEoHOoTjx\nSJnAh7bkCvzAQinFvg5o63UoB0cG8isFiZhiuOSwb6i63XUUdWk1KgEA6O7z2bWvNG58uw+UWb4o\nSTwVwXZtIlGXRCaG1od6/sqgrOr5WhsM8bELC4uzRN4ECCGEEEJME07LPF78n/9O08UtxOoTVIZK\ndL5wAF3SEPMgCBhcdyORIIoXsWiI+QRbX6ar+a0MFBRDebh2cQev9TfTqAyhqVYZypcViWg1GciW\nqpV7JlrQNwwNeoLKPlob2noDHNfFcY8kGaViQCFXQqFobqmtTkg+A9dHnDx5EyCEEEIIMU1k3nUl\nJjB0bTzI3p++Tvsz+6oJABCvjYKvKadmkivZhIGhYjzqX/0puSL05xwiDng2NKQCCCtoHZIrW2hj\nc3g9L21GjeYZY2aTxwVzo+PuiyUiGMslnnDJ1ERwvWpXUykFykKbENe1sSzY023xL4/BUGFy1yAM\nDbl8eOTtgnhT5E2AEEIIIcQ00f2NhyfcVzIO7qwUab8TP1+mx15JXOXoufy9aBNS8hVGQ1uljkTM\n4DqKWJglcFPkKxG0VoAh5hqsCd4CQLVD/9531vDA93rpHzxSEcj1bFIN9biuTeBrSoUK9Q0xQg39\nfSVsxyY/eKhUqQbHtdjT7vOzTR63X33iDr3Whg0/6GLT5hyD2YCGOpcrLk7z3usaqkmGmBR5EyCE\nEEIIMU3kXtg84T6dD2lc3ESsazuD0dnV8f/FenRtM3FXMzSsCY1NMfQItY1rhbi6xNz0EBHbB1Vd\nK6AhHpAvc9whPxe0xvhfn1nAdVelWbkkQX1zhrmLZ5NIV2f6WraF47m07R8kFndIJl2iUYvQGIwx\nI4uIaW3Y0x7Q1nvi3/7gI5388Ff9tHdXKJYM+9vKfOeHPXzvZz2Tvo5C3gQIIYQQQkwbuhJMvDOf\noxjOxGuaD14UFcKBLptV6V5i0XqiBUOprHAyBh0GJMmiTBHLcWiI56mYOIODmh9tsyhUbDJxw9IW\nzVsW6pH5Ab98JsszL+Xp7g/IpByWzY+w4sJGwr1jn8RblsJ2HHq7cmTq4qSTLrnhMpWKj6NtIp5N\nPltGWTF6hqGl4Tg/rRDy7EvDRBJRbNtBKUUYhvjlCj96fID3XteA48iz7cmQqyWEEEIIMU3YNemJ\n9zXW4C2eC/EEaqCbSqFMabjAQJgiCBUrG3qwbZgRz1JjZ7EISTl5ADLRgO6ekJd3WwwXLYJQ0Ze1\n+N12mxd3V7uLT2zM8cgvhzjQGVCuQHdfwFOb8vz+pf4JY3I9h0o5IJFwSCVdvIiDCUOMMaTTNs0z\nE8RjinlNx//d+9pKVEwE1/OwbAtlKRzXIRKPUgngt5tyk7+Y5zlJAoQQQgghpomZn/lvE+6bcfVS\nYpkoQxWHmleeoC7lM+RH2F+ciQ4BY8jEAtJuntmxXookGIrOBKB/2OKNjrHdQoPi9bZqtaBnXy4Q\njrPCb1d3Cb/ij91BdYKx49gYrQk1OK5NuRwQakO5okkkXFbOs6lJHv939wxqbMces92yLJyIx4Gu\n47whEeOSJEAIIYQQYppoevfVzLt5Lco+qgunoOGt81n47hVU+vLUdmwhHjOU8hVyRYuhvEXPoMWw\nU8+c+DBRJ6BAioqKExiXsg+vHYwSYuE4CuuY3uFwQdE3FNI/NH5HOwwNxfzYdQO01hitydTH6O4q\n09FZJhZTFAsVlFL091eoy8ANbznx7x4Y1hNO/rVsi3hMurSTJXMChBBCCCGmCTsSZf4tb2Xeuy+i\n/anX8bNl6le1kFnYTKgsSjv2olLPk4nm2Z2OUCpqUOD7ASVfMc/bQ0gDvTQBhmIFdhxM0pv1que3\nIRKx8H1DpVItPTqUC/irB4fB9lBWGXPMjGHPhVjMJQzDkY66MQajq8OB8rkQYyCbDcgPFwh8KBV9\nojGP5pTBPon+e1PdxF1Wz7V527oTvEoQY0gSIIQQQggxTahYirKbJGZC5ly7YtS+YuDiugo1sB+V\niTJcADA4KmTV7C4KqoaMW6DH12ijyBVh47YU5qiBIdVFwhSuC2GoCAJNfrhMdT6yTTQeoZgb/dR/\n+YIoixa7/H4b6GNWGFCWwoxsMuSzPpZjUykFRGMeEfvkav1felGcx5/Lsadt7LCjq9fFaayVLu1k\nybsTIYQQQohpJFj+Nsq4o7aVQ4ed//Aj/FKF0ms78KNpCgcOYtmG2pRmUX2ONbUHwXLwQ5tSYLF5\ntz0qAYAjqwQrpbAsw2B/gcG+/Mh+27bJZKpvDZJxi3UrY3z05lpuWGdx02WKuU3VcygLLFthHTW2\nqFwKUbaFZSmwFBHXsGzOyf1my1J87NY6Vi6K4B3q72eSFrfdkOa/vLtukldQgLwJEEIIIYSYXmpn\nMjTjUsKffBs7Gad0sJeBre2EpRJ+YYDaixbxb7EPcNm2h9jceg8aGx+HmK0JQodB6nBsRVOtYm/n\n6FPbR829LeZ9BnrGVt1ZtzLO5RfWsGhBBr9cHNl+8WLFxYthwxOw+5jzGmMol30sy0IpRSRicdF8\nqJu42NEYMxpc/vSORvoGfXIFQ0uzi2PLImFvlrwJEEIIIYSYZmLLlvPLtV/kjZ+/wsFfvUy+rQ3L\n86ldtZAXmq8nVzOX5H+6nUTCo6M3ZPdwI7EwxxC1+CoKQF3KoA4N31EKXJdRT+7zufK4311fazNv\ntkdNevxnye+5DBbO0tWJwcYQ+CGFXBm/XC0tZDuKW672eNeaN/fb62tcWmd5kgCcInkTIIQQQggx\nDc2cFefBdz6AXRzmioFf0B+fxebUFQAsqo1izZ5HrMulUNDsG4hT483Gj9SOfL61IWTFzJDf7XRp\nGxjdJaxP+OwtFMZ8Z1OtxdVroseNKxGD299usa8z4J9/WCJXOJJo1KYVn7sjhqzrNfUkCRBCCCGE\nmIZmNzm0tKRob4MnvVtHtjc1xZg1K4YVrWBZCmXZaOXS4TfREK2W+bQw1MQNCQ+uX1Vha1tIx6CN\nMdCU0aycHbCoPs6vNpbZ3xVgK5jf4nDTVTFikZN7At86w+Ev/kiq9pyrJAkQQgghhJiGWmoDFi2I\nU1cfoaeniNFQVxclU+MR9TQeZZTy0GG1jKZjVYfjuJahIaFJVOf3Ylmwck7IyjmjVwJbucBjxQUu\ng9lqGc90Uh7f/0ciSYAQQgghxDRkKbi4tcRGHSWZzBzaaoi6hrpYiWJJUy4HDPQVWTzPZVlzBWVB\nTdSMWRBsIkopatMy9v4/IkkChBBCCCGmqVk1mnctGebFAy6V0MVzNLMSwxQqFpt7a8hmfRbPc1kz\np0R98uRq8ovzgyQBQgghhBDTWCrucOVCzZb9eYaK8NLBDEM5i862Ia5a5fC2i2yUkgRAjCZJgBBC\nCCHENGfbiovmVxcQK1cqBCEkYpEpjkqcyyQJEEIIIYT4DyTiKaT7L05EpnkLIYQQQghxnpEkQAgh\nhBBCiPOMJAFCCCGEEEKcZyQJEEIIIYQQ4jwjSYAQQgghhBDnGUkChBBCCCGEOM9IEiCEEEIIIcR5\nRpIAIYQQQgghzjOSBAghhBBCCHGekSRACCGEEEKI84wkAUIIIYQQQpxnJAkQQgghhBDiPOOc6IBi\nscg999xDX18f5XKZO++8k6VLl/K5z32OIAhwHId7772XxsbGsxHNN+0mAAAH/UlEQVSvEEKIc5C0\nFUIIMb2cMAn49a9/zcqVK/nYxz5GW1sbH/3oR1m9ejW33XYb69ev56GHHuLBBx/ks5/97NmIVwgh\nxDlI2gohhJheTpgErF+/fuSfOzo6aG5u5ktf+hKRSASA2tpaXnvttTMXoRBCiHOetBVCCDG9nDAJ\nOOz222+ns7OT++67j3g8DkAYhmzYsIE//uM/PmMBCiGEmD6krRBCiOlBGWPMyR68bds2PvvZz/LD\nH/4QrTWf/exnmT9/PnfdddeZjFEIIcQ0Im2FEEKc+05YHWjLli10dHQAsGzZMsIwpL+/n8997nO0\ntrbK/9SFEEJIWyGEENPMCZOATZs28cADDwDQ29tLoVDgmWeewXVd/uRP/uSMByiEEOLcJ22FEEJM\nLyccDlQqlfjCF75AR0cHpVKJu+66i/vvv59yuUwymQRgwYIFfPnLXz4b8QohhDgHSVshhBDTy6Tm\nBAghhBBCCCGmP1kxWAghhBBCiPOMJAFCCCGEEEKcZ85IEvD73/+eyy67jF//+tcj27Zv384HPvAB\nPvShD3HnnXdSLBYBePbZZ3nPe97DzTffzCOPPHImwpmUycQOYIzh9ttv5x/+4R+mItxRJhP7v/zL\nv3DLLbfwvve9j4ceemiqQh4xmdj/+Z//mVtuuYVbb72Vp556aqpCHjFe7FprvvrVr3LppZeObAvD\nkC984Qt88IMf5LbbbuP73//+VIQ7ysnGDtPjXp0odjj379WJYj/X7tXTSdqKqTGd2wqQ9mKqSHsx\nNc5ke3Hak4D9+/fz4IMPsnbt2lHbv/KVr3DPPffw7W9/m9bWVh599FGCIOBLX/oSX/va13jooYd4\n5plnTnc4kzKZ2A975JFH8H3/bIc6xmRiP3DgAI8++igPP/ww3/nOd/jGN75BNpudosgnH/tPf/pT\nNmzYwNe+9jX+8i//kjAMpyjyiWO///77mTlzJkdPufnNb35DsVjkoYce4pvf/CZf/epX0Vqf7ZBH\nTCb26XKvjhf7Yef6vTpe7OfavXo6SVsxNaZzWwHSXkwVaS+mxpluL057EtDY2Mg//uM/kkqlRm2/\n7777WLVqFQB1dXUMDg7y2muv0drayowZM4jFYvzt3/7t6Q5nUiYTO0B/fz8/+tGPuP322896rMea\nTOwtLS1s2LABx3HwPI9oNEoul5uKsIHJxb5x40auuuoqPM+jrq6OlpYWdu3aNRVhAxPH/qEPfYgP\nfvCDo7bV1tYyPDyM1ppCoUAikcCypm5E3mRiny736nixw/S4V8eL/Vy7V08naSumxnRuK0Dai6ki\n7cXUONPtxWn/i4rFYti2PWb74RJxhUKBH/zgB9xwww20tbXhui6f+MQnuP322/nxj398usOZlMnE\nDnDvvffyqU99atzPnG2Tid2yLBKJBABPP/00tbW1zJw586zGe7TJxN7b20tdXd3IMXV1dfT09Jy1\nWI91otiPtnr1ambNmsU73/lOrr/+ej796U+fjRAnNJnYp9u9eqzpdK8e7Vy7V08naSumxnRuK0Da\ni6ki7cXUONPthXMqwT3yyCNjxnrdfffdXHXVVeMeXygU+PjHP85HP/pRFixYwPbt2+no6GDDhg2U\nSiVuvvlmrrjiCmpra08lrLMS+/PPP49t26xdu5a9e/ee8XiPdqqxH/byyy/z13/919x///1nNN6j\nnWrsjz322Kj9Z7PC7WRjP9amTZvo6Ojgscceo6+vjw9/+MO87W1vw/O8MxHuKKcauzFm2tyrx5pO\n9+pEpuJePZ2krZgef3/nUlsB0l5IezF50l5M7n49pSTg1ltv5dZbbz2pY4Mg4M477+Smm27i5ptv\nBqC+vp4LL7yQWCxGLBZj0aJFHDhw4Kz8oZxq7I8//jhbtmzhtttuo7+/n0qlwpw5c3jve997JsMG\nTj12qE6i+uIXv8h99913Vp/snGrsTU1N7NmzZ+SYrq4umpqazkisx5pM7ON58cUXueyyy3Ach+bm\nZmpqaujq6mLOnDmnMcrxnWrs0+VeHc90uVcnMlX36ukkbcW5//d3rrUVIO2FtBeTJ+3F5O7XU0oC\nJuPrX/86b33rW0f9wDVr1vA3f/M3lMtllFLs27eP2bNnn62QTtp4sd9zzz0j//zoo4/S1tZ2Vv5I\nJmu82MMw5POf/zx///d/f05e78PGi/3SSy/lwQcf5O6772ZgYIDu7m4WLlw4hVGevNbWVn72s58B\nkMvl6OrqorGxcYqjOjnT5V4dz3S5V8czXe7V00naiqkxndsKkPbiXDJd7tfxTJf7dTxv5n497SsG\nP/nkk3zjG99g9+7d1NXV0djYyAMPPMCVV17J7NmzcV0XgEsuuYS77rqLxx9/nH/6p39CKcWtt97K\n+9///tMZzhmN/bDDfyh33333VIU+qdhXr17Nn/7pn7JkyZKRz3/mM58ZmVR1Lsd+11138a1vfYsf\n/ehHKKX45Cc/yWWXXTYlcR8v9r/4i79gx44dvPjii6xdu5Z3vOMd3HHHHXz5y19m586daK358Ic/\nzB/8wR9Mi9g/8pGPTIt7daLYDzuX79XxYl+0aNE5da+eTtJWTI3p3FaAtBfTIXZpL6Ym9jfTXpz2\nJEAIIYQQQghxbpMVg4UQQgghhDjPSBIghBBCCCHEeUaSACGEEEIIIc4zkgQIIYQQQghxnpEkQAgh\nhBBCiPOMJAFCCCGEEEKcZyQJEEIIIYQQ4jwjSYAQQgghhBDnmf8fmcOYFvVGzu4AAAAASUVORK5C\nYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "metadata": { + "id": "32_DbjnfXJlC", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Wait a second...this should have given us a nice map of the state of California, with red showing up in expensive areas like the San Francisco and Los Angeles.\n", + "\n", + "The training set sort of does, compared to a [real map](https://www.google.com/maps/place/California/@37.1870174,-123.7642688,6z/data=!3m1!4b1!4m2!3m1!1s0x808fb9fe5f285e3d:0x8b5109a227086f55), but the validation set clearly doesn't.\n", + "\n", + "**Go back up and look at the data from Task 1 again.**\n", + "\n", + "Do you see any other differences in the distributions of features or targets between the training and validation data?" + ] + }, + { + "metadata": { + "id": "pECTKgw5ZvFK", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Solution\n", + "\n", + "Click below for the solution." + ] + }, + { + "metadata": { + "id": "49NC4_KIZxk_", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Looking at the tables of summary stats above, it's easy to wonder how anyone would do a useful data check. What's the right 75th percentile value for total_rooms per city block?\n", + "\n", + "The key thing to notice is that for any given feature or column, the distribution of values between the train and validation splits should be roughly equal.\n", + "\n", + "The fact that this is not the case is a real worry, and shows that we likely have a fault in the way that our train and validation split was created." + ] + }, + { + "metadata": { + "id": "025Ky0Dq9ig0", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Task 3: Return to the Data Importing and Pre-Processing Code, and See if You Spot Any Bugs\n", + "If you do, go ahead and fix the bug. Don't spend more than a minute or two looking. If you can't find the bug, check the solution." + ] + }, + { + "metadata": { + "id": "JFsd2eWHAMdy", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "When you've found and fixed the issue, re-run `latitude` / `longitude` plotting cell above and confirm that our sanity checks look better.\n", + "\n", + "By the way, there's an important lesson here.\n", + "\n", + "**Debugging in ML is often *data debugging* rather than code debugging.**\n", + "\n", + "If the data is wrong, even the most advanced ML code can't save things." + ] + }, + { + "metadata": { + "id": "dER2_43pWj1T", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Solution\n", + "\n", + "Click below for the solution." + ] + }, + { + "metadata": { + "id": "BnEVbYJvW2wu", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "The code that randomizes the data (`np.random.permutation`) is commented out, so we're not doing any randomization prior to splitting the data.\n", + "\n", + "If we don't randomize the data properly before creating training and validation splits, then we may be in trouble if the data is given to us in some sorted order, which appears to be the case here." + ] + }, + { + "metadata": { + "id": "xCdqLpQyAos2", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Task 4: Train and Evaluate a Model\n", + "\n", + "**Spend 5 minutes or so trying different hyperparameter settings. Try to get the best validation performance you can.**\n", + "\n", + "Next, we'll train a linear regressor using all the features in the data set, and see how well we do.\n", + "\n", + "Let's define the same input function we've used previously for loading the data into a TensorFlow model.\n" + ] + }, + { + "metadata": { + "id": "rzcIPGxxgG0t", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def my_input_fn(features, targets, batch_size=1, shuffle=True, num_epochs=None):\n", + " \"\"\"Trains a linear regression model of multiple features.\n", + " \n", + " Args:\n", + " features: pandas DataFrame of features\n", + " targets: pandas DataFrame of targets\n", + " batch_size: Size of batches to be passed to the model\n", + " shuffle: True or False. Whether to shuffle the data.\n", + " num_epochs: Number of epochs for which data should be repeated. None = repeat indefinitely\n", + " Returns:\n", + " Tuple of (features, labels) for next data batch\n", + " \"\"\"\n", + " \n", + " # Convert pandas data into a dict of np arrays.\n", + " features = {key:np.array(value) for key,value in dict(features).items()} \n", + " \n", + " # Construct a dataset, and configure batching/repeating.\n", + " ds = Dataset.from_tensor_slices((features,targets)) # warning: 2GB limit\n", + " ds = ds.batch(batch_size).repeat(num_epochs)\n", + " \n", + " # Shuffle the data, if specified.\n", + " if shuffle:\n", + " ds = ds.shuffle(10000)\n", + " \n", + " # Return the next batch of data.\n", + " features, labels = ds.make_one_shot_iterator().get_next()\n", + " return features, labels" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "CvrKoBmNgRCO", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Because we're now working with multiple input features, let's modularize our code for configuring feature columns into a separate function. (For now, this code is fairly simple, as all our features are numeric, but we'll build on this code as we use other types of features in future exercises.)" + ] + }, + { + "metadata": { + "id": "wEW5_XYtgZ-H", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def construct_feature_columns(input_features):\n", + " \"\"\"Construct the TensorFlow Feature Columns.\n", + "\n", + " Args:\n", + " input_features: The names of the numerical input features to use.\n", + " Returns:\n", + " A set of feature columns\n", + " \"\"\" \n", + " return set([tf.feature_column.numeric_column(my_feature)\n", + " for my_feature in input_features])" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "D0o2wnnzf8BD", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Next, go ahead and complete the `train_model()` code below to set up the input functions and calculate predictions.\n", + "\n", + "**NOTE:** It's okay to reference the code from the previous exercises, but make sure to call `predict()` on the appropriate data sets.\n", + "\n", + "Compare the losses on training data and validation data. With a single raw feature, our best root mean squared error (RMSE) was of about 180.\n", + "\n", + "See how much better you can do now that we can use multiple features.\n", + "\n", + "Check the data using some of the methods we've looked at before. These might include:\n", + "\n", + " * Comparing distributions of predictions and actual target values\n", + "\n", + " * Creating a scatter plot of predictions vs. target values\n", + "\n", + " * Creating two scatter plots of validation data using `latitude` and `longitude`:\n", + " * One plot mapping color to actual target `median_house_value`\n", + " * A second plot mapping color to predicted `median_house_value` for side-by-side comparison." + ] + }, + { + "metadata": { + "id": "UXt0_4ZTEf4V", + "colab_type": "code", + "cellView": "both", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def train_model(\n", + " learning_rate,\n", + " steps,\n", + " batch_size,\n", + " training_examples,\n", + " training_targets,\n", + " validation_examples,\n", + " validation_targets):\n", + " \"\"\"Trains a linear regression model of multiple features.\n", + " \n", + " In addition to training, this function also prints training progress information,\n", + " as well as a plot of the training and validation loss over time.\n", + " \n", + " Args:\n", + " learning_rate: A `float`, the learning rate.\n", + " steps: A non-zero `int`, the total number of training steps. A training step\n", + " consists of a forward and backward pass using a single batch.\n", + " batch_size: A non-zero `int`, the batch size.\n", + " training_examples: A `DataFrame` containing one or more columns from\n", + " `california_housing_dataframe` to use as input features for training.\n", + " training_targets: A `DataFrame` containing exactly one column from\n", + " `california_housing_dataframe` to use as target for training.\n", + " validation_examples: A `DataFrame` containing one or more columns from\n", + " `california_housing_dataframe` to use as input features for validation.\n", + " validation_targets: A `DataFrame` containing exactly one column from\n", + " `california_housing_dataframe` to use as target for validation.\n", + " \n", + " Returns:\n", + " A `LinearRegressor` object trained on the training data.\n", + " \"\"\"\n", + "\n", + " periods = 10\n", + " steps_per_period = steps / periods\n", + " \n", + " # Create a linear regressor object.\n", + " my_optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n", + " my_optimizer = tf.contrib.estimator.clip_gradients_by_norm(my_optimizer, 5.0)\n", + " linear_regressor = tf.estimator.LinearRegressor(\n", + " feature_columns=construct_feature_columns(training_examples),\n", + " optimizer=my_optimizer\n", + " )\n", + " training_input_fn = lambda: my_input_fn(\n", + " training_examples, \n", + " training_targets[\"median_house_value\"], \n", + " batch_size=batch_size)\n", + " predict_training_input_fn = lambda: my_input_fn(\n", + " training_examples, \n", + " training_targets[\"median_house_value\"], \n", + " num_epochs=1, \n", + " shuffle=False)\n", + " predict_validation_input_fn = lambda: my_input_fn(\n", + " validation_examples, validation_targets[\"median_house_value\"], \n", + " num_epochs=1, \n", + " shuffle=False)\n", + "\n", + " \n", + " \n", + " # Train the model, but do so inside a loop so that we can periodically assess\n", + " # loss metrics.\n", + " print(\"Training model...\")\n", + " print(\"RMSE (on training data):\")\n", + " training_rmse = []\n", + " validation_rmse = []\n", + " for period in range (0, periods):\n", + " # Train the model, starting from the prior state.\n", + " linear_regressor.train(\n", + " input_fn=training_input_fn,\n", + " steps=steps_per_period,\n", + " )\n", + " # 2. Take a break and compute predictions.\n", + " training_predictions = linear_regressor.predict(input_fn=predict_training_input_fn)\n", + " training_predictions = np.array([item['predictions'][0] for item in training_predictions])\n", + " \n", + " validation_predictions = linear_regressor.predict(input_fn=predict_validation_input_fn)\n", + " validation_predictions = np.array([item['predictions'][0] for item in validation_predictions])\n", + " \n", + " \n", + " \n", + " # Compute training and validation loss.\n", + " training_root_mean_squared_error = math.sqrt(\n", + " metrics.mean_squared_error(training_predictions, training_targets))\n", + " validation_root_mean_squared_error = math.sqrt(\n", + " metrics.mean_squared_error(validation_predictions, validation_targets))\n", + " # Occasionally print the current loss.\n", + " print(\" period %02d : %0.2f\" % (period, training_root_mean_squared_error))\n", + " # Add the loss metrics from this period to our list.\n", + " training_rmse.append(training_root_mean_squared_error)\n", + " validation_rmse.append(validation_root_mean_squared_error)\n", + " print(\"Model training finished.\")\n", + "\n", + " # Output a graph of loss metrics over periods.\n", + " plt.ylabel(\"RMSE\")\n", + " plt.xlabel(\"Periods\")\n", + " plt.title(\"Root Mean Squared Error vs. Periods\")\n", + " plt.tight_layout()\n", + " plt.plot(training_rmse, label=\"training\")\n", + " plt.plot(validation_rmse, label=\"validation\")\n", + " plt.legend()\n", + "\n", + " return linear_regressor" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "zFFRmvUGh8wd", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "linear_regressor = train_model(\n", + " # TWEAK THESE VALUES TO SEE HOW MUCH YOU CAN IMPROVE THE RMSE\n", + " learning_rate=0.00001,\n", + " steps=100,\n", + " batch_size=1,\n", + " training_examples=training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "I-La4N9ObC1x", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Solution\n", + "\n", + "Click below for a solution." + ] + }, + { + "metadata": { + "id": "Xyz6n1YHbGef", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def train_model(\n", + " learning_rate,\n", + " steps,\n", + " batch_size,\n", + " training_examples,\n", + " training_targets,\n", + " validation_examples,\n", + " validation_targets):\n", + " \"\"\"Trains a linear regression model of multiple features.\n", + " \n", + " In addition to training, this function also prints training progress information,\n", + " as well as a plot of the training and validation loss over time.\n", + " \n", + " Args:\n", + " learning_rate: A `float`, the learning rate.\n", + " steps: A non-zero `int`, the total number of training steps. A training step\n", + " consists of a forward and backward pass using a single batch.\n", + " batch_size: A non-zero `int`, the batch size.\n", + " training_examples: A `DataFrame` containing one or more columns from\n", + " `california_housing_dataframe` to use as input features for training.\n", + " training_targets: A `DataFrame` containing exactly one column from\n", + " `california_housing_dataframe` to use as target for training.\n", + " validation_examples: A `DataFrame` containing one or more columns from\n", + " `california_housing_dataframe` to use as input features for validation.\n", + " validation_targets: A `DataFrame` containing exactly one column from\n", + " `california_housing_dataframe` to use as target for validation.\n", + " \n", + " Returns:\n", + " A `LinearRegressor` object trained on the training data.\n", + " \"\"\"\n", + "\n", + " periods = 10\n", + " steps_per_period = steps / periods\n", + " \n", + " # Create a linear regressor object.\n", + " my_optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n", + " my_optimizer = tf.contrib.estimator.clip_gradients_by_norm(my_optimizer, 5.0)\n", + " linear_regressor = tf.estimator.LinearRegressor(\n", + " feature_columns=construct_feature_columns(training_examples),\n", + " optimizer=my_optimizer\n", + " )\n", + " \n", + " # Create input functions.\n", + " training_input_fn = lambda: my_input_fn(\n", + " training_examples, \n", + " training_targets[\"median_house_value\"], \n", + " batch_size=batch_size)\n", + " predict_training_input_fn = lambda: my_input_fn(\n", + " training_examples, \n", + " training_targets[\"median_house_value\"], \n", + " num_epochs=1, \n", + " shuffle=False)\n", + " predict_validation_input_fn = lambda: my_input_fn(\n", + " validation_examples, validation_targets[\"median_house_value\"], \n", + " num_epochs=1, \n", + " shuffle=False)\n", + "\n", + " # Train the model, but do so inside a loop so that we can periodically assess\n", + " # loss metrics.\n", + " print(\"Training model...\")\n", + " print(\"RMSE (on training data):\")\n", + " training_rmse = []\n", + " validation_rmse = []\n", + " for period in range (0, periods):\n", + " # Train the model, starting from the prior state.\n", + " linear_regressor.train(\n", + " input_fn=training_input_fn,\n", + " steps=steps_per_period,\n", + " )\n", + " # Take a break and compute predictions.\n", + " training_predictions = linear_regressor.predict(input_fn=predict_training_input_fn)\n", + " training_predictions = np.array([item['predictions'][0] for item in training_predictions])\n", + " \n", + " validation_predictions = linear_regressor.predict(input_fn=predict_validation_input_fn)\n", + " validation_predictions = np.array([item['predictions'][0] for item in validation_predictions])\n", + " \n", + " \n", + " # Compute training and validation loss.\n", + " training_root_mean_squared_error = math.sqrt(\n", + " metrics.mean_squared_error(training_predictions, training_targets))\n", + " validation_root_mean_squared_error = math.sqrt(\n", + " metrics.mean_squared_error(validation_predictions, validation_targets))\n", + " # Occasionally print the current loss.\n", + " print(\" period %02d : %0.2f\" % (period, training_root_mean_squared_error))\n", + " # Add the loss metrics from this period to our list.\n", + " training_rmse.append(training_root_mean_squared_error)\n", + " validation_rmse.append(validation_root_mean_squared_error)\n", + " print(\"Model training finished.\")\n", + "\n", + " # Output a graph of loss metrics over periods.\n", + " plt.ylabel(\"RMSE\")\n", + " plt.xlabel(\"Periods\")\n", + " plt.title(\"Root Mean Squared Error vs. Periods\")\n", + " plt.tight_layout()\n", + " plt.plot(training_rmse, label=\"training\")\n", + " plt.plot(validation_rmse, label=\"validation\")\n", + " plt.legend()\n", + "\n", + " return linear_regressor" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "i1imhjFzbWwt", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 619 + }, + "outputId": "3bac311b-8b73-421e-85ca-48a584ab1ad2" + }, + "cell_type": "code", + "source": [ + "linear_regressor = train_model(\n", + " learning_rate=0.00003,\n", + " steps=500,\n", + " batch_size=5,\n", + " training_examples=training_examples,\n", + " training_targets=training_targets,\n", + " validation_examples=validation_examples,\n", + " validation_targets=validation_targets)" + ], + "execution_count": 14, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Training model...\n", + "RMSE (on training data):\n", + " period 00 : 207.35\n", + " period 01 : 189.89\n", + " period 02 : 176.80\n", + " period 03 : 168.72\n", + " period 04 : 164.06\n", + " period 05 : 161.84\n", + " period 06 : 160.95\n", + " period 07 : 160.91\n", + " period 08 : 161.73\n", + " period 09 : 163.00\n", + "Model training finished.\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGACAYAAACz01iHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3Xd8FHX+x/HXlnQ2vZDQi7SQQkiQ\n3kto0otAQAS9U1BUPNE78Cx3KlFROggiCopIRAEFREQEVBAIhFACSA0kpDfSs5nfH5z7E4GQQDaz\nm3yej4ePh1tm5r372WU/+c53ZjSKoigIIYQQQlgRrdoBhBBCCCEqShoYIYQQQlgdaWCEEEIIYXWk\ngRFCCCGE1ZEGRgghhBBWRxoYIYQQQlgdvdoBhLBkzZs3p379+uh0OgCMRiNhYWHMnj0bR0fHe17v\nF198wejRo2+5f+PGjbz00kssW7aMHj16mO4vKCigY8eO9O3bl7feeuuet1tely9f5o033uDChQsA\nODg4MH36dHr37m32bVfEkiVLuHz58i3vyYEDB5gyZQp169a9ZZnt27dXVbz7cuXKFXr16kWjRo0A\nUBQFT09P/vWvf9GqVasKrevdd9/Fz8+Phx9+uNzLbNq0iaioKNasWVOhbQlRVaSBEeIu1qxZQ+3a\ntQEoKiri2WefZfny5Tz77LP3tL6UlBRWrlx52wYGwNfXl2+++eamBubHH3/E2dn5nrZ3L55//nmG\nDBnCsmXLAIiJiWHSpEls27YNX1/fKstxP3x9fa2mWbkTnU5302vYunUr06ZN47vvvsPW1rbc65k5\nc6Y54gmhKtmFJEQF2Nra0qVLF06dOgVAYWEhL7/8Mv369aN///689dZbGI1GAOLi4hg7dizh4eEM\nGTKEvXv3AjB27FgSEhIIDw+nqKjolm2EhIRw4MAB8vPzTfdt3bqVTp06mW4XFRXxn//8h379+tGz\nZ09TowFw5MgRhg8fTnh4OAMGDOCXX34BbvxF37lzZz755BMGDx5Mly5d2Lp1621f55kzZwgKCjLd\nDgoK4rvvvjM1cosWLaJbt24MHTqUDz74gJ49ewLw4osvsmTJEtNyf759t1xvvPEGEyZMAODw4cOM\nGDGCPn36MHr0aOLj44EbI1HPPPMMPXr0YMKECVy7du0uFbu9jRs3Mn36dCZNmkRkZCQHDhxg7Nix\nzJgxw/Rjv23bNgYNGkR4eDgTJ07k8uXLACxcuJDZs2czcuRIVq9efdN6Z8yYwapVq0y3T506RefO\nnSktLeW9996jX79+9OvXj4kTJ5KUlFTh3AMGDKCgoIDz588DsH79esLDw+nZsyfPPfccBQUFwI33\n/c0332Tw4MFs27btpjrc6XNZWlrKa6+9Rvfu3Rk5ciRxcXGm7f72228MGzaMAQMG0L9/f7Zt21bh\n7EJUOkUIcUfNmjVTEhMTTbczMzOV8ePHK0uWLFEURVGWL1+uPPbYY0pxcbGSn5+vjBgxQvn6668V\no9Go9O/fX9myZYuiKIpy7NgxJSwsTMnJyVH279+v9O7d+7bb+/LLL5VZs2Ypzz//vGnZnJwcpVev\nXsqGDRuUWbNmKYqiKIsWLVImTZqkFBYWKrm5ucrQoUOVXbt2KYqiKIMGDVK++eYbRVEU5auvvjJt\nKz4+XmnVqpWyZs0aRVEUZevWrUqfPn1um+Opp55SevTooXz88cfK77//ftNjp0+fVkJDQ5Xk5GSl\nuLhYeeKJJ5QePXooiqIos2bNUhYvXmx67p9vl5XL399f2bhxo+n1hoWFKfv27VMURVG2bNmiDBs2\nTFEURVm7dq0yfvx4pbi4WElPT1d69Ohhek/+rKz3+I/3OTg4WLlw4YLp+QEBAcovv/yiKIqiXL16\nVWnbtq1y8eJFRVEU5cMPP1QmTZqkKIqiLFiwQOncubOSlpZ2y3q//fZbZfz48abb8+fPV15//XXl\nzJkzSt++fZWioiJFURTlk08+Ub766qs75vvjfWnZsuUt94eFhSnnzp1TDh48qHTo0EG5du2aoiiK\nMmfOHOWtt95SFOXG+z548GCloKDAdHvx4sVlfi53796t9O3bV7l+/bqSn5+vjBw5UpkwYYKiKIoy\nfPhw5cCBA4qiKMqFCxeU5557rszsQlQFGYER4i4iIiIIDw+nV69e9OrVi/bt2/PYY48BsHv3bkaP\nHo1er8fe3p7Bgwfz888/c+XKFVJTUxk4cCAAAQEB+Pn5ERsbW65tDhw4kG+++QaAnTt30qNHD7Ta\n//+6/vjjj4wbNw5bW1scHR0ZMmQIO3bsAODrr7+mf//+ALRt29Y0egFQUlLC8OHDAfD39ychIeG2\n23/77bcZP348W7ZsYdCgQfTs2ZN169YBN0ZHwsLC8PLyQq/XM2jQoHK9prJyFRcX06dPH9P6fXx8\nTCNOgwYN4vLlyyQkJHDo0CH69OmDXq/Hzc3tpt1sf5WYmEh4ePhN//15rkzDhg1p2LCh6ba9vT0d\nOnQA4Oeff+bBBx+kQYMGAIwaNYoDBw5QUlIC3BiRcnd3v2Wb3bt35+TJk2RmZgLw/fffEx4ejrOz\nM+np6WzZsoWsrCwiIiIYOnRoud63PyiKwvr16/Hx8aFhw4bs2rWLAQMG4OPjA8DDDz9s+gwAdOjQ\nATs7u5vWUdbn8uDBg3Tr1g0nJyfs7e1NtQLw8PDg66+/5ty5czRs2JB33323QtmFMAeZAyPEXfwx\nByY9Pd20+0Ovv/HVSU9Px8XFxfRcFxcX0tLSSE9Px2AwoNFoTI/98SPm6el512126tSJ2bNnk5mZ\nybfffsuTTz5pmlALkJOTw5tvvsm8efOAG7uUAgMDAdiyZQuffPIJubm5lJaWovzpcmc6nc40+Vir\n1VJaWnrb7dvZ2TFlyhSmTJlCdnY227dv54033qBu3bpkZWXdNB/Hw8Pjrq+nPLlq1aoFQHZ2NvHx\n8YSHh5set7W1JT09naysLAwGg+l+Z2dncnNzb7u9u82B+XPd/no7IyPjptdoMBhQFIWMjIzbLvsH\nR0dHOnbsyO7du2nbti3Z2dm0bdsWjUbDwoULWbVqFa+//jphYWG8+uqrd51PZDQaTe+Doig0bdqU\nJUuWoNVqycnJ4fvvv2ffvn2mx4uLi+/4+oAyP5dZWVl4e3vfdP8f3njjDZYuXcrkyZOxt7fnueee\nu6k+QqhBGhghysnd3Z2IiAjefvttli5dCoCnp6fpr22AzMxMPD098fDwICsrC0VRTD8WmZmZ5f6x\nt7GxoUePHnz99ddcunSJNm3a3NTAeHt78+ijj94yApGUlMTs2bPZsGEDLVu25OLFi/Tr169CrzM9\nPZ1Tp06ZRkCcnZ0ZPXo0e/fu5cyZMxgMBnJycm56/h/+2hRlZWVVOJe3tzeNGzdm48aNtzzm7Ox8\nx21XJg8PD44cOWK6nZWVhVarxc3N7a7L9uvXj++//56MjAz69etnqn/79u1p3749eXl5zJ07l3fe\neeeuIxl/ncT7Z97e3gwbNoxZs2ZV6HXd6XNZ1nvr6enJnDlzmDNnDvv27eOpp56iS5cuODk5lXvb\nQlQ22YUkRAVMnjyZI0eO8NtvvwE3dhlERUVhNBrJy8tj06ZNdOvWjbp161K7dm3TJNno6GhSU1MJ\nDAxEr9eTl5dn2h1xJwMHDmTFihW3PXS5V69ebNiwAaPRiKIoLFmyhD179pCeno6joyONGzempKSE\n9evXA9xxlOJ2CgoKePrpp02TOwEuXbpETEwMoaGhtGnThkOHDpGenk5JSQlff/216XleXl6myZ/x\n8fFER0cDVChXUFAQKSkpxMTEmNbzj3/8A0VRCA4OZteuXRiNRtLT09mzZ0+5X1dFdOrUiUOHDpl2\nc33++ed06tTJNPJWlh49enDkyBF27txp2g2zb98+Xn31VUpLS3F0dKRFixY3jYLci549e7Jjxw5T\no7Fz504++OCDMpcp63PZpk0b9u3bR35+Pvn5+abGqbi4mIiICJKTk4Ebux71ev1NuzSFUIOMwAhR\nAbVq1eLxxx9n7ty5REVFERERQXx8PAMHDkSj0RAeHk7//v3RaDTMmzePf//73yxatAgHBwfmz5+P\no6MjzZs3x8XFhU6dOvHVV1/h5+d32221a9cOjUbDgAEDbnls3LhxXLlyhYEDB6IoCq1bt2bSpEk4\nOjrStWtX+vXrh4eHBy+++CLR0dFERESwYMGCcr1GPz8/li5dyoIFC/jPf/6DoijUqlWLl156yXRk\n0pgxYxg2bBhubm707duXs2fPAjB69GimT59O3759adWqlWmUpUWLFuXOZW9vz4IFC3j99dfJzc3F\nxsaGGTNmoNFoGD16NIcOHaJ37974+fnRu3fvm0YN/uyPOTB/FRkZedf3oHbt2vznP//hySefpLi4\nmLp16/L666+X6/2rVasW/v7+nD59muDgYADCwsL49ttv6devH7a2tri7u/PGG28A8MILL5iOJKoI\nf39//v73vxMREUFpaSkeHh68+uqrZS5T1ueyR48e7N69m/DwcDw9PenWrRuHDh3CxsaGkSNH8sgj\njwA3Rtlmz56Ng4NDhfIKUdk0yp93RAshRAUdOnSIF154gV27dqkdRQhRg8gYoBBCCCGsjjQwQggh\nhLA6sgtJCCGEEFZHRmCEEEIIYXWkgRFCCCGE1bHKw6hTUm5/2GRlcHNzJCMjz2zrF/dOamOZpC6W\nS2pjuaQ25ePlZbjjYzIC8xd6vU7tCOIOpDaWSepiuaQ2lktqc/+kgRFCCCGE1ZEGRgghhBBWRxoY\nIYQQQlgdaWCEEEIIYXWkgRFCCCGE1ZEGRgghhBBWRxoYIYQQQlgdaWCEEEKIamb37h/K9bz5898l\nIeHqHR9/8cXnKitSpZMGRgghhKhGEhMT2Lnzu3I9d8aMmfj51bnj42+9Na+yYlU6q7yUgBBCCCFu\nb968uZw6dYIuXcLo27c/iYkJvP/+Et588zVSUpLJz8/n0Ucfp1OnLkyf/jjPPfcCP/74A7m517l8\n+RJXr17h6adn0qFDJwYO7MW33/7A9OmPExb2INHRh8jMzGTu3Pfw9PTktdfmcO1aIgEBgezatZOv\nvtpaZa/TrA1MZGQkhw8fpqSkhL/97W/07dsXgL179zJ16lROnz4NwObNm/n444/RarWMHj2aUaNG\nmTOWEEIIUSW+2PU7B+OSb7lfp9NgNCr3tM6wFt6M7tn0jo8//HAEGzd+QaNGTbh8+SJLlqwkIyOd\ndu3a07//IK5evcKcOS/SqVOXm5ZLTk7inXcWsH//L2za9CUdOnS66XEnJyfmz1/K0qUL2bNnF35+\ndSkqKuSDD1bz8897+eKLdff0eu6V2RqY/fv3c/bsWdavX09GRgbDhg2jb9++FBYW8sEHH+Dl5QVA\nXl4eixcvJioqChsbG0aOHEmfPn1wdXU1V7Q7SstPJynpKj7aOw+nCSGEENaiZUt/AAwGZ06dOsHm\nzRvRaLRkZ2fd8tzAwGAAvL29uX79+i2PBwW1MT2elZXFpUsXCAgIAqBDh07odFV7fSezNTBhYWEE\nBgYC4OzsTH5+PkajkWXLljFu3DjefvttAGJiYggICMBguHHFyZCQEKKjo+nZs6e5ot3R9os/8Evi\nQR4PmEiQV+sq374QQojqZXTPprcdLfHyMpCSkmP27dvY2ADw/ffbyc7OZvHilWRnZzN1asQtz/1z\nA6Iot44O/fVxRVHQam/cp9Fo0Gg0lR2/TGZrYHQ6HY6OjgBERUXRtWtXLl++TFxcHDNmzDA1MKmp\nqbi7u5uWc3d3JyUlpcx1u7k5muVKnsNt+nEo+Shr4zYQUL8pPrW8Kn0b4v6UdWl1oR6pi+WS2lgu\nc9XG3b0WOp0GJyc7atWyx8vLQElJPk2bNsLHx4Xdu7djNJbg5WXA1laPm5vTTc/NyHDC1laPl5cB\njUZz0/O8vAzUqmVPcbEdDzzwAN999x1eXgb27t2L0Wis0s+b2Sfx7ty5k6ioKFatWsXMmTOZPXt2\nmc+/Xdf3VxkZeZUV7yYOODO17cMs+e0TIn9axsy207DR2ZhlW6LiquovFlExUhfLJbWxXOasjYuL\nD7Gxx/Hw8MbGxoGUlBxCQzvx4ovPcfDgYQYOfAhPTy8iI+dRVFRCRkYuubmF2NgUkJKSQ0ZGLkVF\nJaSk5KAoCikpOabnpaTkcP16Abm5hbRuHcq6desZOXI0bdq0xdnZpdJfU1kNkUYpT8dwj/bu3cv8\n+fNZuXIlhYWFjB8/3jTacvLkSYKDg3nqqadYv3498+bdOFTrpZdeom/fvvTo0eOO6zXnF9LLy8B7\ne1bxa+JBOvs9yMMtRphtW6Ji5B9jyyR1sVxSG8tVHWqTnZ1FdPQhunfvRUpKMjNmPMFnn31Zqdso\nq4Ex2whMTk4OkZGRrF692jQhd+fOnabHe/bsydq1aykoKGD27NlkZ2ej0+mIjo7mn//8p7lilcvo\nZkO5nHOFfQkHaOLaiHa1Q1TNI4QQQlgaR0cndu3ayWefrUFRSnnqqao96Z3ZGpitW7eSkZHBM888\nY7pv7ty5+Pn53fQ8e3t7Zs6cyZQpU9BoNEybNs00oVcttjobpraewNyDC1gX9yX1DHXwdfJRNZMQ\nQghhSfR6Pa+99qZq2zfrLiRzMfcupD/WH518jA+Pr6W2ozf/CH0Ke72d2bYr7q46DLlWR1IXyyW1\nsVxSm/IpaxeSXEqgDCHegfSo25lrecl8fnpjuSYYCyGEEML8pIG5i6FNB9DIuT4Hk46wL+GA2nGE\nEEIIgTQwd6XX6nm09Xic9I5EndnE5ZwrakcSQgghajxpYMrB3d6NSf5jKVGMrIxdS15xvtqRhBBC\niPsycuRg8vLyWLNmNcePH7vpsby8PEaOHFzm8rt3/wDA1q1b+OmnH82W806kgSknf48WhDfoSVpB\nOmtPfSHzYYQQQlQLERGP0Lp1YIWWSUxMYOfO7wAYMGAw3brd+dxt5mL2M/FWJwMb9+V81iViUk/w\nQ/weetfvpnYkIYQQ4iaPPjqeN954l9q1a3PtWiIvvTQTLy9v8vPzKSgo4Nln/0GrVv9/vb///vcV\nunfvRXBwG/71rxcoKioyXdgRYMeObURFrUen09KwYRNmzfoX8+bN5dSpE3z00QpKS0txdXVlxIgx\nLFkyn9jYGEpKjIwYMZrw8IFMn/44YWEPEh19iMzMTObOfY/atWvf9+uUBqYCtBotj/iP462D77Pp\n3DYaOTegiWtDtWMJIYSwUBt//4YjybG33K/TajCW3ttIfhvvAIY3HXTHx7t27cHPP+9hxIjR7N37\nE1279qBJkwfo2rU7hw8f5NNPP+a//337luW++24bjRs34emnZ/LDDztMIyz5+fm8++5CDAYD06Y9\nxrlzv/PwwxFs3PgFkyc/xocfLgfg6NFozp8/x9Klq8jPz2fSpLF07dodACcnJ+bPX8rSpQvZs2cX\no0ePu6fX/meyC6mCXOwMPOo/DkVRWHXiU3KKbr3kuBBCCKGWGw3MXgD27fuJzp278dNPP/DEE1NY\nunQhWVlZt13u4sXztG4dBECbNm1N9zs7O/PSSzOZPv1xLl26QFZW5m2Xj4s7SXDwjTPXOzg40LBh\nY+Lj4wEICmoDgLe3N9evV87vpozA3IMH3JrwUONwNp3fxuoT65gWPAWtRnpBIYQQNxvedNBtR0vM\neSK7xo2bkJaWQlLSNXJycti7dzeent7MmfM6cXEnWbTo/dsupyig1WoAKP3f6FBxcTHz5kWyevVn\neHh48sILz9x2WQCNRsOfp4eWlBSb1qfT6f60ncqZQyq/uveod4NutPZoSVzGWbZd/EHtOEIIIYRJ\nhw6d+eCDJXTp0o2srEzq1KkLwE8//UhJScltl6lfvwFxcacAiI4+BEBeXi46nQ4PD0+Skq4RF3eK\nkpIStFotRqPxpuVbtPDnyJHD/1suj6tXr1C3bn1zvURpYO6VVqNlYqsxuNu7se3CTk6ln1E7khBC\nCAFAt2492LnzO7p370V4+EDWr/+UZ5+dhr9/a9LS0vj22823LBMePpATJ2KZMeMJ4uMvodFocHFx\nJSzsQaZOnchHH61g3LgIFiyYR4MGjTh9Oo4FC941LR8UFEzz5i2YNu0xnn12Gn//+3QcHBzM9hrl\nWkh/UdFhvYvZl5l3eCkOenteDJuBm72r2bLVdHLtEMskdbFcUhvLJbUpH7kWkhk1dK7P8AcGcb04\nl1UnPsVYarz7QkIIIYS4L9LAVIJudToS4h3I+axLbDq3Te04QgghRLUnDUwl0Gg0jG8xEm9HT36I\n30NMynG1IwkhhBDVmjQwlcReb8/U1hHYaG1Yc+oLUvPT1I4khBBCVFvSwFSiOrV8GdN8GPklBayM\nXUOxsVjtSEIIIUS1JA1MJevgG0oH3zDirycQdfbWw9SEEEIIcf+kgTGD0c2GUqeWL/sSDvDbtWi1\n4wghhBDVjjQwZmCrs2Fq6wnY6+xYF/cliblJakcSQgghqhVpYMzE29GL8S1HUVRazMrYNRSUFKod\nSQghhKg2pIExoxDvQLrX7cS1vGQ+P72x0i5gJYQQQtR00sCY2bCmA2noXJ+DSUfYl3BA7ThCCCFE\ntSANjJnptXqmtB6Pk96RqDObuJxzRe1IQgghhNWTBqYKuNu7Mcl/LCWKkZWxa8krzlc7khBCCGHV\npIGpIv4eLejXoCdpBemsPfWFzIcRQggh7oM0MFVoYKM+PODamJjUE/wQv0ftOEIIIYTVkgamCum0\nOib7j8fZ1sCmc9s4l3lR7UhCCCGEVTJrAxMZGcmYMWMYMWIEO3bs4MiRIzz88MNEREQwZcoU0tPT\nAdi8eTMjRoxg1KhRbNiwwZyRVOdiZ2Cy/zgURWHViU/JKbqudiQhhBDC6ujNteL9+/dz9uxZ1q9f\nT0ZGBsOGDSMwMJDIyEjq1avHokWL+OKLL5g4cSKLFy8mKioKGxsbRo4cSZ8+fXB1dTVXNNU1c2vC\n4Mb92Hx+O6tPrGNa8BS0GhkME0IIIcrLbL+aYWFhzJ8/HwBnZ2fy8/N57733qFevHoqikJSURO3a\ntYmJiSEgIACDwYC9vT0hISFER1f/6wf1adAdf48WxGWcZdvFH9SOI4QQQlgVszUwOp0OR0dHAKKi\noujatSs6nY49e/YQHh5OamoqDz30EKmpqbi7u5uWc3d3JyUlxVyxLIZWo2VSq7G427ux7cJOTqWf\nUTuSEEIIYTU0ipmP5925cyfLly9n1apVGAwGABRF4Z133sFgMFCnTh1iY2P55z//CcB7772Hn58f\nY8aMueM6S0qM6PU6c8auMr+nXWTOrndwsnEgsu+/cHesvrvOhBBCiMpitjkwAHv37mXZsmWsXLkS\ng8HA999/T58+fdBoNPTr14+FCxfSpk0bUlNTTcskJycTHBxc5nozMvLMltnLy0BKSo7Z1v9XLngw\nvOkgNpzZROSe5TzT5m/otNWjOatsVV0bUT5SF8sltbFcUpvy8fIy3PExs+1CysnJITIykuXLl5sm\n5C5cuJBTp04BEBMTQ6NGjQgKCiI2Npbs7Gxyc3OJjo4mNDTUXLEsUrc6HQnxDuR81kU2nd+mdhwh\nhBDC4pltBGbr1q1kZGTwzDPPmO6bM2cOr776KjqdDnt7eyIjI7G3t2fmzJlMmTIFjUbDtGnTTLua\nagqNRsP4FiO5cj2BHy7voYlLQ4K8WqsdSwghhLBYZp8DYw7mHHZTc1jv6vVE3j60CL1Wx4thM/B0\n8FAlh6WSIVfLJHWxXFIbyyW1KR9VdiGJiqtTy5cxzYeRX1LAytg1FBuL1Y4khBBCWCRpYCxMB99Q\nOviGEX89gaizm9WOI4QQQlgkaWAs0OhmQ6lTy5d9CQf47Vr1P6mfEEIIUVHSwFggW50NU1tPwF5n\nx7q4L0nMTVI7khBCCGFRpIGxUN6OXoxvOYqi0mJWxq6hoKRQ7UhCCCGExZAGxoKFeAfSvW4nruUl\n8/npjVjhAWNCCCGEWUgDY+GGNR1IQ+f6HEw6wr6EA2rHEUIIISyCNDAWTq/VM6X1eJz0jkSd2cTl\nnCtqRxJCCCFUJw2MFXC3d2OS/1hKFCMrY9eSV5yvdiQhhBBCVdLAWAl/jxb0a9CTtIJ01p76QubD\nCCGEqNGkgbEiAxv14QHXxsSknuCH+D1qxxFCCCFUIw2MFdFpdUz2H4+zrYFN57ZxLvOi2pGEEEII\nVUgDY2Vc7AxM9h+HoiisOvEpOUXX1Y4khBBCVDlpYKxQM7cmDG7cj8zCLFafWEepUqp2JCGEEKJK\nSQNjpfo06I6/RwviMs6y7eIPascRQgghqpQ0MFZKq9EysdUY3Oxc2XZhJ6fSz6gdSQghhKgy0sBY\nsVo2TkwNmIBWo2X1iXVkFmapHUkIIYSoEtLAWLmGzvUZ3nQQ14tz+fD4pxhLjWpHEkIIIcxOGphq\noFvdjrTxDuR81kU2nd+mdhwhhBDC7KSBqQY0Gg3jW4zE28GTHy7vISblhNqRhBBCCLOSBqaacNDb\nMzUgAhutnjWn1pOan6Z2JCGEEMJspIGpRurU8mVMs2HklxSw8vhaio3FakcSQgghzEIamGqmg18Y\n7X1Dic+5StTvW9SOI4QQQpiFNDDV0JhmQ/Fzqs2+q/vZe/VXteMIIYQQlU4amGrIVmfLYwETqWXj\nxPrTX3Mo6ajakYQQQohKJQ1MNeXt6Mn04KnY6ez4+OTnHE89pXYkIYQQotJIA1ON1TPU4Ymgyeg0\nOlYeX8PZjHNqRxJCCCEqhTQw1VxT10Y8FjCRUkVh2bHVXMqOVzuSEEIIcd/M2sBERkYyZswYRowY\nwY4dO0hMTOSRRx5hwoQJPPLII6SkpACwefNmRowYwahRo9iwYYM5I9VI/h7NecT/YQqNRSyO+ZDE\n3CS1IwkhhBD3RW+uFe/fv5+zZ8+yfv16MjIyGDZsGA8++CCjR49mwIABfPrpp3z00UdMnz6dxYsX\nExUVhY2NDSNHjqRPnz64urqaK1qNFOIdSEGLAj6Ni2LhkRU81/ZJPB3c1Y4lhBBC3BOzjcCEhYUx\nf/58AJydncnPz+ff//43/fr1A8DNzY3MzExiYmIICAjAYDBgb29PSEgI0dHR5opVo3X0a8fwpoPI\nKspm4dEVZBVmqx1JCCGEuCcjqS9uAAAgAElEQVRma2B0Oh2Ojo4AREVF0bVrVxwdHdHpdBiNRj77\n7DMGDx5Mamoq7u7/PxLg7u5u2rUkKl+v+l3p37AXqflpLDq6ktziPLUjCSGEEBVmtl1If9i5cydR\nUVGsWrUKAKPRyAsvvED79u3p0KEDW7bcfLZYRVHuuk43N0f0ep1Z8gJ4eRnMtm5L8IjnCBQbI9vP\n7uaDE6uZ030GDjb2ascql+peG2sldbFcUhvLJbW5P2ZtYPbu3cuyZctYuXIlBsONQr300ks0aNCA\n6dOnA+Dt7U1qaqppmeTkZIKDg8tcb0aG+UYNvLwMpKTkmG39lmJg3XAycnI4cO0w//1xMU8GTsZG\nZ6N2rDLVlNpYG6mL5ZLaWC6pTfmU1eSZbRdSTk4OkZGRLF++3DQhd/PmzdjY2PD000+bnhcUFERs\nbCzZ2dnk5uYSHR1NaGiouWKJ/9FqtIxvMZIgT3/OZPzOqhOfYSw1qh1LCCGEKBezjcBs3bqVjIwM\nnnnmGdN9CQkJODs7ExERAUCTJk145ZVXmDlzJlOmTEGj0TBt2jTTaI0wL51Wx2T/cSw99hHHUk+w\n5tQGJrYajVYjpwcSQghh2TRKeSadWBhzDrvVxGG9gpJCFh5dwcXsy3Sr25FRDwxBo9GoHesWNbE2\n1kDqYrmkNpZLalM+quxCEtbDXm/Hk0GP4udUm5+u/MI3F3aoHUkIIYQokzQwAgAnG0emBz+Gp4MH\n2y/+wM7LP6kdSQghhLgjaWCEiYudgaeDH8PVzoWvfv+Wn68eUDuSEEIIcVvSwIibeDi481TwVJxs\nHFl3eiOHk46qHUkIIYS4hTQw4ha1nXyYHjQVO50tq09+zom0OLUjCSGEEDeRBkbcVn3nuvw9cDI6\njZYVsWv4PfOC2pGEEEIIE2lgxB094NaYqa0jMCpGlsZ8xOWcK2pHEkIIIQBpYMRdtPZsySOtxlJo\nLGTx0Q+5lpusdiQhhBBCGhhxd219ghnbfBjXi3NZeHQFafnpakcSQghRw0kDI8qlc532DG0ygMzC\nLBYeXUFWoZxBUgghhHqkgRHl1qdBd/o16ElKfhqLjq4gr9h8VwUXQgghyiINjKiQwY370bVOBxJy\nr7Ek5iMKSgrVjiSEEKIGkgZGVIhGo2FUsyGE+YRwIfsSK2I/obi0RO1YQgghahhpYESFaTVaIlqO\nIsCzJXEZZ/noxGcYS41qxxJCCFGDSAMj7olOq2OK/wSauTYhJuU4n8ZFUaqUqh1LCCFEDSENjLhn\nNjob/hY4iQaGehy4dpios1tQFEXtWEIIIWoAaWDEfbHX2/Nk8KP4Ovnw05Wf+fbC92pHEkIIUQNI\nAyPuWy0bJ6YHT8XT3p1tF3ey6/IetSMJIYSo5qSBEZXC1c6Fp9o8joutM1/+/g2/JBxUO5IQQohq\nTBoYUWk8HdyZHjwVJ70jn8VFEZ18TO1IQgghqilpYESl8qtVm2nBU7DV2bD6xDpOpp1WO5IQQohq\nSBoYUekaONfj74GT0Wg0fBD7Cb9nXlA7khBCiGpGGhhhFs3cmjC19QSMipFlxz4iPidB7UhCCCGq\nEWlghNkEeLZiUssxFJQUsujoCpLyUtSOJIQQopqQBkaYVWjtNoxpPpTrxbksPLKC9IIMtSMJIYSo\nBqSBEWbXpU4HhjTuT0ZhJguPriC7KEftSEIIIaycNDCiSvRt2IM+9buTnJfKoqMrySvOVzuSEEII\nKyYNjKgyQ5r0p7Pfg1y9nsjSY6soNBapHUkIIYSVMmsDExkZyZgxYxgxYgQ7duwA4JNPPsHf35/c\n3FzT8zZv3syIESMYNWoUGzZsMGckoSKNRsOY5sMI9QnmfNYlVsR+QnFpidqxhBBCWCG9uVa8f/9+\nzp49y/r168nIyGDYsGHk5eWRlpaGt7e36Xl5eXksXryYqKgobGxsGDlyJH369MHV1dVc0YSKtBot\nE1uOoaCkgONpcaw+sY5H/ceh0+rUjiaEEMKKmG0EJiwsjPnz5wPg7OxMfn4+vXr14tlnn0Wj0Zie\nFxMTQ0BAAAaDAXt7e0JCQoiOjjZXLGEBdFodU1pH8IBrY46mxPLZ6S8pVUrVjiWEEMKKmK2B0el0\nODo6AhAVFUXXrl0xGAy3PC81NRV3d3fTbXd3d1JS5Hwh1Z2tzoa/BT5CfUNd9iceYuPv36Aoitqx\nhBBCWAmz7UL6w86dO4mKimLVqlXlen55fsTc3BzR6823y8HL69ZGS5iDgZfdZ/DvXe/yY/w+vFxc\nGek/sMwlpDaWSepiuaQ2lktqc3/M2sDs3buXZcuWsXLlytuOvgB4e3uTmppqup2cnExwcHCZ683I\nyKvUnH/m5WUgJUXOU1KVngyYwrzDS/ji+DeUFmrpUa/zbZ8ntbFMUhfLJbWxXFKb8imryTPbLqSc\nnBwiIyNZvnx5mRNyg4KCiI2NJTs7m9zcXKKjowkNDTVXLGGBXO1ceCr4cZxtDUSd3cyviYfUjiSE\nEMLCmW0EZuvWrWRkZPDMM8+Y7nvwwQc5cOAAKSkpPPbYYwQHB/PCCy8wc+ZMpkyZgkajYdq0aXcc\nrRHVl5ejB08FP8Z70Uv59NQGHHR2BHsHqB1LCCGEhdIoVjhz0pzDbjKsp64LWZdZcPQDSkuN/D1o\nMi3dm5kek9pYJqmL5ZLaWC6pTfmosgtJiHvRyKU+fw94BDQaPjj2MeezLqkdSQghhAWSBkZYnObu\nTXnUfzwlipElMau4kpOgdiQhhBAWRhoYYZGCvPyJaDma/JJ8Fh1dSXKenBtICCHE/5MG5k8Ki4yk\nZclVki1Fu9ohjG42lJzi6yw4soLUvHS1IwkhhLAQ0sD8yfpdZ3nsjZ2cT8hWO4r4n251OzK4cTgZ\nhZn88/u5nMn4Xe1IQgghLIA0MH/StoU3JcZSFn8VS3ZukdpxxP/0a9CD4U0HkVN4YyTm2wvfy7WT\nhBCihpMG5k/8G7oT0b8lGTmFLNt0HGOp/EhaAo1GQ6/6XXm150zc7F3ZeuF7Fh5ZQVahjJQJIURN\nJQ3MX4zs+QAhzbyIu5zJhh/PqR1H/Ekzz8a8FDaDQE9/zmSe483f3udU+hm1YwkhhFCBNDB/odFo\nmDKwJb4ejuw4GM/+k9fUjiT+xNHGkccDJjLygYfIK8ln8dEP2XJuO8ZSo9rRhBBCVCFpYG7DwU7P\n9OEB2NvqWL01jvjk62pHEn+i0WjoUa8zM9s+ibu9G9sv7WL+kQ/IKMhUO5oQQogqIg3MHfh6ODF1\nUCuKSkpZtPEYuQXFakcSf9HAuR4vtZtBG68AzmVd4M2D73MiLU7tWEIIIaqANDBlCGnmxaCODUjJ\nLOCDzScpLbW6y0ZVew56B6a0nsCYZkMpLClkScwqvv59q+xSEkKIak4amLsY2rkxrRu7E3s+ja/3\nXVA7jrgNjUZD17odeT50Ol4OHnx/eTfvRS8jvSBD7WhCCCHMRBqYu9BqNTw+2B8vV3u++eUiR87I\nKe0tVT1DHWaFzaCtdxAXsi/x5m/vcyzlhNqxhBBCmIE0MOVQy8GGacMCsNVrWfHNSRLTctWOJO7A\nQW/PZP9xjGs+guLSYpbHfsyXZ7dQUlqidjQhhBCVSBqYcqrvY+CR/i0oKDKyaGMs+YXyg2ipNBoN\nneo8yD9Cn8LH0Ytd8XuZF72U1Hy5lpIQQlQX99zAXLx4sRJjWIf2/rXpE1qPxLQ8Vn17CkWRSb2W\nrE4tX14IfZp2tUO4lB3PWwff52hyrNqxhBBCVIIyG5jJkyffdHvJkiWm/3/55ZfNk8jCjerRhOb1\nXDl8JoWt+y+pHUfchb3ejoktxzChxShKSo2sOL6GL858TbHsUhJCCKtWZgNTUnLzP/L79+83/X9N\nHX3Q67Q8MbQ1bgY7Nu45z/ELaWpHEneh0Wjo4BfGrLCnqe3kw09XfuHdw4tJzktVO5oQQoh7VGYD\no9Fobrr956blr4/VJM5OtkwbFoBOq2H5phOkZOarHUmUg6+TD7NCn6KDbxjxOVeZe3A+h5Ni1I4l\nhBDiHlRoDkxNblr+qrGfMxP6Nie3oITFG2MpLJYTp1kDW50tE1qOYlKrsZSisOrEp6w7vZEio5xp\nWQghrIm+rAezsrL49ddfTbezs7PZv38/iqKQnZ1t9nCWrmuQHxcSs/npaAKfbI9j6qBW0uRZiXa1\nQ6hvqMuHx9ey7+p+LmRdYor/eHycvNWOJoQQohw0ShmTWSIiIspceM2aNZUeqDxSUnLMtm4vL0OF\n1l9cUsrcz6I5n5DNuN4P0Du0ntmy1XQVrU15FBmL+fLsZvYlHMBWZ8vDzYfTrnZIpW6jujNHXUTl\nkNpYLqlN+Xh5Ge74WJkNjKWypAYGID27gNdWHyS3oIR/PNyGZvVczZSuZjPnF/5Q0lHWxX1JgbGQ\njr5hjGo2BFudrVm2Vd3IP8SWS2pjuaQ25VNWA1PmHJjr16+zevVq0+3PP/+cIUOG8PTTT5OaKkdw\n/MHd2Z4nhrYGYMlXsWTkFKqcSFRUqE8ws8Kepl4tP35JPEjkoYUk5iapHUsIIcQdlNnAvPzyy6Sl\n3ThM+MKFC8ybN49Zs2bRsWNH/vvf/1ZJQGvRvL4bo3s2JTuvmCVfxVJcUqp2JFFB3o5ezGw7ja51\nOpKYm0TkwQX8mnhI7VhCCCFuo8wGJj4+npkzZwLw3XffER4eTseOHRk7dqyMwNxG77Z1ae/vw7mE\nbNbtPKN2HHEPbHQ2jGk+lKmtI9Bpdaw99QWfnFxPQYmMqgkhhCUps4FxdHQ0/f9vv/1G+/btTbfl\naJtbaTQaJoW3oL53LXYfTWBPTILakcQ9auMdwIthM2hgqMeBa4eJPLSQq9cT1Y4lhBDif8psYIxG\nI2lpaVy+fJkjR47QqVMnAHJzc8nPv/vJ2yIjIxkzZgwjRoxgx44dJCYmEhERwbhx45gxYwZFRUUA\nbN68mREjRjBq1Cg2bNhQCS9LPXY2OqYND8DJXs/aHac5nyCHm1srTwcPnmv7BD3qdSYpL5m3Dy3k\n54QDNfYs1EIIYUnKbGAee+wxBgwYwODBg3nyySdxcXGhoKCAcePGMXTo0DJXvH//fs6ePcv69etZ\nuXIlb7zxBgsWLGDcuHF89tlnNGjQgKioKPLy8li8eDGrV69mzZo1fPzxx2RmZlbqi6xqXq4O/O0h\nf4xGhcVfxZKdW6R2JHGP9Fo9Ix94iMcDJqHX2vBZ3JesPrmOgpICtaMJIUSNpnvllVdeudODDRs2\n5JFHHmHSpEl06NABAL1eT7169Rg0aFCZK/b19aVPnz7Y2Nhga2vL8uXLSU5O5uWXX0an02Fvb8+W\nLVvw9vYmLS2NwYMHo9friYuLw87OjkaNGt1x3Xl55msInJzsKmX93m6O6HUaos+kcvFaNu39fdDK\nbrf7Ulm1uRe1nbxp6x3MxezLnEw/zdHkWJq4NsLF7s6H+NUUatZFlE1qY7mkNuXj5GR3x8fKHIFJ\nSEggJSWF7OxsEhISTP81btyYhISy53fodDrTHJqoqCi6du1Kfn4+trY3zq3h4eFBSkoKqampuLu7\nm5Zzd3cnJSWl3C/Okg1o34CQZl7EXc5kw4/n1I4j7pOHgxvPhjxB7/rdSM5P5Z3Di9hz5VfZpSSE\nECoo81ICPXv2pFGjRnh5eQG3Xszxk08+uesGdu7cSVRUFKtWraJv376m++/0j355fgzc3BzR63V3\nfd69KuvEORU1a1IYM+fvYcfBeAKbedMtpG6lrbsmqsza3KvHfcYS2sCfxQc+Zv2Zr7iUf4m/h07A\n0dZB7WiqsYS6iNuT2lguqc39KbOBmTt3Lps2bSI3N5eBAwcyaNCgm0ZL7mbv3r0sW7aMlStXYjAY\ncHR0pKCgAHt7e5KSkvD29sbb2/umQ7KTk5MJDg4uc70ZGXnlzlBR5jg74hND/Hn940MsWH8Eg52O\net61KnX9NYUlnbmynk1DZoXO4KMTn7E/PprfUy7yaOvxNHCueZeSsKS6iJtJbSyX1KZ87vlMvEOG\nDGHVqlW8//77XL9+nfHjxzN16lS2bNlCQUHZkxhzcnKIjIxk+fLluLreOLV+x44d+e677wDYsWMH\nXbp0ISgoiNjYWLKzs8nNzSU6OprQ0NCKvkaL5uvhxNRBrSgqKWXRxmPkFsiVj6sDN3tXZrT5G30b\n9CC1IJ13Dy/hx/h9sktJCCGqQIWvhbRhwwbeeecdjEYjhw7d+Syl69evZ+HChTdNxn3rrbeYPXs2\nhYWF+Pn58eabb2JjY8P27dv58MMP0Wg0TJgwgYceeqjMDJZ2LaTy2rjnHN/8comAxh7MGBmIViuT\neivCkv9iOZl2mo9Pfs714lyCvFozocVIHG0c775gNWDJdanppDaWS2pTPvd9Mcfs7Gw2b97Mxo0b\nMRqNDBkyhEGDBuHt7V2pQcvLWhuY0lKF96NiOH4+nUEdGzK8a2OzbKe6svQvfGZhFqtPrONs5nnc\n7d141H88jVzqqx3L7Cy9LjWZ1MZySW3K554bmH379vHll19y/Phx+vbty5AhQ2jWrJlZQlaEtTYw\nANfzi3n944OkZBbw1PAA2jTzMtu2qhtr+MKXKqVsvbCT7Rd/QKPRMKRJf3rV61qtz1xtDXWpqaQ2\nlktqUz733MC0aNGChg0bEhQUhFZ763SZN998s3ISVpA1NzAAl5NyeGPNYbRaDXMmheLr4WTW7VUX\n1vSFj0s/y+qT68gpuk5rj5ZEtBpNLZvqWWdrqktNI7WxXFKb8rnnBua3334DICMjAzc3t5seu3Ll\nCsOHD6+kiBVj7Q0MwP6T1/hg80l8PRyZPTEUB7syDwgTWN8XPqswh49PruN0xu+42bky2X8cTVwb\nqh2r0llbXWoSqY3lktqUzz0fhaTVapk5cyZz5szh5ZdfxsfHh3bt2nHmzBnef//9Sg9ak7RvVZu+\nYfVITMtj1ben5MiVasjFzsD04KkMatSPzMIs3j+yjB0Xf6RUKVU7mhBCWL0y/+x/7733WL16NU2a\nNOGHH37g5ZdfprS0FBcXF6u/6KIlGNm9CZeu5XD4TApb919iYIeGakcSlUyr0dK/US+aujbkoxPr\n2HR+G2czzzOx1RgMtnI+ICGEuFd3HYFp0qQJAL169eLq1atMnDiRRYsW4ePjUyUBqzO9TssTQ1vj\nZrBj457zHL+QpnYkYSYPuDXhpXbP0Mq9OSfTT/OfA+/yY/w+iktL1I4mhBBWqcwG5q9HTvxxgUZR\neZydbJk2LACdVsPyTSdIycxXO5IwE4NtLZ4ImsywpgMpKS0h6uxmXv01kl8TDmIsNaodTwghrEqZ\nDcxfVedDQdXU2M+ZCX2bk1tQwuKNsRQWy49ZdaXVaOldvxuvdniRXvW6klN8nbVxG/jvb/OITj4m\n82OEEKKcyjwKKSAgAA8PD9PttLQ0PDw8UBQFjUbD7t27qyLjLarDUUi38/H2OH46mkAHfx+mDmol\nDeNfVMdZ+xkFmWy/+AO/JB6kVCmlXi0/BjcJp5V7c6upf3WsS3UhtbFcUpvyKesopDIn8W7fvr3S\nw4g7G9e7GfHJ1/n1RBINfZ3pE1rzLgxY07jZu/JwixH0qt+NrRe+51DSUZbErKKJS0MeatKfpq6N\n7r4SIYSogSp8LSRLUF1HYADSswt4bfVBcgtKeH5sMM3ru919oRpC7dpUhavXE/nm/A6OpZ4AoJV7\ncwY36Ud9Q12Vk91ZTaiLtZLaWC6pTfnc83lgRNVzd7bniaGtAVj69XEycgpVTiSqUp1avvwtcBLP\nt51GM9cmnEw/zdyDC1gZu4ZruclqxxNCCIshDYwFal7fjdE9m5KdV8zir2IpLpGJnTVNI5cGzAj5\nG08FP0YD53ocSYnlPwfeZc2pL0jLz1A7nhBCqE7OX2+herety4XEbPafSGLdzjNMDG+hdiShghbu\nD9DcrSnHUk+y5fx29ice4uC1I3Su055+DXriYnfn4VUhhKjOpIGxUBqNhknhLUhIyWX30QQa+jrT\nNchP7VhCBRqNhiAvfwI8W3Io6Sjfnt/BT1d+5teE3+herzN96nfD0cZR7ZhCCFGlZBeSBbOz0TFt\neABO9nrW7jjN+YRstSMJFWk1WtrVDmFO++cZ23wYDnp7dlz6kZd/nct3F3dRaCxSO6IQQlQZaWAs\nnJerA397yB+jUWHxV7Fk58qPVE2n1+rpUqcDr3R4kWFNB6JFw+bz2/n3L2+xO/5nuTyBEKJG0L3y\nyiuvqB2iovLyzPcj7uRkZ9b13wtvN0f0Og3RZ1K5eC2b9v4+aK3kJGeVyRJroyadVkdjl4Z0rvMg\neo2e37POcyz1JAcSD+Ogt8fPqTZajfn/RpG6WC6pjeWS2pSPk5PdHR+TBuYvLPVD9UBdF66k5BJ7\nPp3CIiOtG3vcfaFqxlJrozYbrQ3N3JrQ0a8dpUopZzPPczTlONHJxzDYGvBx9DLrWX2lLpZLamO5\npDblU1YDI7uQrIRGo2HKwJb4ejiy42A8+09eUzuSsDAG21qMeGAwr7R/gU5+D5KSn8aHx9cSeXAB\nJ9LisMJzVgohxB1JA2NFHOz0TB8egL2tjtVb44hPvq52JGGB3OxdGddiBHMefJ5Qn2CuXE9kScwq\n3otexu+ZF9SOJ4QQlUIaGCvj6+HE1EGtKCopZdHGY1zPL1Y7krBQ3o6eTPYfx0vtniHAsxXnsi7w\nXvRSFsd8yOWcK2rHE0KI+yJzYP7CGvZL+no4YSwt5ejZNOJTrvNgSx+ruXLx/bCG2lgiZ1sDoT7B\ntHRvRmp+GqczfufnhAMk5ibh51SbWrZO97V+qYvlktpYLqlN+cgk3gqwlg9V83punE/M5vj5dEoV\naNmg+l/00VpqY6nc7F15sHZbmrg2Iik3hbiMs+y9+itpBRnUreWHo43DPa1X6mK5pDaWS2pTPjKJ\ntxrSajU8PtgfL1d7vvnlItFnUtSOJKyARqOhhfsD/CN0Oo8HTKS2kzf7Ew/x2v5INpzZRHaRXB1X\nCGEdpIGxYrUcbJg2LABbvZaV35wkMS1X7UjCSty4PEFr/tnuWSa2HIOLnQu7r/zMv395i83ntpNX\nnKd2RCGEKJM0MFauvo+BRwa0oKDIyKKNseQXyllYRflpNVoe9G3Ly+2fZ0yzG5cn+O7SLrk8gRDC\n4skcmL+wxv2Sdb1qkV9YQszvaSSl5xHWwrtaTuq1xtpYC61GSwPnenSp0wEHvT0Xsi5xPO0UvyT+\nhl6jp47BD90dzuordbFcUhvLJbUpH5kDUwOM7N6E5vVcOXwmha37L6kdR1gpW50tfRp059WOs+jf\nsDdFxiI2nN3Ea/vf5tfEQxhLjWpHFEIIwMwjMGfOnGHMmDFotVoCAwM5d+4cTz31FF999RXR0dF0\n7doVrVbL5s2b+ec//0lUVBQajQZ/f/8y1ysjMLfSajUENvHgwKkkjpxNpUkdZ7zdHNWOVamstTbW\n6K+XJziTeY6jKbFEJ8disK110+UJpC6WS2pjuaQ25aPKCExeXh6vv/46HTp0MN33zjvv8Pjjj7N2\n7Vp8fX3Ztm0beXl5LF68mNWrV7NmzRo+/vhjMjMzzRWrWnN2smXasAB0Wg3LN50gJTNf7UjCyt18\neYJ2pOSn3rg8waGFnEg7LZcnEEKoxmwNjK2tLStWrMDb29t036VLlwgMDASgS5cu/Pzzz8TExBAQ\nEIDBYMDe3p6QkBCio6PNFavaa+znzIS+zcktKGHRxlgKi2TIX9y/G5cnGMmcB2cS6hNMfM5VlsR8\nyHvRyziVclYaGSFEldObbcV6PXr9zatv1qwZP/30E0OHDmXv3r2kpqaSmpqKu7u76Tnu7u6kpJR9\nThM3N0f0ep1ZcgN4eRnMtu6qMKJ3cxIz8vlu/yUWfX2cl6c8iKO9jdqxKoW118baeWHAv8HfuJhx\nhc+PbyY6IZZ/75pHHUNtOtQPoWO9UOq6+KodU/yJfGcsl9Tm/pitgbmdWbNm8corr7Bx40batWt3\n27/ayvOXXEaG+c5R4eVlICXF+k/mNaJLI9Iy8zkUl8yLi/by7OhgajlYdxNTXWpTHTjhwpQWEfTw\nvcS+pF+ITogl6sRWok5sxdfJh7beQYR4B+Lj5H33lQmzke+M5ZLalE9ZTV6VNjC+vr4sX74cgL17\n95KcnIy3tzepqamm5yQnJxMcHFyVsaolvU7L3x/yZ7WNjn2xicz9LJrnxwTjUuvOE6KEqKjGLg14\nsGlr4hNTOZ56kujkY5xIP803F3bwzYUd1KnlS8j/mhlvR0+14wohqpEqPYx6wYIF7N69G4CNGzfS\ns2dPgoKCiI2NJTs7m9zcXKKjowkNDa3KWNWWVqvhkQEt6N22LldTcnnz02hSs2Rir6h89no7Qmu3\n4fHASbzV+WUmtRpLa4+WXMtNZsv57by6P5K3Ds7n+0u7Sc1PVzuuEKIa0Chmmn13/Phx5s6dy9Wr\nV9Hr9fj4+PD888/z+uuvoygKoaGhvPTSSwBs376dDz/8EI1Gw4QJE3jooYfKXLc5h92q47Ceoih8\ntfc83/xyCXdnO54f24ba7tZ3iHV1rE11UFZd8orziUk9QXRyDHHpZylVSgFo4FyPEO9AQrwDcbev\n/hciVYt8ZyyX1KZ8ytqFZLYGxpykgbk3W/dfImr3OZydbJk5Jph63rXUjlQh1bk21qy8dblenMux\nlBMcTorhTOY5UzPTyLkBIT43mhlXOxdzx61R5DtjuaQ25SMNTAVU9w/VrugrrN1xBid7Pc+ODqax\nn7PakcqtutfGWt1LXXKKrnM05TjRycc4m3EOhRv/DDVxaUiITxBtvAJwsbOez6alku+M5ZLalI80\nMBVQEz5UvxxP5MNvT2Fro+OZkYE0r28dQ/g1oTbW6H7rklWYQ0xKLIeTYziXeREFBQ0amro2IsQ7\niDbeARhsrWu00FLId8ZySW3KRxqYCqgpH6rDp5NZtukEWq2GacNaE9jE8o8QqSm1sTaVWZfMwiyO\nJh/ncHIM57MuAqBBQ9AFYHoAACAASURBVDO3JrT1DiLIqzW1bJ0qZVs1gXxnLJfUpnykgamAmvSh\nOn4+jUUbYzGWKjz+kD9hLSz7nB01qTbWxFx1ySjI5EjyMaKTj3Eh+zJw46rZzd2aEuIdRJCXP042\n1jcZvSrJd8ZySW3KRxqYCqhpH6oz8Zm8vyGGwmIjk/u3pHOg5Z5FtabVxlpURV3S8jM4knKMw0kx\nXM65AoBOo6OF+wO09Q4i0KsVDnoHs2awRvKdsVxSm/KRBqYCauKH6kJiNvPWHyW3oITxfZrRq21d\ntSPdVk2sjTWo6rqk5qcRnXSM6OQY4q8nAKDX6Gjp0ZwQ70ACPVthr7evsjyWTL4zlktqUz7SwFRA\nTf1QXUm5zrufHyUrt4gR3RozsENDtSPdoqbWxtKpWZekvBSOJN8YmUnIvQaAXqvH36MFId6BtPZo\nib2+5p59Wr4zlktqUz7SwFRATf5QJWXk8c66I6RlFzKgfQNGdGuMRqNRO5ZJTa6NJbOUulzLTeJw\n8v+1d+/xTdX3/8BfJzm539o0TWkLvUPLHQpVQECcoF9104lyEcvGvvuq+yKb2xe3IdOBPza3epsP\n7wpuczhGFZ2X6UDdQHGCFCoFKqUUCtLSNr2k17RNc/n9kTRtacEWSHPSvp4P+0hyzknyqe+T8srn\nfM75HEJ+VQEqHTYAgEKmwISoDGTGTMaEqAwo5coQt3JwSaU21Btr0z8MMAMw3HequsY2PLb1IKrq\nHPhWZjyWLRgDmURCzHCvjVRJsS5nmyuRbyvAAVsBbA7fXGtKuRITo8YiM2YyxpvToZCH9+Sm/SHF\n2pAPa9M/DDADwJ0KaGhx4omtB1FW3YyrJozAihszIJcN6rRZfWJtpEnKdfF6vShvrkC+7RAO2ApQ\n01oLAFDLVZhoGYdpMZORYR4DhWxQ57UdNFKuzXDH2vQPA8wAcKfyaW7twFNvFODk2UZMS4/G3d8Z\nD4UY2hDD2khTuNTF6/XiTHN5YABwbZsdAKAR1ZhkGY9M6yRkmEdDHEJhJlxqMxyxNv3DADMA3Km6\ntLa78Mybh1D0dT0mpJhx760ToVLIQ9Ye1kaawrEuXq8Xp5vO+MPMIdjb6wEAWlGDDPNopEWkIC0i\nGbG6GMiE0Pc+XqxwrM1wwdr0DwPMAHCn6snZ4cbzbx/BoRO1GDMqAvfdPgkaVWi+obI20hTudfF4\nPTjVeAb5VQX4svow6tsbAuu0ogapEclIi0jG6IgUjNTHQS4LXYgfqHCvzVDG2vQPA8wAcKfqzeX2\n4OX3vsL+IhuSYw342eIp0GsGfwAkayNNQ6kuXq8X1a01KKkv9f+cDBxqAnwDgVOMiYEemiTjKEkP\nBh5KtRlqWJv+YYAZAO5UffN4vPjz9iJ8dqgC8dE6rF4yBRH6wb2+BmsjTUO9Lva2+kCYKakvDZyi\nDfguoJdoTMDoiGSkRaQg2ZQgqYvoDfXahDPWpn8YYAaAO9X5ebxebP34OD4+UAZrpAb3L50Ci2nw\nLt/O2kjTcKtLk7MZJ7r10JQ1V8AL359RmSDDSH0c0vyBJjUiCXpF6CafHG61CSesTf8wwAwAd6oL\n83q9+Pvuk/jH56dhNqpw/9KpGGEenAn1WBtpGu51aXW14mTD6UCgOd1YBrfXHVgfpxvhDzTJSI1I\nRoTKNGhtG+61kTLWpn8YYAaAO1X//HPvabyx6wSMOiVWL5mCUVZ90N+TtZEm1qUnp9uJU41f47i/\nl6a04TQ6PB2B9RZNVKCHZnREMqLU5qBd8Zq1kS7Wpn8YYAaAO1X/7cwvw+YPi6FTi/jp4slIjQvu\nN0vWRppYlwtzeVw401Qe6KE50XAKra62wPoIlSnQQ5MWkYIRWutlCzSsjXSxNv3DADMA3KkG5vMj\nFXjl/aNQKuS477ZJyEiMDNp7sTbSxLoMjMfrQXlzZWBQcEn9STR3tATW6xU6pJqSAoEmXh970adu\nszbSxdr0DwPMAHCnGrgDx2x48Z1CyGQC7r11AialWoLyPqyNNLEul8br9cLmqEZJfan/sNPJwIX1\nAN+0ByndAk2CcWS/pz5gbaSLtekfBpgB4E51cY6crMWzbx2G2+PF3TePR1aG9bK/B2sjTazL5Vfb\nag/00JxoKEWVozqwTiETkWRMCASaZFMiVOeZZZu1kS7Wpn8YYAaAO9XFKz5Tj6feKEB7hxsrbsjA\nnElxl/X1WRtpYl2Cr9HZ1OPiemebK3ucup1gGNl1ppMpCVqF78xA1ka6WJv+YYAZAO5Ul6a0ohFP\n5h5ES5sLy+aPxvzpoy7ba7M20sS6DD5HhwMnGk7hRP0p36nbTWXweD0AAAEC4vS+U7cnxo+Bxq2H\nVWMJhBqSBn5u+ocBZgC4U126supmPLH1IBpanFg4NwXfnpV0WV6XtZEm1iX02t1OlHa7Fs2pxq/R\n4XH12Ean0MKqsSBaa+l1q5HQ1YOHC35u+ocBZgC4U10eVXYHHv/bl6htbMeNMxJx29Upl3xqKGsj\nTayL9HR4XPi6sQz1qEWprRy21hpUO2pQ01YX6KnpzqDQ9ww2WguiNRZEa6KgFgd3ypDhgp+b/rlQ\ngAnNtMI05MVEavFA9jQ8tvUgPth7Gq1OF+5cMAayIF2wi4i6KGQiUiOSEB09EdURXf9Iuj1u1LbZ\nUd1aA5ujpuvWUYPShtM42XCq12uZlIY+e22iNRYoJTyRJQ19DDAUNGajGmvuzMQTWw9iZ3452p1u\n/ODGDMhlslA3jWhYksvksPp7WMZH9Vzn8rhQ21oX6K2xtdb6b2v8Y21Ke71ehMrUq9fGqrXAoonq\n96neRBcrqHtYcXExVq5ciRUrViA7Oxt5eXl48sknIYoitFotHn30UZhMJmzatAnbt2+HIAhYtWoV\nrr766mA2iwaRSafEL5ZNxVNvFODzI5Vod7px983joRAZYoikRJSJiNFZEaPrfQmEDncHatrqevXa\n2FprUFx/AsX1J3psL0BApDqiW69NVKD3JkpjhshwQ5dB0MbAOBwO3HPPPUhKSkJ6ejqys7OxcOFC\nPP7440hJScGLL74ImUyGG264Affddx+2bt2K5uZmLFu2DO+//z7k8vNfeZJjYMJPa7sLz7x5CEVf\n12NCihn33joRKsXAri7K2kgT6yJdg1Ebp9uJ6m69Nd1vG5y931smyGBWRfTqtYnWWBCljrzoqw6H\nG35u+ickY2CUSiU2btyIjRs3BpZFRkaivt53hcmGhgakpKTgiy++wJw5c6BUKmE2mxEfH4+SkhKk\np6cHq2kUAhqViJ8umozn3z6CQydq8Yfcg7hv0WRoVPwmRhTOlHIl4vWxiNfH9lrX5mr3hRt/r43N\nUR24f7SuGEfrintsLxNksKjNfY65MasjIBPYc0tdgvavhyiKEMWeL7927VpkZ2fDaDTCZDJh9erV\n2LRpE8xmc2Abs9mM6urqCwaYyEgtRDF4Kf1CiY8uzfq7Z+HJLQfwWcFZ/GHbITx810wYdX1fRbQv\nrI00sS7SFdraGDAKFgC9/547nK2oaLahosmGys7bJhsqmqtRWFuEwnO2F2UiRuijEWuwIs4Qg1hD\nDOL89w0qfdBm9A4mfm4uzaB+/d2wYQOeffZZTJs2DTk5OdiyZUuvbfpzRMtudwSjeQDYrTcYVlyf\nDni9+OxQBX7x9KdYvXQKIvTffKomayNNrIt0Sb02Rphh1JqRrs0Aug29aelw9Bxv0+rrvbG11KCs\nsaLX62hEjW9wsiYaMdpo/0Bl3+35plkINanXRiokcxr1sWPHMG3aNADArFmz8N5772HGjBkoLe0a\n3V5VVQWr9fLPo0PSIZMJWHFDBtQKOT4+UIbf/zUf9y+dAotJE+qmEZEE6BRaJJsSkGxK6LHc6/Wi\nqaMZVS3VsLVWw+aoQZXDd1vWdBanG8/0eq0IlQlWbbdgo7EgRmuFWR0xbMbbDFWDGmAsFgtKSkqQ\nlpaGw4cPIzExETNmzMCf/vQn/PjHP4bdbofNZkNaWtpgNotCQCYIuGP+aKhVIv7x+Sl/iJmKEWZe\n7pyI+iYIAoxKA4xKA0ZHpvRY5/a4UddWjyqHDTZ/z40v3FSj2F6CYntJj+3lghwWTVS3Hht/D44u\nGgZFeB6SGm6CFmCOHDmCnJwclJeXQxRF7NixAw8//DAefPBBKBQKmEwmPPLIIzAajVi8eDGys7Mh\nCALWr18PGa8TMiwIgoCFc1OgUcrxxq4T+P1rB7B66VSMsupD3TQiCjNymRzR2ihEa6N6rWt3O1Hd\nrbeme+9NlcPWa3u1XA2r1tLjcFSMNhrRGguvTCwhnErgHDwuGRo788uw+cNiaFUifrZkMlLjTL22\nYW2kiXWRLtbmwrxeL5o7Wnr01tj8422qHTVwed29nmNSGnuNs4nRRiNKbR7QISnWpn8kMwaG6Hyu\nyRwJlVKOV94/ise3HsR9t01CRmJkqJtFREOYIAgwKPUwKPVIjUjqsc7j9aCurR42R3VXz43/fl8X\n75MJMlg0Zl+40UR368GJhlFp4CGpIGAPzDmYikPrwDEbXnynEDKZgHtvnYBJqZbAOtZGmlgX6WJt\ngsPp7gicIRXoufEHnBZX77Nk1XJVV4+NxnebGDMCrhYBOoUOeoWWA4rPg7NRDwA/8KF35GQtnn3r\nMNweL+6+eTyyMnxnpbE20sS6SBdrM/g6D0md23NT3VqDDo/rvM9Ty9XQK7TQKXXQKbTQK3TQK3z3\ndd3u+26HT+jhISQKKxNSovB/S6bgqTcK8OI7R9DmzMCcSXGhbhYR0TfSK3TQm3RIMSX2WO7xemBv\nawgMIHaLTlQ32NHc0YLmDgdaOlrQ0uFAeXMFXBcIOt0N99DDHphz8BuLdJRWNOLJ3INoaXNh2fzR\nuOOGcayNBPEzI12sjXSdrzZerxftbida/KGm2R9sfLddYad76Gl2Nvc54Lgv4RZ62ANDYSk51ohf\n3pmJJ7YexJaPj6PZ6cb100Zy/iQiGrIEQYBaVEEtqhCl6d+JDH2Fns7g01foaXa2oLzp7GULPamm\nJMTpR1zKr31R2ANzDn5jkZ4quwN/yC2Arb4VJp0St89LxcwJIyDjqH5J4GdGulgb6Qp1bbpCT1cP\nz7mhp8djp68HqK/QM0JrxUMz7g9KOzmIdwBCvVNR35wdbnx6pArb/lUMp8uDlDgj7lwwBsmxxlA3\nbdjjZ0a6WBvpCsfadA893Q9txepiMMoQH5T35CEkCntKhRx3XJeOqSlmvL6zBHlFNmx4dT9mT4zF\nbfNSYRrAjNZERDRwPQ9vmUPdHAYYCi9RJjX+97sT8K2v7fjrR8fx2eEKHCi24TuzkjF/+kiIck5D\nQUQ0HPCvPYWl9IRIrPvBdCy/bgxkgoDXd5bg16/sw+GTtaFuGhERDQIGGApbcpkM12SOxO/umYlr\nMuN9g31fL8DT2w6hyt77aphERDR08BAShT29RoHl16Vj3pR4bPmoGAdLanCktBbXZSXg27MSoVZy\nNyciGmrYA0NDxiirHr9YNhU/umU8jDolPth7Gmtf3os9RyoRhifbERHRBTDA0JAiCAKuGBuD3941\nAzdflYSWNhc2/uMr/O61fJyqbAx184iI6DJhgKEhSaWQ47tzUvDb/7kS09KjUVLegA1/3o8///Mo\nGlucoW4eERFdIg4OoCHNEqHBvbdOxNFTddjy8XF8WlCBvKJq3DI7Gd/KjOdp10REYYp/vWlYGJtk\nxvr/zsKdC8ZAALD1X8ex7o/7UFhaF+qmERHRRWCAoWFDLpPh2mkj8bt7ZmDe1HhU1jrwRO5BPPPm\nIdjqW0PdPCIiGgAeQqJhx6BV4nvXp2PelDhs+agYXx6vweGTdfivK0fhphlJUClDN3U8ERH1D3tg\naNhKiDHgl3dm4p6bx8OgVeAfn5/G2o17sfcrnnZNRCR1DDA0rAmCgCvHxeCRu2bg27OS0OTowMvv\nfoXf/zUfpyvDa6ZYIqLhhAGGCIBKKcfCuSn4zV1XInNMNI6XNeD//TkPf9lehCYHT7smIpIajoEh\n6sYaocGqhRNReKoOf/v4OHYdPIt9R2347pxkXJMZD7mMmZ+ISAr415ioD+OTzFj/gyzcce1oeAFs\n+fg41v8pD0dP8bRrIiIpYIAhOg9RLsOCrFH43T0zMHdyHM5Wt+CxrQfx3FuHUcPTromIQoqHkIi+\ngVGrxIobMjBvahy2fHQcB4qrcehkLW64MgE3zEiESsHTromIBht7YIj6KWmEEQ9kZ+Ku74yDTi3i\n3f+cwq827sW+o1U87ZqIaJAFtQemuLgYK1euxIoVK5CdnY2f/OQnsNvtAID6+npMmTIFGzZswKZN\nm7B9+3YIgoBVq1bh6quvDmaziC6aIAiYOX4Epo624P09p7Fj39d48Z1C7Mwvx7IFYzDKqg91E4mI\nhoWgBRiHw4ENGzZg5syZgWVPP/104P4DDzyARYsW4cyZM/jggw+wdetWNDc3Y9myZZg9ezbkcnbL\nk3SplSJuuzoVcybFYuu/SnCwpAbr/7QP86bE49a5KdBrFKFuIhHRkBa0Q0hKpRIbN26E1Wrtte7k\nyZNoamrCpEmT8MUXX2DOnDlQKpUwm82Ij49HSUlJsJpFdFlZI7X4ye2T8H+LJyMmUoudX5bjgZf2\n4N/5ZXB7PKFuHhHRkBW0HhhRFCGKfb/8X/7yF2RnZwMAampqYDabA+vMZjOqq6uRnp5+3teOjNRC\nFIPXQxMdbQjaa9OlkWptrok2YPa0BLz/n5P424fH8NqHxfjscCXu/u5ETEyzhLp5QSfVuhBrI2Ws\nzaUZ9LOQnE4nDhw4gPXr1/e5vj+DIe12x2VuVZfoaAOqq3kJeSkKh9pcNS4GExIj8dYnJ/DZoQqs\nfeE/mJ5hxZJr0hBlUoe6eUERDnUZrlgb6WJt+udCIW/QA0xeXh4mTZoUeGy1WlFaWhp4XFVV1edh\nJ6JwYdIp8YMbx2Le1Hhs+agY+4tsOFRSgxtnJOK/rkyAkqddExFdskE/jfrw4cPIyMgIPJ4xYwZ2\n7doFp9OJqqoq2Gw2pKWlDXaziC675FgjHlg+Df/z7bHQqES8/VkpfrXxC+wvsvG0ayKiSxS0Hpgj\nR44gJycH5eXlEEURO3bswDPPPIPq6mokJCQEtouLi8PixYuRnZ0NQRCwfv16yDjfDA0RMkHArAmx\nmDo6Gv/Ycwof7juD598+gtR4I2ZNiMW09GgYtcpQN5OIKOwI3jD8KhjM44Y8LildQ6E2VXUO5P7b\nd9o14As4GYkRuGJsDDLHRIfl6ddDoS5DFWsjXaxN/0hqDAzRcBZj9p12XdfYhrwiG/KKbPjqlB1f\nnbJj845jGJsUiawMKzLHREOnDr8wQ0Q0WNgDcw6mYukaqrWpqW9F3jEb8o7acKrS9/vJZQLGJ5uR\nlWHF1NHR0Kql+11jqNZlKGBtpIu16R/2wBBJmCVCgxuuTMQNVybCZncEemYOnajFoRO1EOVFmJAc\nhSvGWjE5zQKNih9bIiL+JSSSEGukFjfNTMJNM5NQWecPM0ercLCkBgdLaqAQZZiUEoWssVZMTrVA\npeQp2UQ0PDHAEEnUCLMW35mVhO/MSsLZmhbkFdmw72gVDhRX40BxNZSiDJPSLLgiw4qJqVFQ8foy\nRDSMMMAQhYE4iw63zE7GLbOTUVbdjLyjNuwrsmG//0elkGNyWhSuGBuDiSlmKII41QYRkRQwwBCF\nmZHReoyM1uO7c5JxxtbsP8xkwz7/j1opx9TRFmRlxGB8shkKkddVIqKhhwGGKEwJgoCEGAMSYgxY\nODcFX1c1Y9/RKuQV2bCnsAp7CqugUYnIHG1B1tgYjEuKhChnmCGioYEBhmgIEAQBiSMMSBxhwO3z\nUlFa0YS8Il+Y+c+RSvznSCV0ahGZY6KRNdaKjASGGSIKbwwwREOMIAhIiTMiJc6IRdek4eTZRuw7\nWoX9RTbsPlSB3YcqoNcoMC09GlkZVqQnREDO6TuIKMwwwBANYTJBQFq8CWnxJiy9djRKyhqQd9SG\n/cds+OTgWXxy8CyMWgWmpVtxxVgrRo+MgEwmhLrZRETfiAGGaJiQCQLGjIrAmFERuGP+aBSfqUde\nkS/M7PyyHDu/LIdJr8T0dCuyMqxIG2mCTGCYISJpYoAhGoZkMgEZiZHISIzEsgWjcezreuw7akN+\ncTX+daAM/zpQhkiDCtP9PTMpcUYIDDNEJCEMMETDnFwmw7gkM8YlmZF93RgUnbZjX5EN+ceq8dH+\nM/ho/xlEGVWYnmHFFWNjkDTCwDBDRCHHyRzPwQm2pIu1GVwutwdfnapD3lEb8o/XoLXdBQCwmNTI\nGmvFFRkxSIjRw2o1si4Sxc+MdLE2/XOhyRwZYM7BnUq6WJvQ6XB5UFhah31FVfjyeA3anW4AgDVS\ngzlT4mE1qREbpUVMpJYXzpMQfmaki7XpH85GTUSXRCHKMGW0BVNGW+DscONIaR32Ha1CQUkt3txZ\nEthOJgiIjlAjNkqHWIsWcVE63/0oLWfRJqLLin9RiGhAlAo5MsdEI3NMNNo73LA1OvHViWpU1DpQ\nUduCilqHf/bsns+LNKgQG6VFbJQOcf7bWIsORq2CY2qIaMAYYIjooqkUcmRmWDEqStNjeaPDiYoa\nX5g56w81FbUt+OqUHV+dsvfYVqcWA700sVE6xFl8t1EmNU/jJqLzYoAhosvOqFXCmKBEekJkj+Wt\n7S5U1nX11Jz1h5yTZxtRUt7QY1ulKMMIsxaxFl+4iYvSYQTH2RCFhNfrhaPdhcYWJxqanWh0dN0m\nxhgwPcM66G1igCGiQaNRiUiONSI51thjucvtQZW91d9r09VzU1nrwNe25h7bcpwN0eXh9XrR5nT7\nQkmLM3Dru9+OxpYONLS0B9a53H2f8zPKqmeAIaLhSZTLEG/RId6i67Hc4/WirqENZwPja1p892ta\nOM6G6DzaO9xdgSTQW9LeK6g0tjjhdHku+FqiXAaTToFRVgNMOiWM/h+T/8eoU2JktH6QfrNz2haS\ndyUi6geZIMASoYElQoNJqVGB5V6vF02Ojh6BpvM+x9nQUNTh6gwlvl6RvntNfLedlzk4H7lMgFGn\nRKxFFwghpj6CiUmnhEYlSjb8M8AQUdgRBCHwTfB842w6x9d0Bpv+jrOJ0KugUYvQqUVoVSKUCvlg\n/mo0jLjcHjQ5/Idpmp199pB0hpPOC0mejyD4xp5ZIzTnDSNGnRImvQpatTgkgjsDDBENKRccZ1Pn\n6HlmVE0LKut6j7PpTpTLoO0WaLRqBbRq0fejEqHrfKzyb9NtvUY1NP6hoL55PF60Ol1obXPB0e5C\na3vXbWu7u2tZW+eyrm2aW11ocji/8T30GgXMRhVMOkOvnhLfrQomnRJ6jWLYzSTPAENEw4IolyE+\nWo/4c47Xdx9nU1nbgkZHBxztLjjaOuBoc6GlrfMfnA7Y7K1we/p/8XIBgFrVPfx0CzjnBh5/GGLv\nz+DweL1oa3f3CBWO9nOCRlv3UNJ72286VNMXuUyARiUi0qhCvEV7zpgSVY9wYtAqIMp5xt35MMAQ\n0bB2vnE2ffF6vXB2eNDS1hlyXP6Q0/Oxw/+4pfNxewds9a1oG+A/eKJc5g85nT0+CujUYreQc+He\nn6Gq8+yZ1nNCR/eg0b3Xo/c2LrS1uzHQeXRkggCNSg6NSkRMhAYalRj46fx/rlWJgW20Kl+ttN22\nU4oyCILAqQQug6G7hxMRXWaCIECllEOllMN8Ec93ezz+QDOA8NPWgSbHxfX+KBRyAF4IEHwL/Mt9\nR7WEcx6jx2BNoespgCCg21P8z+lrWyHwuGt515OEbiu7v2/ne5/7HoAQ2Mbr9Xb1gjhdGOgsfoIA\naJS+EBFl1ECrkkOrVgTCRlf46AojPZfLoVLIJTugdTgKaoApLi7GypUrsWLFCmRnZ6OjowNr1qzB\n6dOnodPp8PTTT8NkMuHdd9/Fq6++CplMhsWLF2PRokXBbBYRUUjIZTIYtEoYtMoBP9fr9aK9w90V\ndNr94eecx63dDnt5AHR0uNHZ1eCFF/7/ugUAr291YBvfe3W9r3+Vf1nntv5nBV6n++t5vOe8vrf3\ne6BzWY/X6ON9A8/w9X5EGlWIU+l8h+RU3YOGvM8ej877aiXDx1ATtADjcDiwYcMGzJw5M7Ds9ddf\nR2RkJJ544gnk5uZi//79mDlzJp577jls27YNCoUCt99+OxYsWICIiIhgNY2IKOwIggC1UoRaKcJs\n/ObtAc54TENb0EYHKZVKbNy4EVZr19X5du7ciZtvvhkAsGTJElx77bUoKCjAxIkTYTAYoFarkZmZ\nifz8/GA1i4iIiIaAoPXAiKIIUez58uXl5fj000/x2GOPwWKxYN26daipqYHZ3HU02Ww2o7q6+oKv\nHRmphSgGb3R+dLQhaK9Nl4a1kSbWRbpYG+libS7NoA7i9Xq9SE5OxqpVq/D888/jpZdewrhx43pt\n803sdkewmsguVwljbaSJdZEu1ka6WJv+uVDIG9QTzC0WC7KysgAAs2fPRklJCaxWK2pqagLb2Gy2\nHoediIiIiM41qAFm7ty52L17NwCgsLAQycnJmDx5Mg4fPozGxka0tLQgPz8f06dPH8xmERERUZgJ\n2iGkI0eOICcnB+Xl5RBFETt27MDjjz+O3/72t9i2bRu0Wi1ycnKgVquxevVq/PCHP4QgCLj33nth\nMPC4IBEREZ2f4O3PoBOJCeZxQx6XlC7WRppYF+libaSLtekfyYyBISIiIrocGGCIiIgo7DDAEBER\nUdhhgCEiIqKwwwBDREREYYcBhoiIiMJOWJ5GTURERMMbe2CIiIgo7DDAEBERUdhhgCEiIqKwwwBD\nREREYYcBhoiIiMIOAwwRERGFHQaYbh555BEsWbIES5cuxaFDh0LdHOrm0UcfxZIlS3Dbbbfhww8/\nDHVzqJu2tjbMnz8fb731VqibQt28++67uPnmm7Fw4ULs2rUr1M0hAC0tLVi1ahWWL1+OpUuXYvfu\n3aFuUlgTQ90AdYVg7QAABetJREFUqdi3bx9Onz6N3NxcnDhxAmvXrkVubm6om0UA9u7di+PHjyM3\nNxd2ux233norrrvuulA3i/xeeOEFmEymUDeDurHb7Xjuuefw5ptvwuFw4JlnnsG8efNC3axh7+9/\n/zuSk5OxevVqVFVV4fvf/z62b98e6maFLQYYvz179mD+/PkAgNTUVDQ0NKC5uRl6vT7ELaOsrCxM\nmjQJAGA0GtHa2gq32w25XB7iltGJEydQUlLCfxwlZs+ePZg5cyb0ej30ej02bNgQ6iYRgMjISBw7\ndgwA0NjYiMjIyBC3KLzxEJJfTU1Nj53JbDajuro6hC2iTnK5HFqtFgCwbds2zJ07l+FFInJycrBm\nzZpQN4POUVZWhra2NvzoRz/CsmXLsGfPnlA3iQDcdNNNOHv2LBYsWIDs7Gz88pe/DHWTwhp7YM6D\nMyxIz8cff4xt27bhj3/8Y6ibQgDefvttTJkyBaNGjQp1U6gP9fX1ePbZZ3H27Fl873vfw86dOyEI\nQqibNay98847iIuLwyuvvIKioiKsXbuWY8cuAQOMn9VqRU1NTeCxzWZDdHR0CFtE3e3evRsvvvgi\nNm3aBIPBEOrmEIBdu3bhzJkz2LVrFyorK6FUKjFixAjMmjUr1E0b9qKiojB16lSIooiEhATodDrU\n1dUhKioq1E0b1vLz8zF79mwAQEZGBmw2Gw+HXwIeQvK76qqrsGPHDgBAYWEhrFYrx79IRFNTEx59\n9FG89NJLiIiICHVzyO+pp57Cm2++iddffx2LFi3CypUrGV4kYvbs2di7dy88Hg/sdjscDgfHW0hA\nYmIiCgoKAADl5eXQ6XQML5eAPTB+mZmZGD9+PJYuXQpBELBu3bpQN4n8PvjgA9jtdvz0pz8NLMvJ\nyUFcXFwIW0UkXTExMbj++uuxePFiAMCDDz4ImYzfV0NtyZIlWLt2LbKzs+FyubB+/fpQNymsCV4O\n9iAiIqIww0hOREREYYcBhoiIiMIOAwwRERGFHQYYIiIiCjsMMERERBR2GGCIKKjKysowYcIELF++\nPDAL7+rVq9HY2Njv11i+fDncbne/t7/jjjvwxRdfXExziShMMMAQUdCZzWZs3rwZmzdvxtatW2G1\nWvHCCy/0+/mbN2/mBb+IqAdeyI6IBl1WVhZyc3NRVFSEnJwcuFwudHR04Ne//jXGjRuH5cuXIyMj\nA0ePHsWrr76KcePGobCwEE6nEw899BAqKyvhcrlwyy23YNmyZWhtbcXPfvYz2O12JCYmor29HQBQ\nVVWF+++/HwDQ1taGJUuW4Pbbbw/lr05ElwkDDBENKrfbjY8++gjTpk3Dz3/+czz33HNISEjoNbmd\nVqvFa6+91uO5mzdvhtFoxBNPPIG2tjbceOONmDNnDj7//HOo1Wrk5ubCZrPh2muvBQD885//REpK\nCh5++GG0t7fjjTfeGPTfl4iCgwGGiIKurq4Oy5cvBwB4PB5Mnz4dt912G55++mn86le/CmzX3NwM\nj8cDwDe9x7kKCgqwcOFCAIBarcaECRNQWFiI4uJiTJs2DYBvYtaUlBQAwJw5c7BlyxasWbMGV199\nNZYsWRLU35OIBg8DDBEFXecYmO6ampqgUCh6Le+kUCh6LRMEocdjr9cLQRDg9Xp7zPXTGYJSU1Px\n/vvvIy8vD9u3b8err76KrVu3XuqvQ0QSwEG8RBQSBoMBI0eOxCeffAIAKC0txbPPPnvB50yePBm7\nd+8GADgcDhQWFmL8+PFITU3Fl19+CQCoqKhAaWkpAOC9997D4cOHMWvWLKxbtw4VFRVwuVxB/K2I\naLCwB4aIQiYnJwe/+c1v8PLLL8PlcmHNmjUX3H758uV46KGHcOedd8LpdGLlypUYOXIkbrnlFvz7\n3//GsmXLMHLkSEycOBEAkJaWhnXr1kGpVMLr9eKuu+6CKPLPHtFQwNmoiYiIKOzwEBIRERGFHQYY\nIiIiCjsMMERERBR2GGCIiIgo7DDAEBERUdhhgCEiIqKwwwBDREREYYcBhoiIiMLO/wdO/53WMeSg\nOwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "metadata": { + "id": "65sin-E5NmHN", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Task 5: Evaluate on Test Data\n", + "\n", + "**In the cell below, load in the test data set and evaluate your model on it.**\n", + "\n", + "We've done a lot of iteration on our validation data. Let's make sure we haven't overfit to the pecularities of that particular sample.\n", + "\n", + "Test data set is located [here](https://download.mlcc.google.com/mledu-datasets/california_housing_test.csv).\n", + "\n", + "How does your test performance compare to the validation performance? What does this say about the generalization performance of your model?" + ] + }, + { + "metadata": { + "id": "icEJIl5Vp51r", + "colab_type": "code", + "cellView": "both", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "outputId": "50e9f6e2-9c9c-44cb-c747-92396e6d1d47" + }, + "cell_type": "code", + "source": [ + "california_housing_test_data = pd.read_csv(\"https://download.mlcc.google.com/mledu-datasets/california_housing_test.csv\", sep=\",\")\n", + "#\n", + "# YOUR CODE HERE\n", + "#\n", + "test_examples = preprocess_features(california_housing_test_data)\n", + "test_targets = preprocess_targets(california_housing_test_data)\n", + "\n", + "predict_test_input_fn = lambda: my_input_fn(\n", + " test_examples, \n", + " test_targets[\"median_house_value\"], \n", + " num_epochs=1, \n", + " shuffle=False)\n", + "\n", + "test_predictions = linear_regressor.predict(input_fn=predict_test_input_fn)\n", + "test_predictions = np.array([item['predictions'][0] for item in test_predictions])\n", + "\n", + "root_mean_squared_error = math.sqrt(\n", + " metrics.mean_squared_error(test_predictions, test_targets))\n", + "\n", + "print(\"Final RMSE (on test data): %0.2f\" % root_mean_squared_error)" + ], + "execution_count": 15, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Final RMSE (on test data): 160.90\n" + ], + "name": "stdout" + } + ] + }, + { + "metadata": { + "id": "yTghc_5HkJDW", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Solution\n", + "\n", + "Click below for the solution." + ] + }, + { + "metadata": { + "id": "_xSYTarykO8U", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "california_housing_test_data = pd.read_csv(\"https://download.mlcc.google.com/mledu-datasets/california_housing_test.csv\", sep=\",\")\n", + "\n", + "test_examples = preprocess_features(california_housing_test_data)\n", + "test_targets = preprocess_targets(california_housing_test_data)\n", + "\n", + "predict_test_input_fn = lambda: my_input_fn(\n", + " test_examples, \n", + " test_targets[\"median_house_value\"], \n", + " num_epochs=1, \n", + " shuffle=False)\n", + "\n", + "test_predictions = linear_regressor.predict(input_fn=predict_test_input_fn)\n", + "test_predictions = np.array([item['predictions'][0] for item in test_predictions])\n", + "\n", + "root_mean_squared_error = math.sqrt(\n", + " metrics.mean_squared_error(test_predictions, test_targets))\n", + "\n", + "print(\"Final RMSE (on test data): %0.2f\" % root_mean_squared_error)" + ], + "execution_count": 0, + "outputs": [] + } + ] +} \ No newline at end of file